umbrellio-utils 1.3.0 → 1.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (16) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +1 -1
  3. data/.rubocop.yml +1 -1
  4. data/Gemfile +21 -22
  5. data/Gemfile.lock +99 -103
  6. data/lib/umbrellio_utils/control.rb +1 -1
  7. data/lib/umbrellio_utils/database.rb +33 -13
  8. data/lib/umbrellio_utils/formatting.rb +1 -1
  9. data/lib/umbrellio_utils/jobs.rb +86 -0
  10. data/lib/umbrellio_utils/misc.rb +1 -1
  11. data/lib/umbrellio_utils/parsing.rb +15 -1
  12. data/lib/umbrellio_utils/vault.rb +1 -1
  13. data/lib/umbrellio_utils/version.rb +1 -1
  14. data/lib/umbrellio_utils.rb +3 -2
  15. data/umbrellio_utils.gemspec +1 -1
  16. metadata +5 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d17fd4116bc29709f96f27f812fdf8f9b5e977a1f0982ec6ca39616c6458e879
4
- data.tar.gz: 64de97d755d6759be25ff38cc07d1680d4501f355982e22003f59520c21c01bb
3
+ metadata.gz: 16502aeac71a5a8860b3dcd6ea3acbf3714ee86c243b2cfa78ca3356805211dd
4
+ data.tar.gz: 8c9744334c49cf933d1dd2f7b60b5d22cd00468450bbc90971e083479ed9e7db
5
5
  SHA512:
6
- metadata.gz: f32e5a5768428d0a2b7d3a8b0737916947088c35490ea8da10bddad4296683eac86612494cabae0dff9a50b6ec4aeb29dcd2adba7677133079282ae4fea7d7a0
7
- data.tar.gz: 559fba254624cee4c16bb6f078cff3ea7c3b40f64579429d663fb89ff4c90e6afff20ec9f8d76cbb94c134598ab833c09cc5a74d61553cfd45e757145268553d
6
+ metadata.gz: 80a4cf07687c9b3c8aa4680533e14977ea7730834ba78393fef5121db2636c971072795145aa42305e41b181a29d48379e9fcb486abd1dd5539e6164441ca0a1
7
+ data.tar.gz: d52ecd94d480776572e2d70be1963b536d6032b2b7c8a9c9547884911825262f405bf6f9d40bc7c1e6577ce7fddd3ef4602395730de370d77b06b5a9174823bb
data/.github/workflows/test.yml CHANGED
@@ -15,7 +15,7 @@ jobs:
15
15
  strategy:
16
16
  fail-fast: false
17
17
  matrix:
18
- ruby: ["3.0", "3.1", "3.2", "3.3"]
18
+ ruby: ["3.1", "3.2", "3.3"]
19
19
 
20
20
  services:
21
21
  postgres:
data/.rubocop.yml CHANGED
@@ -3,7 +3,7 @@ inherit_gem:
3
3
 
4
4
  AllCops:
5
5
  DisplayCopNames: true
6
- TargetRubyVersion: 3.0
6
+ TargetRubyVersion: 3.1
7
7
  SuggestExtensions: false
8
8
 
9
9
  Naming/MethodParameterName:
data/Gemfile CHANGED
@@ -5,25 +5,24 @@ source "https://rubygems.org"
5
5
  # Specify your gem's dependencies in umbrellio_utils.gemspec
6
6
  gemspec
7
7
 
8
- group :development, :test do
9
- gem "activesupport"
10
- gem "bundler"
11
- gem "ci-helper"
12
- gem "http"
13
- gem "nokogiri"
14
- gem "nori"
15
- gem "pg"
16
- gem "pry"
17
- gem "rake"
18
- gem "rspec"
19
- gem "rspec-json_matcher"
20
- gem "rubocop-config-umbrellio"
21
- gem "semantic_logger"
22
- gem "sequel"
23
- gem "sequel-batches"
24
- gem "simplecov"
25
- gem "simplecov-lcov"
26
- gem "table_sync"
27
- gem "timecop"
28
- gem "yard"
29
- end
8
+ gem "activesupport"
9
+ gem "bundler"
10
+ gem "ci-helper"
11
+ gem "http"
12
+ gem "net-pop"
13
+ gem "nokogiri"
14
+ gem "nori"
15
+ gem "pg"
16
+ gem "pry"
17
+ gem "rake"
18
+ gem "rspec"
19
+ gem "rspec-json_matcher"
20
+ gem "rubocop-config-umbrellio"
21
+ gem "semantic_logger"
22
+ gem "sequel"
23
+ gem "sequel-batches"
24
+ gem "simplecov"
25
+ gem "simplecov-lcov"
26
+ gem "table_sync"
27
+ gem "timecop"
28
+ gem "yard"
data/Gemfile.lock CHANGED
@@ -1,86 +1,82 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- umbrellio-utils (1.3.0)
4
+ umbrellio-utils (1.5.0)
5
5
  memery (~> 1)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
- actioncable (7.1.3.4)
11
- actionpack (= 7.1.3.4)
12
- activesupport (= 7.1.3.4)
10
+ actioncable (7.2.1)
11
+ actionpack (= 7.2.1)
12
+ activesupport (= 7.2.1)
13
13
  nio4r (~> 2.0)
14
14
  websocket-driver (>= 0.6.1)
15
15
  zeitwerk (~> 2.6)
16
- actionmailbox (7.1.3.4)
17
- actionpack (= 7.1.3.4)
18
- activejob (= 7.1.3.4)
19
- activerecord (= 7.1.3.4)
20
- activestorage (= 7.1.3.4)
21
- activesupport (= 7.1.3.4)
22
- mail (>= 2.7.1)
23
- net-imap
24
- net-pop
25
- net-smtp
26
- actionmailer (7.1.3.4)
27
- actionpack (= 7.1.3.4)
28
- actionview (= 7.1.3.4)
29
- activejob (= 7.1.3.4)
30
- activesupport (= 7.1.3.4)
31
- mail (~> 2.5, >= 2.5.4)
32
- net-imap
33
- net-pop
34
- net-smtp
16
+ actionmailbox (7.2.1)
17
+ actionpack (= 7.2.1)
18
+ activejob (= 7.2.1)
19
+ activerecord (= 7.2.1)
20
+ activestorage (= 7.2.1)
21
+ activesupport (= 7.2.1)
22
+ mail (>= 2.8.0)
23
+ actionmailer (7.2.1)
24
+ actionpack (= 7.2.1)
25
+ actionview (= 7.2.1)
26
+ activejob (= 7.2.1)
27
+ activesupport (= 7.2.1)
28
+ mail (>= 2.8.0)
35
29
  rails-dom-testing (~> 2.2)
36
- actionpack (7.1.3.4)
37
- actionview (= 7.1.3.4)
38
- activesupport (= 7.1.3.4)
30
+ actionpack (7.2.1)
31
+ actionview (= 7.2.1)
32
+ activesupport (= 7.2.1)
39
33
  nokogiri (>= 1.8.5)
40
34
  racc
41
- rack (>= 2.2.4)
35
+ rack (>= 2.2.4, < 3.2)
42
36
  rack-session (>= 1.0.1)
43
37
  rack-test (>= 0.6.3)
44
38
  rails-dom-testing (~> 2.2)
45
39
  rails-html-sanitizer (~> 1.6)
46
- actiontext (7.1.3.4)
47
- actionpack (= 7.1.3.4)
48
- activerecord (= 7.1.3.4)
49
- activestorage (= 7.1.3.4)
50
- activesupport (= 7.1.3.4)
40
+ useragent (~> 0.16)
41
+ actiontext (7.2.1)
42
+ actionpack (= 7.2.1)
43
+ activerecord (= 7.2.1)
44
+ activestorage (= 7.2.1)
45
+ activesupport (= 7.2.1)
51
46
  globalid (>= 0.6.0)
52
47
  nokogiri (>= 1.8.5)
53
- actionview (7.1.3.4)
54
- activesupport (= 7.1.3.4)
48
+ actionview (7.2.1)
49
+ activesupport (= 7.2.1)
55
50
  builder (~> 3.1)
56
51
  erubi (~> 1.11)
57
52
  rails-dom-testing (~> 2.2)
58
53
  rails-html-sanitizer (~> 1.6)
59
- activejob (7.1.3.4)
60
- activesupport (= 7.1.3.4)
54
+ activejob (7.2.1)
55
+ activesupport (= 7.2.1)
61
56
  globalid (>= 0.3.6)
62
- activemodel (7.1.3.4)
63
- activesupport (= 7.1.3.4)
64
- activerecord (7.1.3.4)
65
- activemodel (= 7.1.3.4)
66
- activesupport (= 7.1.3.4)
57
+ activemodel (7.2.1)
58
+ activesupport (= 7.2.1)
59
+ activerecord (7.2.1)
60
+ activemodel (= 7.2.1)
61
+ activesupport (= 7.2.1)
67
62
  timeout (>= 0.4.0)
68
- activestorage (7.1.3.4)
69
- actionpack (= 7.1.3.4)
70
- activejob (= 7.1.3.4)
71
- activerecord (= 7.1.3.4)
72
- activesupport (= 7.1.3.4)
63
+ activestorage (7.2.1)
64
+ actionpack (= 7.2.1)
65
+ activejob (= 7.2.1)
66
+ activerecord (= 7.2.1)
67
+ activesupport (= 7.2.1)
73
68
  marcel (~> 1.0)
74
- activesupport (7.1.3.4)
69
+ activesupport (7.2.1)
75
70
  base64
76
71
  bigdecimal
77
- concurrent-ruby (~> 1.0, >= 1.0.2)
72
+ concurrent-ruby (~> 1.0, >= 1.3.1)
78
73
  connection_pool (>= 2.2.5)
79
74
  drb
80
75
  i18n (>= 1.6, < 2)
76
+ logger (>= 1.4.2)
81
77
  minitest (>= 5.1)
82
- mutex_m
83
- tzinfo (~> 2.0)
78
+ securerandom (>= 0.3)
79
+ tzinfo (~> 2.0, >= 2.0.5)
84
80
  addressable (2.8.7)
85
81
  public_suffix (>= 2.0.2, < 7.0)
86
82
  amazing_print (1.6.0)
@@ -98,15 +94,15 @@ GEM
98
94
  umbrellio-sequel-plugins (~> 0.14)
99
95
  coderay (1.1.3)
100
96
  colorize (1.1.0)
101
- concurrent-ruby (1.3.3)
97
+ concurrent-ruby (1.3.4)
102
98
  connection_pool (2.4.1)
103
99
  crass (1.0.6)
104
100
  date (3.3.4)
105
101
  diff-lcs (1.5.1)
106
- docile (1.4.0)
102
+ docile (1.4.1)
107
103
  domain_name (0.6.20240107)
108
104
  drb (2.2.1)
109
- dry-inflector (1.0.0)
105
+ dry-inflector (1.1.0)
110
106
  erubi (1.13.0)
111
107
  ffi (1.17.0)
112
108
  ffi (1.17.0-arm64-darwin)
@@ -123,22 +119,23 @@ GEM
123
119
  http-cookie (~> 1.0)
124
120
  http-form_data (~> 2.2)
125
121
  llhttp-ffi (~> 0.5.0)
126
- http-cookie (1.0.6)
122
+ http-cookie (1.0.7)
127
123
  domain_name (~> 0.5)
128
124
  http-form_data (2.3.0)
129
125
  i18n (1.14.5)
130
126
  concurrent-ruby (~> 1.0)
131
127
  io-console (0.7.2)
132
- irb (1.13.2)
128
+ irb (1.14.0)
133
129
  rdoc (>= 4.0.0)
134
130
  reline (>= 0.4.2)
135
131
  json (2.7.2)
136
- lamian (1.9.0)
132
+ lamian (1.10.0)
137
133
  rails (>= 4.2)
138
134
  language_server-protocol (3.17.0.3)
139
135
  llhttp-ffi (0.5.0)
140
136
  ffi-compiler (~> 1.0)
141
137
  rake (~> 13.0)
138
+ logger (1.6.1)
142
139
  loofah (2.22.0)
143
140
  crass (~> 1.0.2)
144
141
  nokogiri (>= 1.12.0)
@@ -153,47 +150,47 @@ GEM
153
150
  method_source (1.1.0)
154
151
  mini_mime (1.1.5)
155
152
  mini_portile2 (2.8.7)
156
- minitest (5.24.1)
157
- mutex_m (0.2.0)
158
- net-imap (0.4.14)
153
+ minitest (5.25.1)
154
+ net-imap (0.4.16)
159
155
  date
160
156
  net-protocol
161
157
  net-pop (0.1.2)
158
+ net-protocol
162
159
  net-protocol (0.2.2)
163
160
  timeout
164
161
  net-smtp (0.5.0)
165
162
  net-protocol
166
163
  nio4r (2.7.3)
167
- nokogiri (1.16.6)
164
+ nokogiri (1.16.7)
168
165
  mini_portile2 (~> 2.8.2)
169
166
  racc (~> 1.4)
170
- nokogiri (1.16.6-arm64-darwin)
167
+ nokogiri (1.16.7-arm64-darwin)
171
168
  racc (~> 1.4)
172
- nokogiri (1.16.6-x86_64-darwin)
169
+ nokogiri (1.16.7-x86_64-darwin)
173
170
  racc (~> 1.4)
174
- nokogiri (1.16.6-x86_64-linux)
171
+ nokogiri (1.16.7-x86_64-linux)
175
172
  racc (~> 1.4)
176
- nori (2.7.0)
173
+ nori (2.7.1)
177
174
  bigdecimal
178
- parallel (1.25.1)
179
- parser (3.3.3.0)
175
+ parallel (1.26.3)
176
+ parser (3.3.5.0)
180
177
  ast (~> 2.4.1)
181
178
  racc
182
- pg (1.5.6)
179
+ pg (1.5.7)
183
180
  pry (0.14.2)
184
181
  coderay (~> 1.1)
185
182
  method_source (~> 1.0)
186
183
  psych (5.1.2)
187
184
  stringio
188
- public_suffix (6.0.0)
185
+ public_suffix (6.0.1)
189
186
  rabbit_messaging (0.15.0)
190
187
  bunny (~> 2.0)
191
188
  lamian
192
189
  rails (>= 5.2)
193
190
  sneakers (~> 2.0)
194
191
  tainbox
195
- racc (1.8.0)
196
- rack (3.1.4)
192
+ racc (1.8.1)
193
+ rack (3.1.7)
197
194
  rack-session (2.0.0)
198
195
  rack (>= 3.0.0)
199
196
  rack-test (2.1.0)
@@ -201,20 +198,20 @@ GEM
201
198
  rackup (2.1.0)
202
199
  rack (>= 3)
203
200
  webrick (~> 1.8)
204
- rails (7.1.3.4)
205
- actioncable (= 7.1.3.4)
206
- actionmailbox (= 7.1.3.4)
207
- actionmailer (= 7.1.3.4)
208
- actionpack (= 7.1.3.4)
209
- actiontext (= 7.1.3.4)
210
- actionview (= 7.1.3.4)
211
- activejob (= 7.1.3.4)
212
- activemodel (= 7.1.3.4)
213
- activerecord (= 7.1.3.4)
214
- activestorage (= 7.1.3.4)
215
- activesupport (= 7.1.3.4)
201
+ rails (7.2.1)
202
+ actioncable (= 7.2.1)
203
+ actionmailbox (= 7.2.1)
204
+ actionmailer (= 7.2.1)
205
+ actionpack (= 7.2.1)
206
+ actiontext (= 7.2.1)
207
+ actionview (= 7.2.1)
208
+ activejob (= 7.2.1)
209
+ activemodel (= 7.2.1)
210
+ activerecord (= 7.2.1)
211
+ activestorage (= 7.2.1)
212
+ activesupport (= 7.2.1)
216
213
  bundler (>= 1.15.0)
217
- railties (= 7.1.3.4)
214
+ railties (= 7.2.1)
218
215
  rails-dom-testing (2.2.0)
219
216
  activesupport (>= 5.0.0)
220
217
  minitest
@@ -222,10 +219,10 @@ GEM
222
219
  rails-html-sanitizer (1.6.0)
223
220
  loofah (~> 2.21)
224
221
  nokogiri (~> 1.14)
225
- railties (7.1.3.4)
226
- actionpack (= 7.1.3.4)
227
- activesupport (= 7.1.3.4)
228
- irb
222
+ railties (7.2.1)
223
+ actionpack (= 7.2.1)
224
+ activesupport (= 7.2.1)
225
+ irb (~> 1.13)
229
226
  rackup (>= 1.0.0)
230
227
  rake (>= 12.2)
231
228
  thor (~> 1.0, >= 1.2.2)
@@ -236,17 +233,16 @@ GEM
236
233
  rdoc (6.7.0)
237
234
  psych (>= 4.0.0)
238
235
  regexp_parser (2.9.2)
239
- reline (0.5.9)
236
+ reline (0.5.10)
240
237
  io-console (~> 0.5)
241
- rexml (3.3.1)
242
- strscan
238
+ rexml (3.3.7)
243
239
  rspec (3.13.0)
244
240
  rspec-core (~> 3.13.0)
245
241
  rspec-expectations (~> 3.13.0)
246
242
  rspec-mocks (~> 3.13.0)
247
- rspec-core (3.13.0)
243
+ rspec-core (3.13.1)
248
244
  rspec-support (~> 3.13.0)
249
- rspec-expectations (3.13.1)
245
+ rspec-expectations (3.13.2)
250
246
  diff-lcs (>= 1.2.0, < 2.0)
251
247
  rspec-support (~> 3.13.0)
252
248
  rspec-json_matcher (0.2.0)
@@ -267,7 +263,7 @@ GEM
267
263
  rubocop-ast (>= 1.31.1, < 2.0)
268
264
  ruby-progressbar (~> 1.7)
269
265
  unicode-display_width (>= 2.4.0, < 3.0)
270
- rubocop-ast (1.31.3)
266
+ rubocop-ast (1.32.3)
271
267
  parser (>= 3.3.1.0)
272
268
  rubocop-capybara (2.21.0)
273
269
  rubocop (~> 1.41)
@@ -301,10 +297,11 @@ GEM
301
297
  rubocop (~> 1.0)
302
298
  ruby-progressbar (1.13.0)
303
299
  ruby2_keywords (0.0.5)
300
+ securerandom (0.3.1)
304
301
  self_data (1.3.0)
305
- semantic_logger (4.15.0)
302
+ semantic_logger (4.16.0)
306
303
  concurrent-ruby (~> 1.0)
307
- sequel (5.81.0)
304
+ sequel (5.84.0)
308
305
  bigdecimal
309
306
  sequel-batches (2.0.2)
310
307
  sequel
@@ -329,8 +326,6 @@ GEM
329
326
  rbtree
330
327
  set (~> 1.0)
331
328
  stringio (3.1.1)
332
- strscan (3.1.0)
333
- symbiont-ruby (0.7.0)
334
329
  table_sync (6.5.0)
335
330
  memery
336
331
  rabbit_messaging (~> 0.13)
@@ -338,21 +333,21 @@ GEM
338
333
  self_data
339
334
  tainbox (2.1.2)
340
335
  activesupport
341
- thor (1.3.1)
336
+ thor (1.3.2)
342
337
  timecop (0.9.10)
343
338
  timeout (0.4.1)
344
339
  tzinfo (2.0.6)
345
340
  concurrent-ruby (~> 1.0)
346
- umbrellio-sequel-plugins (0.15.0.198)
341
+ umbrellio-sequel-plugins (0.16.0.211)
347
342
  sequel
348
- symbiont-ruby
349
343
  unicode-display_width (2.5.0)
344
+ useragent (0.16.10)
350
345
  webrick (1.8.1)
351
346
  websocket-driver (0.7.6)
352
347
  websocket-extensions (>= 0.1.0)
353
348
  websocket-extensions (0.1.5)
354
- yard (0.9.36)
355
- zeitwerk (2.6.16)
349
+ yard (0.9.37)
350
+ zeitwerk (2.6.18)
356
351
 
357
352
  PLATFORMS
358
353
  arm64-darwin-20
@@ -367,6 +362,7 @@ DEPENDENCIES
367
362
  bundler
368
363
  ci-helper
369
364
  http
365
+ net-pop
370
366
  nokogiri
371
367
  nori
372
368
  pg
@@ -386,4 +382,4 @@ DEPENDENCIES
386
382
  yard
387
383
 
388
384
  BUNDLED WITH
389
- 2.5.14
385
+ 2.5.18
data/lib/umbrellio_utils/control.rb CHANGED
@@ -21,7 +21,7 @@ module UmbrellioUtils
21
21
  def retry_on_unique_violation(
22
22
  times: Float::INFINITY, retry_on_all_constraints: false, checked_constraints: [], &block
23
23
  )
24
- retry_on(Sequel::UniqueConstraintViolation, times: times) do
24
+ retry_on(Sequel::UniqueConstraintViolation, times:) do
25
25
  DB.transaction(savepoint: true, &block)
26
26
  rescue Sequel::UniqueConstraintViolation => e
27
27
  constraint_name = Database.get_violated_constraint_name(e)
data/lib/umbrellio_utils/database.rb CHANGED
@@ -7,8 +7,8 @@ module UmbrellioUtils
7
7
  HandledConstaintError = Class.new(StandardError)
8
8
  InvalidPkError = Class.new(StandardError)
9
9
 
10
- def handle_constraint_error(constraint_name, &block)
11
- DB.transaction(savepoint: true, &block)
10
+ def handle_constraint_error(constraint_name, &)
11
+ DB.transaction(savepoint: true, &)
12
12
  rescue Sequel::UniqueConstraintViolation => e
13
13
  if constraint_name.to_s == get_violated_constraint_name(e)
14
14
  raise HandledConstaintError
@@ -22,8 +22,8 @@ module UmbrellioUtils
22
22
  error.result.error_field(PG::Result::PG_DIAG_CONSTRAINT_NAME)
23
23
  end
24
24
 
25
- def each_record(dataset, **options, &block)
26
- primary_key = primary_key_from(dataset, **options)
25
+ def each_record(dataset, primary_key: nil, **options, &block)
26
+ primary_key = primary_key_from(dataset, primary_key:)
27
27
 
28
28
  with_temp_table(dataset, **options) do |ids|
29
29
  rows = ids.map { |id| row(id.is_a?(Hash) ? id.values : [id]) }
@@ -31,11 +31,28 @@ module UmbrellioUtils
31
31
  end
32
32
  end
33
33
 
34
- def with_temp_table(dataset, page_size: 1_000, sleep: nil, **options)
35
- primary_key = primary_key_from(dataset, **options)
34
+ # Iterates over a dataset and yields batches of primary keys.
35
+ # First, a temporary table is created and populated with dataset primary keys.
36
+ # After that, a batch of rows is deleted from the temp table on each iteration
37
+ # and gets yielded to the caller.
38
+ # @option [Integer] page_size max size of each yielded PK batch
39
+ # @option [Integer] sleep interval to sleep between each iteration
40
+ # @option [Array] primary_key custom primary key to use for dataset
41
+ # @option [Symbol, String] temp_table_name custom name for temporary table,
42
+ # table is reused if already exists
43
+ def with_temp_table(
44
+ dataset,
45
+ page_size: 1_000,
46
+ sleep: nil,
47
+ primary_key: nil,
48
+ temp_table_name: nil
49
+ )
50
+ primary_key = primary_key_from(dataset, primary_key:)
36
51
  sleep_interval = sleep_interval_from(sleep)
37
52
 
38
- temp_table_name = create_temp_table(dataset, primary_key: primary_key)
53
+ temp_table_name = create_temp_table(
54
+ dataset, primary_key:, temp_table_name: temp_table_name&.to_sym
55
+ )
39
56
 
40
57
  pk_set = []
41
58
 
@@ -49,23 +66,26 @@ module UmbrellioUtils
49
66
 
50
67
  Kernel.sleep(sleep_interval) if sleep_interval.positive?
51
68
  end
52
- ensure
69
+
53
70
  DB.drop_table(temp_table_name)
54
71
  end
55
72
 
56
- def create_temp_table(dataset, **options)
73
+ def create_temp_table(dataset, primary_key: nil, temp_table_name: nil)
57
74
  time = Time.current
58
75
  model = dataset.model
59
- temp_table_name = :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}"
60
- primary_key = primary_key_from(dataset, **options)
76
+
77
+ temp_table_name ||= :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}"
78
+ return temp_table_name if DB.table_exists?(temp_table_name)
79
+
80
+ primary_key = primary_key_from(dataset, primary_key:)
61
81
 
62
82
  DB.create_table(temp_table_name, unlogged: true) do
63
83
  primary_key.each do |field|
64
84
  type = model.db_schema[field][:db_type]
65
- column field, type
85
+ column(field, type)
66
86
  end
67
87
 
68
- primary_key primary_key
88
+ primary_key(primary_key)
69
89
  end
70
90
 
71
91
  insert_ds = dataset.select(*qualified_pk(model.table_name, primary_key))
data/lib/umbrellio_utils/formatting.rb CHANGED
@@ -58,7 +58,7 @@ module UmbrellioUtils
58
58
  end
59
59
 
60
60
  def to_date_part_string(part)
61
- format("%<part>02d", part: part)
61
+ format("%<part>02d", part:)
62
62
  end
63
63
 
64
64
  #
data/lib/umbrellio_utils/jobs.rb ADDED
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ module UmbrellioUtils::Jobs
4
+ extend self
5
+
6
+ Worker = Struct.new(:name)
7
+ Capsule = Struct.new(:name, :worker, :weight)
8
+ Queue = Struct.new(:name, :capsule, :weight)
9
+ Entry = Struct.new(:capsule, :queues, :concurrency)
10
+
11
+ def workers
12
+ @workers ||= []
13
+ end
14
+
15
+ def capsules
16
+ @capsules ||= []
17
+ end
18
+
19
+ def queues
20
+ @queues ||= []
21
+ end
22
+
23
+ def register_worker(name)
24
+ workers << Worker.new(name)
25
+ end
26
+
27
+ def register_capsule(name, worker: :default, weight: 1)
28
+ workers.find { |x| x.name == worker } or raise "Worker not found: #{worker.inspect}"
29
+ capsules << Capsule.new(name, worker, weight)
30
+ end
31
+
32
+ def register_queue(name, capsule: :default, weight: 1)
33
+ capsules.find { |x| x.name == capsule } or raise "Capsule not found: #{capsule.inspect}"
34
+ queues << Queue.new(name, capsule, weight)
35
+ end
36
+
37
+ def retry_interval(error_count, min_interval:, max_interval:)
38
+ interval = min_interval * (1.3**(error_count - 3))
39
+ interval.clamp(min_interval, max_interval).round
40
+ end
41
+
42
+ def configure_capsules!(config, priority_level:, max_concurrency:)
43
+ entries = capsules_for(priority_level, max_concurrency)
44
+
45
+ unless entries.find { |x| x.capsule == :default }
46
+ entries.last.capsule = :default # Default capsule should always be present in sidekiq
47
+ end
48
+
49
+ entries.each do |entry|
50
+ config.capsule(entry.capsule) do |capsule|
51
+ capsule.queues = entry.queues
52
+ capsule.concurrency = entry.concurrency
53
+ end
54
+ end
55
+ end
56
+
57
+ def capsules_for(worker, max_concurrency)
58
+ capsules = self.capsules.select do |capsule|
59
+ next unless capsule.worker.to_s == worker.underscore.to_s
60
+ next unless queues.any? { |queue| queue.capsule == capsule.name }
61
+ true
62
+ end
63
+
64
+ total_weight = capsules.sum(&:weight)
65
+
66
+ result = capsules.filter_map do |capsule|
67
+ weight_coef = capsule.weight / total_weight.to_f
68
+ concurrency = (max_concurrency * weight_coef).to_i
69
+ concurrency = 1 unless concurrency > 1
70
+ queues = self.queues.select { |x| x.capsule == capsule.name }.map { |x| [x.name, x.weight] }
71
+ Entry.new(capsule.name, queues, concurrency)
72
+ end
73
+
74
+ raise "No queues found for worker #{worker.inspect}" if result.empty?
75
+
76
+ result
77
+ end
78
+
79
+ def validate_queue_name!(queue_name)
80
+ found = queues.any? do |queue|
81
+ queue.name.to_s == queue_name.to_s
82
+ end
83
+
84
+ raise "Unknown queue: #{queue_name.inspect}" unless found
85
+ end
86
+ end
data/lib/umbrellio_utils/misc.rb CHANGED
@@ -13,7 +13,7 @@ module UmbrellioUtils
13
13
  TableSync::Publishing::Batch.new(
14
14
  object_class: model_class,
15
15
  original_attributes: batch_for_sync.map(&:values),
16
- routing_key: routing_key,
16
+ routing_key:,
17
17
  ).publish_now
18
18
 
19
19
  sleep delay
data/lib/umbrellio_utils/parsing.rb CHANGED
@@ -20,7 +20,7 @@ module UmbrellioUtils
20
20
  xml.remove_namespaces!
21
21
  xml.xpath("//@*").remove if remove_attributes
22
22
 
23
- tags_converter = snakecase ? -> (tag) { tag.snakecase.to_sym } : -> (tag) { tag.to_sym }
23
+ tags_converter = snakecase ? -> (tag) { snakecase(tag).to_sym } : -> (tag) { tag.to_sym }
24
24
  nori = Nori.new(convert_tags_to: tags_converter, convert_dashes_to_underscores: false)
25
25
  nori.parse(xml.to_xml(save_with: Nokogiri::XML::Node::SaveOptions::NO_DECLARATION))
26
26
  end
@@ -72,5 +72,19 @@ module UmbrellioUtils
72
72
 
73
73
  phone.sanitized
74
74
  end
75
+
76
+ private
77
+
78
+ # See https://github.com/savonrb/nori/blob/main/lib/nori/string_utils.rb
79
+ def snakecase(string)
80
+ str = string.dup
81
+ str.gsub!("::", "/")
82
+ str.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
83
+ str.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
84
+ str.tr!(".", "_")
85
+ str.tr!("-", "_")
86
+ str.downcase!
87
+ str
88
+ end
75
89
  end
76
90
  end
data/lib/umbrellio_utils/vault.rb CHANGED
@@ -22,7 +22,7 @@ module UmbrellioUtils
22
22
  def write_to_kv(engine_path:, secret_path:, data:)
23
23
  full_data_path = File.join(engine_path, "data", secret_path)
24
24
  full_meta_path = File.join(engine_path, "metadata", secret_path)
25
- ::Vault.logical.write(full_data_path, data: data)
25
+ ::Vault.logical.write(full_data_path, data:)
26
26
  ::Vault.logical.write(full_meta_path, id: secret_path, max_versions: 1, cas_required: false)
27
27
  end
28
28
  end
data/lib/umbrellio_utils/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module UmbrellioUtils
4
- VERSION = "1.3.0"
4
+ VERSION = "1.5.0"
5
5
  end
data/lib/umbrellio_utils.rb CHANGED
@@ -41,8 +41,8 @@ module UmbrellioUtils
41
41
  }
42
42
  end
43
43
 
44
- def synchronize(&block)
45
- GLOBAL_MUTEX.owned? ? yield : GLOBAL_MUTEX.synchronize(&block)
44
+ def synchronize(&)
45
+ GLOBAL_MUTEX.owned? ? yield : GLOBAL_MUTEX.synchronize(&)
46
46
  end
47
47
  end
48
48
 
@@ -53,6 +53,7 @@ require_relative "umbrellio_utils/control"
53
53
  require_relative "umbrellio_utils/database"
54
54
  require_relative "umbrellio_utils/formatting"
55
55
  require_relative "umbrellio_utils/http_client"
56
+ require_relative "umbrellio_utils/jobs"
56
57
  require_relative "umbrellio_utils/misc"
57
58
  require_relative "umbrellio_utils/parsing"
58
59
  require_relative "umbrellio_utils/passwords"
data/umbrellio_utils.gemspec CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
12
12
  spec.description = "UmbrellioUtils is collection of utility classes and helpers"
13
13
  spec.homepage = "https://github.com/umbrellio/utils"
14
14
  spec.license = "MIT"
15
- spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
15
+ spec.required_ruby_version = Gem::Requirement.new(">= 3.1.0")
16
16
 
17
17
  spec.metadata["homepage_uri"] = spec.homepage
18
18
  spec.metadata["source_code_uri"] = "https://github.com/umbrellio/utils"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: umbrellio-utils
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Team Umbrellio
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-07-01 00:00:00.000000000 Z
11
+ date: 2024-09-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: memery
@@ -52,6 +52,7 @@ files:
52
52
  - lib/umbrellio_utils/database.rb
53
53
  - lib/umbrellio_utils/formatting.rb
54
54
  - lib/umbrellio_utils/http_client.rb
55
+ - lib/umbrellio_utils/jobs.rb
55
56
  - lib/umbrellio_utils/misc.rb
56
57
  - lib/umbrellio_utils/parsing.rb
57
58
  - lib/umbrellio_utils/passwords.rb
@@ -79,14 +80,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
79
80
  requirements:
80
81
  - - ">="
81
82
  - !ruby/object:Gem::Version
82
- version: 3.0.0
83
+ version: 3.1.0
83
84
  required_rubygems_version: !ruby/object:Gem::Requirement
84
85
  requirements:
85
86
  - - ">="
86
87
  - !ruby/object:Gem::Version
87
88
  version: '0'
88
89
  requirements: []
89
- rubygems_version: 3.5.11
90
+ rubygems_version: 3.5.18
90
91
  signing_key:
91
92
  specification_version: 4
92
93
  summary: A set of utilities that speed up development