yabeda-delayed_job 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Binary file
data/example/Gemfile ADDED
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ source 'https://rubygems.org'
4
+
5
+ gem 'daemons', require: false
6
+ gem 'delayed_job'
7
+ gem 'delayed_job_active_record'
8
+ gem 'pg'
9
+ gem 'rack'
10
+ gem 'webrick'
11
+ gem 'yabeda-delayed_job', path: '/gem'
12
+ gem 'yabeda-prometheus'
@@ -0,0 +1,62 @@
1
+ PATH
2
+ remote: /gem
3
+ specs:
4
+ yabeda-delayed_job (0.1.0)
5
+ delayed_job
6
+ yabeda
7
+
8
+ GEM
9
+ remote: https://rubygems.org/
10
+ specs:
11
+ activemodel (6.1.3.2)
12
+ activesupport (= 6.1.3.2)
13
+ activerecord (6.1.3.2)
14
+ activemodel (= 6.1.3.2)
15
+ activesupport (= 6.1.3.2)
16
+ activesupport (6.1.3.2)
17
+ concurrent-ruby (~> 1.0, >= 1.0.2)
18
+ i18n (>= 1.6, < 2)
19
+ minitest (>= 5.1)
20
+ tzinfo (~> 2.0)
21
+ zeitwerk (~> 2.3)
22
+ concurrent-ruby (1.1.9)
23
+ daemons (1.4.0)
24
+ delayed_job (4.1.9)
25
+ activesupport (>= 3.0, < 6.2)
26
+ delayed_job_active_record (4.1.6)
27
+ activerecord (>= 3.0, < 6.2)
28
+ delayed_job (>= 3.0, < 5)
29
+ dry-initializer (3.0.4)
30
+ i18n (1.8.10)
31
+ concurrent-ruby (~> 1.0)
32
+ minitest (5.14.4)
33
+ pg (1.2.3)
34
+ prometheus-client (2.1.0)
35
+ rack (2.2.3)
36
+ tzinfo (2.0.4)
37
+ concurrent-ruby (~> 1.0)
38
+ webrick (1.7.0)
39
+ yabeda (0.9.0)
40
+ concurrent-ruby
41
+ dry-initializer
42
+ yabeda-prometheus (0.6.1)
43
+ prometheus-client (>= 0.10, < 3.0)
44
+ rack
45
+ yabeda (~> 0.5)
46
+ zeitwerk (2.4.2)
47
+
48
+ PLATFORMS
49
+ x86_64-linux
50
+
51
+ DEPENDENCIES
52
+ daemons
53
+ delayed_job
54
+ delayed_job_active_record
55
+ pg
56
+ rack
57
+ webrick
58
+ yabeda-delayed_job!
59
+ yabeda-prometheus
60
+
61
+ BUNDLED WITH
62
+ 2.2.15
@@ -0,0 +1,23 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'active_record'
5
+ require 'delayed_job'
6
+ require 'delayed_job_active_record'
7
+ require_relative 'jobs'
8
+
9
+ ActiveRecord::Base.establish_connection(
10
+ url: ENV.fetch('DATABASE_URL')
11
+ )
12
+ require '/gem/spec/support/migrations'
13
+ require 'delayed/command'
14
+
15
+ require 'yabeda/delayed_job'
16
+ ::Yabeda.configure!
17
+
18
+ require_relative 'prometheus_store'
19
+
20
+ ::Yabeda::Prometheus::Exporter.start_metrics_server!
21
+
22
+ Delayed::Worker.logger = Logger.new($stdout)
23
+ Delayed::Command.new(['run']).daemonize
@@ -0,0 +1,72 @@
1
+ version: '3.4'
2
+
3
+ services:
4
+ app: &app
5
+ image: ruby:latest
6
+ environment:
7
+ - BUNDLE_PATH=/bundle
8
+ - BUNDLE_CONFIG=/app/.bundle/config
9
+
10
+ - DATABASE_URL=postgresql://postgres@postgres/yabeda-delayed_job
11
+ working_dir: /app
12
+ volumes:
13
+ - ..:/gem:cached
14
+ - .:/app:cached
15
+ - bundler_data:/bundle
16
+ tmpfs:
17
+ - /tmp
18
+ depends_on:
19
+ - postgres
20
+
21
+ enqueuer:
22
+ <<: *app
23
+ command: bundle exec ./enqueuer.rb
24
+
25
+ delayed_job:
26
+ <<: *app
27
+ command: bundle exec ./delayed_job.rb
28
+ postgres:
29
+ image: postgres
30
+ volumes:
31
+ - postgres:/var/lib/postgresql/data
32
+ ports:
33
+ - 5432
34
+ environment:
35
+ - POSTGRES_HOST_AUTH_METHOD=trust
36
+ - POSTGRES_DB=yabeda-delayed_job
37
+
38
+ prometheus:
39
+ image: prom/prometheus:v2.25.2
40
+ volumes:
41
+ - ./prometheus/config.yml:/etc/prometheus/prometheus.yml
42
+ ports:
43
+ - 9090:9090/tcp
44
+ depends_on:
45
+ - enqueuer
46
+ - delayed_job
47
+ command:
48
+ - --config.file=/etc/prometheus/prometheus.yml
49
+
50
+ grafana:
51
+ image: grafana/grafana:7.4.3
52
+ depends_on:
53
+ - prometheus
54
+ ports:
55
+ - 9091:9091/tcp
56
+ environment:
57
+ GF_SECURITY_ADMIN_USER: admin@admin.ru
58
+ GF_SECURITY_ADMIN_PASSWORD: "12345678"
59
+ volumes:
60
+ - ./grafana/grafana.ini:/etc/grafana/grafana.ini
61
+ - ./grafana/dashboards:/etc/grafana/dashboards
62
+ - ./grafana/datasources:/etc/grafana/datasources
63
+ - ./grafana/provisioning:/etc/grafana/provisioning
64
+ - grafana-data:/var/lib/grafana
65
+ user: "104"
66
+
67
+
68
+ volumes:
69
+ bundler_data:
70
+ postgres:
71
+ grafana-data:
72
+
@@ -0,0 +1,27 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'active_record'
5
+ require 'delayed_job'
6
+ require 'delayed_job_active_record'
7
+ require_relative 'jobs'
8
+
9
+ ActiveRecord::Base.establish_connection(
10
+ url: ENV.fetch('DATABASE_URL')
11
+ )
12
+ require '/gem/spec/support/migrations'
13
+
14
+ require 'yabeda/delayed_job'
15
+ ::Yabeda.configure!
16
+
17
+ require_relative 'prometheus_store'
18
+ ::Yabeda::Prometheus::Exporter.start_metrics_server!
19
+
20
+ loop do
21
+ sample = [RandomJob, RandomFailJob].sample
22
+ queue = %i[queue1 queue2 queue3].sample
23
+
24
+ puts "Enqueue #{sample.name} job into #{queue} queue"
25
+ sample.new.delay(queue: queue).call
26
+ sleep(0.1)
27
+ end
@@ -0,0 +1,618 @@
1
+ {
2
+ "annotations": {
3
+ "list": [
4
+ {
5
+ "builtIn": 1,
6
+ "datasource": "-- Grafana --",
7
+ "enable": true,
8
+ "hide": true,
9
+ "iconColor": "rgba(0, 211, 255, 1)",
10
+ "name": "Annotations & Alerts",
11
+ "type": "dashboard"
12
+ }
13
+ ]
14
+ },
15
+ "editable": true,
16
+ "gnetId": null,
17
+ "graphTooltip": 0,
18
+ "id": null,
19
+ "links": [],
20
+ "panels": [
21
+ {
22
+ "aliasColors": {},
23
+ "bars": false,
24
+ "dashLength": 10,
25
+ "dashes": false,
26
+ "datasource": null,
27
+ "fieldConfig": {
28
+ "defaults": {
29
+ "custom": {}
30
+ },
31
+ "overrides": []
32
+ },
33
+ "fill": 1,
34
+ "fillGradient": 0,
35
+ "gridPos": {
36
+ "h": 8,
37
+ "w": 12,
38
+ "x": 0,
39
+ "y": 0
40
+ },
41
+ "hiddenSeries": false,
42
+ "id": 8,
43
+ "legend": {
44
+ "avg": false,
45
+ "current": false,
46
+ "max": false,
47
+ "min": false,
48
+ "show": true,
49
+ "total": false,
50
+ "values": false
51
+ },
52
+ "lines": true,
53
+ "linewidth": 1,
54
+ "nullPointMode": "null",
55
+ "options": {
56
+ "alertThreshold": true
57
+ },
58
+ "percentage": false,
59
+ "pluginVersion": "7.4.3",
60
+ "pointradius": 2,
61
+ "points": false,
62
+ "renderer": "flot",
63
+ "seriesOverrides": [],
64
+ "spaceLength": 10,
65
+ "stack": false,
66
+ "steppedLine": false,
67
+ "targets": [
68
+ {
69
+ "expr": "max (delayed_job_queue_latency) by (queue)",
70
+ "interval": "",
71
+ "legendFormat": "",
72
+ "refId": "A"
73
+ }
74
+ ],
75
+ "thresholds": [],
76
+ "timeFrom": null,
77
+ "timeRegions": [],
78
+ "timeShift": null,
79
+ "title": "Latency",
80
+ "tooltip": {
81
+ "shared": true,
82
+ "sort": 0,
83
+ "value_type": "individual"
84
+ },
85
+ "type": "graph",
86
+ "xaxis": {
87
+ "buckets": null,
88
+ "mode": "time",
89
+ "name": null,
90
+ "show": true,
91
+ "values": []
92
+ },
93
+ "yaxes": [
94
+ {
95
+ "$$hashKey": "object:214",
96
+ "format": "s",
97
+ "label": null,
98
+ "logBase": 1,
99
+ "max": null,
100
+ "min": null,
101
+ "show": true
102
+ },
103
+ {
104
+ "$$hashKey": "object:215",
105
+ "format": "short",
106
+ "label": null,
107
+ "logBase": 1,
108
+ "max": null,
109
+ "min": null,
110
+ "show": true
111
+ }
112
+ ],
113
+ "yaxis": {
114
+ "align": false,
115
+ "alignLevel": null
116
+ }
117
+ },
118
+ {
119
+ "aliasColors": {},
120
+ "bars": false,
121
+ "dashLength": 10,
122
+ "dashes": false,
123
+ "datasource": null,
124
+ "fieldConfig": {
125
+ "defaults": {
126
+ "custom": {}
127
+ },
128
+ "overrides": []
129
+ },
130
+ "fill": 1,
131
+ "fillGradient": 0,
132
+ "gridPos": {
133
+ "h": 8,
134
+ "w": 12,
135
+ "x": 12,
136
+ "y": 0
137
+ },
138
+ "hiddenSeries": false,
139
+ "id": 4,
140
+ "legend": {
141
+ "avg": false,
142
+ "current": false,
143
+ "max": false,
144
+ "min": false,
145
+ "show": true,
146
+ "total": false,
147
+ "values": false
148
+ },
149
+ "lines": true,
150
+ "linewidth": 1,
151
+ "nullPointMode": "null",
152
+ "options": {
153
+ "alertThreshold": true
154
+ },
155
+ "percentage": false,
156
+ "pluginVersion": "7.4.3",
157
+ "pointradius": 2,
158
+ "points": false,
159
+ "renderer": "flot",
160
+ "seriesOverrides": [],
161
+ "spaceLength": 10,
162
+ "stack": false,
163
+ "steppedLine": false,
164
+ "targets": [
165
+ {
166
+ "expr": "max (delayed_job_jobs_waiting_count) by (queue)",
167
+ "interval": "",
168
+ "legendFormat": "{{ queue }}",
169
+ "refId": "A"
170
+ }
171
+ ],
172
+ "thresholds": [],
173
+ "timeRegions": [],
174
+ "title": "Queue size",
175
+ "tooltip": {
176
+ "shared": true,
177
+ "sort": 0,
178
+ "value_type": "individual"
179
+ },
180
+ "type": "graph",
181
+ "xaxis": {
182
+ "buckets": null,
183
+ "mode": "time",
184
+ "name": null,
185
+ "show": true,
186
+ "values": []
187
+ },
188
+ "yaxes": [
189
+ {
190
+ "$$hashKey": "object:100",
191
+ "format": "short",
192
+ "label": null,
193
+ "logBase": 1,
194
+ "max": null,
195
+ "min": null,
196
+ "show": true
197
+ },
198
+ {
199
+ "$$hashKey": "object:101",
200
+ "format": "short",
201
+ "label": null,
202
+ "logBase": 1,
203
+ "max": null,
204
+ "min": null,
205
+ "show": true
206
+ }
207
+ ],
208
+ "yaxis": {
209
+ "align": false,
210
+ "alignLevel": null
211
+ }
212
+ },
213
+ {
214
+ "aliasColors": {},
215
+ "bars": false,
216
+ "dashLength": 10,
217
+ "dashes": false,
218
+ "datasource": null,
219
+ "fieldConfig": {
220
+ "defaults": {
221
+ "custom": {}
222
+ },
223
+ "overrides": []
224
+ },
225
+ "fill": 1,
226
+ "fillGradient": 0,
227
+ "gridPos": {
228
+ "h": 8,
229
+ "w": 12,
230
+ "x": 0,
231
+ "y": 8
232
+ },
233
+ "hiddenSeries": false,
234
+ "id": 2,
235
+ "legend": {
236
+ "avg": false,
237
+ "current": false,
238
+ "max": false,
239
+ "min": false,
240
+ "show": true,
241
+ "total": false,
242
+ "values": false
243
+ },
244
+ "lines": true,
245
+ "linewidth": 1,
246
+ "nullPointMode": "null",
247
+ "options": {
248
+ "alertThreshold": true
249
+ },
250
+ "percentage": false,
251
+ "pluginVersion": "7.4.3",
252
+ "pointradius": 2,
253
+ "points": false,
254
+ "renderer": "flot",
255
+ "seriesOverrides": [],
256
+ "spaceLength": 10,
257
+ "stack": false,
258
+ "steppedLine": false,
259
+ "targets": [
260
+ {
261
+ "expr": "sum (rate(delayed_job_jobs_enqueued_total[5m])) by (queue, worker)",
262
+ "interval": "",
263
+ "legendFormat": "{{ queue }}/{{ worker }}",
264
+ "refId": "A"
265
+ }
266
+ ],
267
+ "thresholds": [],
268
+ "timeRegions": [],
269
+ "title": "Enqueue rate",
270
+ "tooltip": {
271
+ "shared": true,
272
+ "sort": 0,
273
+ "value_type": "individual"
274
+ },
275
+ "type": "graph",
276
+ "xaxis": {
277
+ "buckets": null,
278
+ "mode": "time",
279
+ "name": null,
280
+ "show": true,
281
+ "values": []
282
+ },
283
+ "yaxes": [
284
+ {
285
+ "$$hashKey": "object:43",
286
+ "format": "short",
287
+ "label": null,
288
+ "logBase": 1,
289
+ "max": null,
290
+ "min": null,
291
+ "show": true
292
+ },
293
+ {
294
+ "$$hashKey": "object:44",
295
+ "format": "short",
296
+ "label": null,
297
+ "logBase": 1,
298
+ "max": null,
299
+ "min": null,
300
+ "show": true
301
+ }
302
+ ],
303
+ "yaxis": {
304
+ "align": false,
305
+ "alignLevel": null
306
+ }
307
+ },
308
+ {
309
+ "aliasColors": {},
310
+ "bars": false,
311
+ "dashLength": 10,
312
+ "dashes": false,
313
+ "datasource": null,
314
+ "fieldConfig": {
315
+ "defaults": {
316
+ "custom": {}
317
+ },
318
+ "overrides": []
319
+ },
320
+ "fill": 1,
321
+ "fillGradient": 0,
322
+ "gridPos": {
323
+ "h": 8,
324
+ "w": 12,
325
+ "x": 12,
326
+ "y": 8
327
+ },
328
+ "hiddenSeries": false,
329
+ "id": 6,
330
+ "legend": {
331
+ "avg": false,
332
+ "current": false,
333
+ "max": false,
334
+ "min": false,
335
+ "show": true,
336
+ "total": false,
337
+ "values": false
338
+ },
339
+ "lines": true,
340
+ "linewidth": 1,
341
+ "nullPointMode": "null",
342
+ "options": {
343
+ "alertThreshold": true
344
+ },
345
+ "percentage": false,
346
+ "pluginVersion": "7.4.3",
347
+ "pointradius": 2,
348
+ "points": false,
349
+ "renderer": "flot",
350
+ "seriesOverrides": [],
351
+ "spaceLength": 10,
352
+ "stack": false,
353
+ "steppedLine": false,
354
+ "targets": [
355
+ {
356
+ "expr": "sum (rate(delayed_job_jobs_errored_total[5m])) by (worker)",
357
+ "interval": "",
358
+ "legendFormat": "{{ worker }}",
359
+ "refId": "A"
360
+ },
361
+ {
362
+ "expr": "sum (rate(delayed_job_jobs_failed_total[5m])) by (worker)",
363
+ "hide": false,
364
+ "interval": "",
365
+ "legendFormat": "",
366
+ "refId": "B"
367
+ }
368
+ ],
369
+ "thresholds": [],
370
+ "timeRegions": [],
371
+ "title": "Error/Fail rate",
372
+ "tooltip": {
373
+ "shared": true,
374
+ "sort": 0,
375
+ "value_type": "individual"
376
+ },
377
+ "type": "graph",
378
+ "xaxis": {
379
+ "buckets": null,
380
+ "mode": "time",
381
+ "name": null,
382
+ "show": true,
383
+ "values": []
384
+ },
385
+ "yaxes": [
386
+ {
387
+ "$$hashKey": "object:157",
388
+ "format": "short",
389
+ "label": null,
390
+ "logBase": 1,
391
+ "max": null,
392
+ "min": null,
393
+ "show": true
394
+ },
395
+ {
396
+ "$$hashKey": "object:158",
397
+ "format": "short",
398
+ "label": null,
399
+ "logBase": 1,
400
+ "max": null,
401
+ "min": null,
402
+ "show": true
403
+ }
404
+ ],
405
+ "yaxis": {
406
+ "align": false,
407
+ "alignLevel": null
408
+ }
409
+ },
410
+ {
411
+ "aliasColors": {},
412
+ "bars": false,
413
+ "dashLength": 10,
414
+ "dashes": false,
415
+ "datasource": null,
416
+ "fieldConfig": {
417
+ "defaults": {
418
+ "custom": {}
419
+ },
420
+ "overrides": []
421
+ },
422
+ "fill": 1,
423
+ "fillGradient": 0,
424
+ "gridPos": {
425
+ "h": 8,
426
+ "w": 12,
427
+ "x": 0,
428
+ "y": 16
429
+ },
430
+ "hiddenSeries": false,
431
+ "id": 10,
432
+ "legend": {
433
+ "avg": false,
434
+ "current": false,
435
+ "max": false,
436
+ "min": false,
437
+ "show": true,
438
+ "total": false,
439
+ "values": false
440
+ },
441
+ "lines": true,
442
+ "linewidth": 1,
443
+ "nullPointMode": "null",
444
+ "options": {
445
+ "alertThreshold": true
446
+ },
447
+ "percentage": false,
448
+ "pluginVersion": "7.4.3",
449
+ "pointradius": 2,
450
+ "points": false,
451
+ "renderer": "flot",
452
+ "seriesOverrides": [],
453
+ "spaceLength": 10,
454
+ "stack": false,
455
+ "steppedLine": false,
456
+ "targets": [
457
+ {
458
+ "expr": "histogram_quantile(0.95, sum by (le, worker) (rate(delayed_job_job_runtime_seconds_bucket[5m])))",
459
+ "interval": "",
460
+ "legendFormat": "{{ worker }}",
461
+ "refId": "A"
462
+ }
463
+ ],
464
+ "thresholds": [],
465
+ "timeFrom": null,
466
+ "timeRegions": [],
467
+ "timeShift": null,
468
+ "title": "Job runtime 95q",
469
+ "tooltip": {
470
+ "shared": true,
471
+ "sort": 0,
472
+ "value_type": "individual"
473
+ },
474
+ "type": "graph",
475
+ "xaxis": {
476
+ "buckets": null,
477
+ "mode": "time",
478
+ "name": null,
479
+ "show": true,
480
+ "values": []
481
+ },
482
+ "yaxes": [
483
+ {
484
+ "$$hashKey": "object:271",
485
+ "format": "s",
486
+ "label": null,
487
+ "logBase": 1,
488
+ "max": null,
489
+ "min": null,
490
+ "show": true
491
+ },
492
+ {
493
+ "$$hashKey": "object:272",
494
+ "format": "short",
495
+ "label": null,
496
+ "logBase": 1,
497
+ "max": null,
498
+ "min": null,
499
+ "show": true
500
+ }
501
+ ],
502
+ "yaxis": {
503
+ "align": false,
504
+ "alignLevel": null
505
+ }
506
+ },
507
+ {
508
+ "aliasColors": {},
509
+ "bars": false,
510
+ "dashLength": 10,
511
+ "dashes": false,
512
+ "datasource": null,
513
+ "fieldConfig": {
514
+ "defaults": {
515
+ "custom": {}
516
+ },
517
+ "overrides": []
518
+ },
519
+ "fill": 1,
520
+ "fillGradient": 0,
521
+ "gridPos": {
522
+ "h": 8,
523
+ "w": 12,
524
+ "x": 12,
525
+ "y": 16
526
+ },
527
+ "hiddenSeries": false,
528
+ "id": 12,
529
+ "legend": {
530
+ "avg": false,
531
+ "current": false,
532
+ "max": false,
533
+ "min": false,
534
+ "show": true,
535
+ "total": false,
536
+ "values": false
537
+ },
538
+ "lines": true,
539
+ "linewidth": 1,
540
+ "nullPointMode": "null",
541
+ "options": {
542
+ "alertThreshold": true
543
+ },
544
+ "percentage": false,
545
+ "pluginVersion": "7.4.3",
546
+ "pointradius": 2,
547
+ "points": false,
548
+ "renderer": "flot",
549
+ "seriesOverrides": [],
550
+ "spaceLength": 10,
551
+ "stack": false,
552
+ "steppedLine": false,
553
+ "targets": [
554
+ {
555
+ "expr": "max (delayed_job_running_job_runtime_seconds) by (worker)",
556
+ "interval": "",
557
+ "legendFormat": "{{ worker }}",
558
+ "refId": "A"
559
+ }
560
+ ],
561
+ "thresholds": [],
562
+ "timeRegions": [],
563
+ "title": "Current job runtime",
564
+ "tooltip": {
565
+ "shared": true,
566
+ "sort": 0,
567
+ "value_type": "individual"
568
+ },
569
+ "type": "graph",
570
+ "xaxis": {
571
+ "buckets": null,
572
+ "mode": "time",
573
+ "name": null,
574
+ "show": true,
575
+ "values": []
576
+ },
577
+ "yaxes": [
578
+ {
579
+ "$$hashKey": "object:379",
580
+ "format": "s",
581
+ "label": null,
582
+ "logBase": 1,
583
+ "max": null,
584
+ "min": null,
585
+ "show": true
586
+ },
587
+ {
588
+ "$$hashKey": "object:380",
589
+ "format": "short",
590
+ "label": null,
591
+ "logBase": 1,
592
+ "max": null,
593
+ "min": null,
594
+ "show": true
595
+ }
596
+ ],
597
+ "yaxis": {
598
+ "align": false,
599
+ "alignLevel": null
600
+ }
601
+ }
602
+ ],
603
+ "schemaVersion": 27,
604
+ "style": "dark",
605
+ "tags": [],
606
+ "templating": {
607
+ "list": []
608
+ },
609
+ "time": {
610
+ "from": "now-6h",
611
+ "to": "now"
612
+ },
613
+ "timepicker": {},
614
+ "timezone": "",
615
+ "title": "Delayed Job",
616
+ "uid": null,
617
+ "version": 0
618
+ }