robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Owns the worker threads of one capsule.
|
|
5
|
+
class Manager
|
|
6
|
+
# Worker deaths less than RESPAWN_WINDOW seconds apart count as a
|
|
7
|
+
# streak; each one in a streak doubles the wait before the replacement
|
|
8
|
+
# starts (1s, 2s, 4s... up to MAX_RESPAWN_DELAY), so a persistent fault
|
|
9
|
+
# can't spin up threads in a hot loop.
|
|
10
|
+
RESPAWN_WINDOW = 60
|
|
11
|
+
MAX_RESPAWN_DELAY = 30
|
|
12
|
+
|
|
13
|
+
attr_reader :capsule, :fetcher, :launcher
|
|
14
|
+
|
|
15
|
+
def initialize(capsule, launcher)
|
|
16
|
+
@capsule = capsule
|
|
17
|
+
@config = capsule.config
|
|
18
|
+
@launcher = launcher
|
|
19
|
+
@fetcher = Fetcher.new(capsule, launcher.identity)
|
|
20
|
+
@mutex = Mutex.new
|
|
21
|
+
@slots = concurrency # the next new worker's number
|
|
22
|
+
@processors = Array.new(@slots) { |i| Processor.new(self, capsule, @fetcher, i) }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# How many workers it runs: the capsule's concurrency, until #resize.
|
|
26
|
+
def concurrency = @mutex.synchronize { capsule.concurrency }
|
|
27
|
+
|
|
28
|
+
# Runs +count+ workers from now on. New ones start right away (or with
|
|
29
|
+
# the others, before #start); ones no longer needed finish their
|
|
30
|
+
# current job, then stop.
|
|
31
|
+
def resize(count)
|
|
32
|
+
@mutex.synchronize do
|
|
33
|
+
return if @stopping
|
|
34
|
+
|
|
35
|
+
active = @processors.reject(&:retired?)
|
|
36
|
+
if count > active.size
|
|
37
|
+
added = Array.new(count - active.size) { Processor.new(self, capsule, @fetcher, (@slots += 1) - 1) }
|
|
38
|
+
@processors.concat(added)
|
|
39
|
+
added.each { |processor| processor.start(0) } if @started
|
|
40
|
+
else
|
|
41
|
+
active.last(active.size - count).each(&:retire)
|
|
42
|
+
end
|
|
43
|
+
capsule.resized(count)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def start
|
|
48
|
+
@mutex.synchronize do
|
|
49
|
+
@started = true
|
|
50
|
+
@processors.each { |processor| processor.start(0) }
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def processors
|
|
55
|
+
@mutex.synchronize { @processors.dup }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def quiet
|
|
59
|
+
@stopping = true
|
|
60
|
+
processors.each(&:terminate)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Waits for running jobs until +deadline+ (monotonic seconds), then
|
|
64
|
+
# interrupts whatever is still running and waits for it to wind down
|
|
65
|
+
# until +hard_deadline+ (shared by every capsule).
|
|
66
|
+
def stop(deadline, hard_deadline)
|
|
67
|
+
quiet
|
|
68
|
+
sleep 0.05 while processors.any?(&:alive?) && monotonic < deadline
|
|
69
|
+
remaining = processors.select(&:alive?)
|
|
70
|
+
return if remaining.empty?
|
|
71
|
+
|
|
72
|
+
@config.logger.warn("interrupting #{remaining.size} job(s) still running after #{@config.timeout}s")
|
|
73
|
+
Events.emit(@config, "jobs.interrupted", count: remaining.size)
|
|
74
|
+
remaining.each(&:kill)
|
|
75
|
+
remaining.each { |p| p.thread&.join([hard_deadline - monotonic, 0].max) }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def processor_stopped(processor)
|
|
79
|
+
@mutex.synchronize { @processors.delete(processor) }
|
|
80
|
+
@launcher.work_finished_cleanup(processor)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def processor_died(processor, exception)
|
|
84
|
+
@config.handle_exception(exception, {context: "worker thread died"})
|
|
85
|
+
@launcher.work_finished_cleanup(processor)
|
|
86
|
+
delay = @mutex.synchronize do
|
|
87
|
+
index = @processors.index(processor)
|
|
88
|
+
@processors.delete(processor)
|
|
89
|
+
unless @stopping || processor.retired?
|
|
90
|
+
replacement = Processor.new(self, capsule, @fetcher, index || @processors.size)
|
|
91
|
+
@processors << replacement
|
|
92
|
+
respawn_delay.tap { |d| replacement.start(d) }
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
Events.emit(@config, "worker.died", error: exception, delay: delay)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def work_started(processor, unit)
|
|
99
|
+
@launcher.work_started(processor, unit)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def work_finished(processor, payload, elapsed_ms, ok)
|
|
103
|
+
@launcher.work_finished(processor, payload, elapsed_ms, ok)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
private
|
|
107
|
+
|
|
108
|
+
# Seconds to wait before replacing a dead worker. Called with the
|
|
109
|
+
# mutex held.
|
|
110
|
+
def respawn_delay
|
|
111
|
+
now = monotonic
|
|
112
|
+
@deaths = (@last_death && now - @last_death < RESPAWN_WINDOW) ? @deaths + 1 : 0
|
|
113
|
+
@last_death = now
|
|
114
|
+
return 0 if @deaths.zero?
|
|
115
|
+
|
|
116
|
+
delay = [2**(@deaths - 1), MAX_RESPAWN_DELAY].min #: Integer
|
|
117
|
+
@config.logger.warn("worker thread died #{@deaths + 1} times in a row; restarting it in #{delay}s")
|
|
118
|
+
delay
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def monotonic
|
|
122
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Per-job-class execution metrics.
|
|
5
|
+
#
|
|
6
|
+
# Workers aggregate counts, total time and a latency histogram in memory;
|
|
7
|
+
# the heartbeat flushes them to Redis once every few seconds, so recording
|
|
8
|
+
# costs no extra round-trips per job.
|
|
9
|
+
#
|
|
10
|
+
# Storage (UTC):
|
|
11
|
+
# robin:m:<YYYYMMDDHHMM> hash "<Class>|s|f|ms" counts, 8h TTL
|
|
12
|
+
# robin:h:<Class>:<YYYYMMDDHHMM> hash bucket index => count, 8h TTL
|
|
13
|
+
# robin:mh:<YYYYMMDDHH> hash hourly rollup, kept metrics_retention_days
|
|
14
|
+
# robin:qh:<YYYYMMDDHH> hash "<queue>|size" / "<queue>|latency" hourly maxima
|
|
15
|
+
# robin:qm:<YYYYMMDDHHMM> hash the same, per minute, 8h TTL
|
|
16
|
+
module Metrics
|
|
17
|
+
# Upper bounds (ms) of the histogram buckets; the last is open-ended.
|
|
18
|
+
BUCKETS = [5, 10, 20, 35, 50, 75, 100, 150, 250, 400, 600, 1_000, 1_500, 2_500, 4_000,
|
|
19
|
+
6_000, 10_000, 15_000, 25_000, 40_000, 60_000, 120_000, 300_000, 600_000, Float::INFINITY].freeze
|
|
20
|
+
|
|
21
|
+
MINUTE_TTL = 8 * 3600
|
|
22
|
+
|
|
23
|
+
# The daily job totals (stat:processed:<YYYY-MM-DD>, stat:failed:...)
|
|
24
|
+
# are two small keys a day, kept five years.
|
|
25
|
+
DAILY_TOTALS_TTL = 5 * 365 * 86_400
|
|
26
|
+
|
|
27
|
+
class << self
|
|
28
|
+
def bucket_for(ms)
|
|
29
|
+
BUCKETS.bsearch_index { |b| ms <= b } #: Integer
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def minute_stamp(time) = time.getutc.strftime("%Y%m%d%H%M")
|
|
33
|
+
|
|
34
|
+
def hour_stamp(time) = time.getutc.strftime("%Y%m%d%H")
|
|
35
|
+
|
|
36
|
+
# Records queue sizes and latencies (leader only, once a minute), in
|
|
37
|
+
# two round trips.
|
|
38
|
+
# mutant:disable -- Connection#evalsha splats ARGV, so passing the
|
|
39
|
+
# prefix bare or wrapped in an Array is identical.
|
|
40
|
+
def sample_queues(config, conn, time = Time.now)
|
|
41
|
+
found = conn.evalsha(Scripts::QUEUE_SAMPLES, [config.key("queues")], ["#{config.key("queue")}:"])
|
|
42
|
+
return if found.empty?
|
|
43
|
+
|
|
44
|
+
samples = found.each_slice(3).to_h { |q, size, oldest| [q, {size: size, latency: Queue.latency_of(oldest)}] }
|
|
45
|
+
fields = samples.flat_map { |q, s| ["#{q}|size", s.fetch(:size), "#{q}|latency", s.fetch(:latency)] }
|
|
46
|
+
retention = config.metrics_retention_days * 86_400
|
|
47
|
+
conn.pipelined do |p|
|
|
48
|
+
p.evalsha(Scripts::HMAX, [config.key("qm", minute_stamp(time))], [MINUTE_TTL, *fields])
|
|
49
|
+
p.evalsha(Scripts::HMAX, [config.key("qh", hour_stamp(time))], [retention, *fields])
|
|
50
|
+
end
|
|
51
|
+
config.statsd&.queue_metrics(samples)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Each thread records into a stripe of its own, so workers never wait
|
|
56
|
+
# on each other (with real threads, as on JRuby, one shared lock would
|
|
57
|
+
# serialize every job); draining merges the stripes.
|
|
58
|
+
class Collector
|
|
59
|
+
# One thread's counts.
|
|
60
|
+
class Stripe
|
|
61
|
+
attr_reader :thread
|
|
62
|
+
|
|
63
|
+
def initialize(thread)
|
|
64
|
+
@thread = thread
|
|
65
|
+
@mutex = Mutex.new
|
|
66
|
+
reset
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def record(klass, ms, ok)
|
|
70
|
+
@mutex.synchronize do
|
|
71
|
+
entry = Collector.entry(@jobs, klass)
|
|
72
|
+
# Timings cover successful runs only: a job failing fast (or
|
|
73
|
+
# timing out) would skew the averages and the histogram.
|
|
74
|
+
if ok
|
|
75
|
+
entry[:s] += 1
|
|
76
|
+
entry[:ms] += ms
|
|
77
|
+
entry.fetch(:hist)[Metrics.bucket_for(ms)] += 1
|
|
78
|
+
else
|
|
79
|
+
entry[:f] += 1
|
|
80
|
+
end
|
|
81
|
+
@processed += 1
|
|
82
|
+
@failed += 1 unless ok
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Adds counts drained earlier.
|
|
87
|
+
def add(drained)
|
|
88
|
+
jobs, processed, failed = drained
|
|
89
|
+
@mutex.synchronize do
|
|
90
|
+
Collector.merge(@jobs, jobs)
|
|
91
|
+
@processed += processed
|
|
92
|
+
@failed += failed
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def drain
|
|
97
|
+
@mutex.synchronize do
|
|
98
|
+
snapshot = [@jobs, @processed, @failed] #: Metrics::drained
|
|
99
|
+
reset
|
|
100
|
+
snapshot
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
private
|
|
105
|
+
|
|
106
|
+
def reset
|
|
107
|
+
@jobs = {}
|
|
108
|
+
@processed = 0
|
|
109
|
+
@failed = 0
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def self.entry(jobs, klass) = (jobs[klass] ||= {s: 0, f: 0, ms: 0.0, hist: Hash.new(0)})
|
|
114
|
+
|
|
115
|
+
# Adds +from+'s per-class counts to +into+'s.
|
|
116
|
+
def self.merge(into, from)
|
|
117
|
+
from.each do |klass, data|
|
|
118
|
+
entry = entry(into, klass)
|
|
119
|
+
entry[:s] += data.fetch(:s)
|
|
120
|
+
entry[:f] += data.fetch(:f)
|
|
121
|
+
entry[:ms] += data.fetch(:ms)
|
|
122
|
+
data.fetch(:hist).each { |b, c| entry.fetch(:hist)[b] += c }
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def initialize
|
|
127
|
+
@mutex = Mutex.new
|
|
128
|
+
@stripes = []
|
|
129
|
+
@key = :"robin_metrics_#{object_id}"
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def record(klass, ms, ok) = stripe.record(klass, ms, ok)
|
|
133
|
+
|
|
134
|
+
# [per-class counts, processed, failed] since the last drain. Stripes
|
|
135
|
+
# of threads that have finished go once they're drained.
|
|
136
|
+
def drain
|
|
137
|
+
stripes = @mutex.synchronize do
|
|
138
|
+
all = @stripes
|
|
139
|
+
@stripes = all.select { |s| s.thread.alive? }
|
|
140
|
+
all
|
|
141
|
+
end
|
|
142
|
+
jobs = {} #: Metrics::jobs
|
|
143
|
+
processed = failed = 0
|
|
144
|
+
stripes.each do |stripe|
|
|
145
|
+
stripe_jobs, stripe_processed, stripe_failed = stripe.drain
|
|
146
|
+
Collector.merge(jobs, stripe_jobs)
|
|
147
|
+
processed += stripe_processed
|
|
148
|
+
failed += stripe_failed
|
|
149
|
+
end
|
|
150
|
+
[jobs, processed, failed]
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Puts drained data back (used when a flush fails).
|
|
154
|
+
def restore(drained) = stripe.add(drained)
|
|
155
|
+
|
|
156
|
+
# Writes what was recorded since the last flush in one pipeline.
|
|
157
|
+
def flush(config, conn, time = Time.now)
|
|
158
|
+
drained = nil #: Metrics::drained?
|
|
159
|
+
conn.pipelined { |p| drained = queue(config, p, time) }
|
|
160
|
+
# (Pipelined runs the block, so drained is set.)
|
|
161
|
+
flushed(config, drained) # steep:ignore ArgumentTypeMismatch
|
|
162
|
+
rescue
|
|
163
|
+
restore(drained) if drained
|
|
164
|
+
raise
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Queues the commands that write what was recorded since the last
|
|
168
|
+
# flush on +pipe+ (a pipeline or transaction about to be sent), and
|
|
169
|
+
# returns what they write: pass it to #flushed once they're sent, or
|
|
170
|
+
# to #restore if sending them failed.
|
|
171
|
+
def queue(config, pipe, time = Time.now)
|
|
172
|
+
drained = drain
|
|
173
|
+
write(config, pipe, time, drained) unless drained.fetch(1).zero?
|
|
174
|
+
drained
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# Reports queued metrics to StatsD once they're written.
|
|
178
|
+
def flushed(config, drained)
|
|
179
|
+
jobs, processed, _failed = drained
|
|
180
|
+
config.statsd&.job_metrics(jobs) unless processed.zero?
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
private
|
|
184
|
+
|
|
185
|
+
# This thread's stripe, made on first use.
|
|
186
|
+
def stripe
|
|
187
|
+
Thread.current[@key] ||= Stripe.new(Thread.current).tap do |s|
|
|
188
|
+
@mutex.synchronize { @stripes << s }
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def write(config, p, time, drained)
|
|
193
|
+
jobs, processed, failed = drained
|
|
194
|
+
minute = Metrics.minute_stamp(time)
|
|
195
|
+
hour = Metrics.hour_stamp(time)
|
|
196
|
+
day = time.getutc.strftime("%Y-%m-%d")
|
|
197
|
+
mkey = config.key("m", minute)
|
|
198
|
+
hkey = config.key("mh", hour)
|
|
199
|
+
p.call("INCRBY", config.key("stat", "processed"), processed)
|
|
200
|
+
p.call("INCRBY", config.key("stat", "processed", day), processed)
|
|
201
|
+
p.call("EXPIRE", config.key("stat", "processed", day), DAILY_TOTALS_TTL)
|
|
202
|
+
if failed > 0
|
|
203
|
+
p.call("INCRBY", config.key("stat", "failed"), failed)
|
|
204
|
+
p.call("INCRBY", config.key("stat", "failed", day), failed)
|
|
205
|
+
p.call("EXPIRE", config.key("stat", "failed", day), DAILY_TOTALS_TTL)
|
|
206
|
+
end
|
|
207
|
+
jobs.each do |klass, d|
|
|
208
|
+
[mkey, hkey].each do |key|
|
|
209
|
+
if d.fetch(:s) > 0
|
|
210
|
+
p.call("HINCRBY", key, "#{klass}|s", d.fetch(:s))
|
|
211
|
+
p.call("HINCRBYFLOAT", key, "#{klass}|ms", d.fetch(:ms).round(1))
|
|
212
|
+
end
|
|
213
|
+
p.call("HINCRBY", key, "#{klass}|f", d.fetch(:f)) if d.fetch(:f) > 0
|
|
214
|
+
end
|
|
215
|
+
hist = config.key("h", klass, minute)
|
|
216
|
+
d.fetch(:hist).each { |bucket, count| p.call("HINCRBY", hist, bucket, count) }
|
|
217
|
+
p.call("EXPIRE", hist, MINUTE_TTL)
|
|
218
|
+
end
|
|
219
|
+
p.call("EXPIRE", mkey, MINUTE_TTL)
|
|
220
|
+
p.call("EXPIRE", hkey, config.metrics_retention_days * 86_400)
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
# Query helpers for the web UI and API.
|
|
225
|
+
class Query
|
|
226
|
+
GRANULARITIES = {minute: [60, "qm", :minute_stamp], hour: [3600, "qh", :hour_stamp]} #: Hash[Symbol, [Integer, String, Symbol]]
|
|
227
|
+
.freeze
|
|
228
|
+
|
|
229
|
+
def initialize(config = Robin.config)
|
|
230
|
+
@config = config
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Per-class totals over the last +minutes+ minutes, plus a per-minute
|
|
234
|
+
# series for charting. Returns {"classes" => {...}, "series" => [...]}
|
|
235
|
+
def recent(minutes: 60, now: Time.now)
|
|
236
|
+
stamps = (0...minutes).map { |i| Metrics.minute_stamp(now - i * 60) }.reverse
|
|
237
|
+
rows = @config.redis do |c|
|
|
238
|
+
c.pipelined { |p| stamps.each { |s| p.call("HGETALL", @config.key("m", s)) } }
|
|
239
|
+
end
|
|
240
|
+
summarize(stamps, rows)
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Hourly history over the last +hours+ hours (historical metrics).
|
|
244
|
+
def history(hours: 24 * 7, now: Time.now)
|
|
245
|
+
stamps = (0...hours).map { |i| Metrics.hour_stamp(now - i * 3600) }.reverse
|
|
246
|
+
rows = @config.redis do |c|
|
|
247
|
+
c.pipelined { |p| stamps.each { |s| p.call("HGETALL", @config.key("mh", s)) } }
|
|
248
|
+
end
|
|
249
|
+
summarize(stamps, rows)
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Queue size/latency maxima, per minute (granularity :minute) or per
|
|
253
|
+
# hour (granularity :hour). Other granularities raise KeyError.
|
|
254
|
+
def queue_history(granularity: :minute, points: 60, now: Time.now)
|
|
255
|
+
step, prefix, fmt = GRANULARITIES.fetch(granularity)
|
|
256
|
+
stamps = (0...points).map { |i| Metrics.public_send(fmt, now - i * step) }.reverse
|
|
257
|
+
rows = @config.redis do |c|
|
|
258
|
+
c.pipelined { |p| stamps.each { |s| p.call("HGETALL", @config.key(prefix, s)) } }
|
|
259
|
+
end
|
|
260
|
+
stamps.zip(rows).map do |stamp, row|
|
|
261
|
+
data = Hash.new { |h, k| h[k] = {} }
|
|
262
|
+
row.each_slice(2) do |field, value|
|
|
263
|
+
queue, _, kind = field.rpartition("|")
|
|
264
|
+
data[queue][kind] = value.to_i
|
|
265
|
+
end
|
|
266
|
+
{"t" => stamp, "queues" => data}
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# Latency histogram for one class over the last +minutes+ minutes.
|
|
271
|
+
def histogram(klass, minutes: 60, now: Time.now)
|
|
272
|
+
stamps = (0...minutes).map { |i| Metrics.minute_stamp(now - i * 60) }
|
|
273
|
+
rows = @config.redis do |c|
|
|
274
|
+
c.pipelined { |p| stamps.each { |s| p.call("HGETALL", @config.key("h", klass, s)) } }
|
|
275
|
+
end
|
|
276
|
+
totals = Array.new(BUCKETS.size, 0)
|
|
277
|
+
rows.each { |row| row.each_slice(2) { |b, n| totals[Integer(b)] += Integer(n) } }
|
|
278
|
+
BUCKETS.each_with_index.map { |limit, i| {"le" => (limit unless limit.infinite?), "count" => totals.fetch(i)} }
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
private
|
|
282
|
+
|
|
283
|
+
def summarize(stamps, rows)
|
|
284
|
+
classes = Hash.new { |h, k| h[k] = {"s" => 0, "f" => 0, "ms" => 0.0} } #: Hash[String, Hash[String, Numeric]]
|
|
285
|
+
series = stamps.zip(rows).map do |stamp, row|
|
|
286
|
+
point = {"t" => stamp, "s" => 0, "f" => 0, "ms" => 0.0} #: Hash[String, untyped]
|
|
287
|
+
row.each_slice(2) do |field, value|
|
|
288
|
+
klass, _, kind = field.rpartition("|")
|
|
289
|
+
v = (kind == "ms") ? Float(value) : Integer(value)
|
|
290
|
+
classes[klass][kind] += v
|
|
291
|
+
point[kind] += v
|
|
292
|
+
end
|
|
293
|
+
point
|
|
294
|
+
end
|
|
295
|
+
classes.each_value do |c|
|
|
296
|
+
n = c.fetch("s") # "ms" only counts successful runs
|
|
297
|
+
c["avg_ms"] = n.zero? ? 0 : (c.fetch("ms") / n).round(1)
|
|
298
|
+
end
|
|
299
|
+
{"classes" => classes, "series" => series}
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Middleware wraps job pushes (client side) and job execution (server
|
|
5
|
+
# side), Rack style.
|
|
6
|
+
#
|
|
7
|
+
# Client middleware:
|
|
8
|
+
#
|
|
9
|
+
# class AddTenant
|
|
10
|
+
# def call(job_class, payload, queue, redis)
|
|
11
|
+
# payload["tenant"] ||= Current.tenant
|
|
12
|
+
# yield # return nil instead of yielding to stop the push
|
|
13
|
+
# end
|
|
14
|
+
# end
|
|
15
|
+
#
|
|
16
|
+
# Server middleware:
|
|
17
|
+
#
|
|
18
|
+
# class TimeIt
|
|
19
|
+
# def call(job, payload, queue)
|
|
20
|
+
# t = Time.now
|
|
21
|
+
# yield
|
|
22
|
+
# ensure
|
|
23
|
+
# puts "took #{Time.now - t}s"
|
|
24
|
+
# end
|
|
25
|
+
# end
|
|
26
|
+
#
|
|
27
|
+
# Middleware classes may define +initialize(*args)+ to receive arguments
|
|
28
|
+
# passed to Chain#add. If they include Robin::Middleware::ConfigAware
|
|
29
|
+
# they also get access to the Robin config.
|
|
30
|
+
#
|
|
31
|
+
# A middleware is built anew for every job, unless it includes
|
|
32
|
+
# Robin::Middleware::Stateless: then one instance serves every job, on
|
|
33
|
+
# every thread, so it must not keep per-job state in instance variables.
|
|
34
|
+
#
|
|
35
|
+
# A middleware that only matters to some jobs can say which with a class
|
|
36
|
+
# method +applies?+, taking +call+'s arguments: the chain skips it (and
|
|
37
|
+
# doesn't build it) when that's false.
|
|
38
|
+
#
|
|
39
|
+
# class Audit
|
|
40
|
+
# def self.applies?(_job, payload, _queue) = payload.key?("audit")
|
|
41
|
+
# ...
|
|
42
|
+
# end
|
|
43
|
+
module Middleware
|
|
44
|
+
module ConfigAware
|
|
45
|
+
attr_accessor :config
|
|
46
|
+
|
|
47
|
+
def redis(&) = config.redis(&)
|
|
48
|
+
|
|
49
|
+
def logger = config.logger
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
module Stateless; end
|
|
53
|
+
|
|
54
|
+
class Entry
|
|
55
|
+
attr_reader :klass
|
|
56
|
+
|
|
57
|
+
def initialize(config, klass, args)
|
|
58
|
+
@config = config
|
|
59
|
+
@klass = klass
|
|
60
|
+
@args = args
|
|
61
|
+
@stateless = klass.include?(Stateless)
|
|
62
|
+
@filtered = klass.respond_to?(:applies?)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def applies?(args) = !@filtered || @klass.applies?(*args)
|
|
66
|
+
|
|
67
|
+
# The instance to call. A stateless middleware's is built once (two
|
|
68
|
+
# threads racing to build it first is harmless).
|
|
69
|
+
def build
|
|
70
|
+
return @instance ||= instantiate if @stateless
|
|
71
|
+
|
|
72
|
+
instantiate
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def instantiate
|
|
78
|
+
instance = @klass.new(*@args)
|
|
79
|
+
instance.config = @config if instance.respond_to?(:config=)
|
|
80
|
+
instance
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
class Chain
|
|
85
|
+
include Enumerable
|
|
86
|
+
|
|
87
|
+
def initialize(config)
|
|
88
|
+
@config = config
|
|
89
|
+
@entries = [] #: Array[Entry]
|
|
90
|
+
.freeze
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Frozen: change the chain with its methods.
|
|
94
|
+
attr_reader :entries
|
|
95
|
+
|
|
96
|
+
def each(&) = @entries.each(&)
|
|
97
|
+
|
|
98
|
+
# The chain is replaced on every change, never changed in place, so a
|
|
99
|
+
# job being invoked keeps the middleware it started with (and a dup'd
|
|
100
|
+
# chain changes independently).
|
|
101
|
+
def add(klass, *args)
|
|
102
|
+
@entries = (without(klass) << Entry.new(@config, klass, args)).freeze
|
|
103
|
+
self
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def prepend(klass, *args)
|
|
107
|
+
@entries = without(klass).unshift(Entry.new(@config, klass, args)).freeze
|
|
108
|
+
self
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def insert_before(target, klass, *args)
|
|
112
|
+
entries = without(klass)
|
|
113
|
+
index = entries.index { |e| e.klass.equal?(target) } || 0
|
|
114
|
+
@entries = entries.insert(index, Entry.new(@config, klass, args)).freeze
|
|
115
|
+
self
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def insert_after(target, klass, *args)
|
|
119
|
+
entries = without(klass)
|
|
120
|
+
index = entries.index { |e| e.klass.equal?(target) } || (entries.size - 1)
|
|
121
|
+
@entries = entries.insert(index + 1, Entry.new(@config, klass, args)).freeze
|
|
122
|
+
self
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def remove(klass)
|
|
126
|
+
@entries = without(klass).freeze
|
|
127
|
+
self
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def exists?(klass)
|
|
131
|
+
any? { |e| e.klass.equal?(klass) }
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def empty? = @entries.empty?
|
|
135
|
+
|
|
136
|
+
def clear
|
|
137
|
+
@entries = [] #: Array[Entry]
|
|
138
|
+
.freeze
|
|
139
|
+
self
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Invokes the chain around the block, outermost middleware first.
|
|
143
|
+
# Each middleware is instantiated as the chain reaches it. Returns the
|
|
144
|
+
# block's value, or whatever a middleware that halted the chain
|
|
145
|
+
# returned instead of yielding.
|
|
146
|
+
def invoke(*args, &block)
|
|
147
|
+
call_from(@entries, 0, args, block)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
private
|
|
151
|
+
|
|
152
|
+
# Calls the first middleware from +index+ on that applies; yielding
|
|
153
|
+
# calls the next one. (No closures are built per job, so a long chain
|
|
154
|
+
# costs little.)
|
|
155
|
+
def call_from(entries, index, args, block)
|
|
156
|
+
index += 1 while (entry = entries.at(index)) && !entry.applies?(args)
|
|
157
|
+
return block.call unless entry
|
|
158
|
+
|
|
159
|
+
entry.build.call(*args) { call_from(entries, index + 1, args, block) }
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def without(klass) = reject { |e| e.klass.equal?(klass) }
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "robin"
|
|
4
|
+
require "opentelemetry"
|
|
5
|
+
|
|
6
|
+
module Robin
|
|
7
|
+
# OpenTelemetry tracing for pushing and running jobs. Needs the
|
|
8
|
+
# opentelemetry-api gem (and an SDK to export spans somewhere):
|
|
9
|
+
#
|
|
10
|
+
# require "robin/opentelemetry"
|
|
11
|
+
#
|
|
12
|
+
# Robin.configure do |config|
|
|
13
|
+
# Robin::OpenTelemetry.install(config)
|
|
14
|
+
# end
|
|
15
|
+
#
|
|
16
|
+
# Pushing a job opens a "publish <queue>" producer span and stores the
|
|
17
|
+
# trace context in the job ("otel"). Running it opens a "process <queue>"
|
|
18
|
+
# consumer span, a child of the push, so a request and the jobs it pushed
|
|
19
|
+
# share one trace. With +parent: :link+ each run starts its own trace
|
|
20
|
+
# instead, linked to the push (better for jobs that run long after they
|
|
21
|
+
# were pushed). Spans follow the messaging semantic conventions, plus
|
|
22
|
+
# robin.job.class and robin.job.retry_count.
|
|
23
|
+
module OpenTelemetry
|
|
24
|
+
def self.install(config, tracer_provider: ::OpenTelemetry.tracer_provider, parent: :child)
|
|
25
|
+
raise ArgumentError, "parent: must be :child or :link" unless %i[child link].include?(parent)
|
|
26
|
+
|
|
27
|
+
tracer = tracer_provider.tracer("robin", VERSION)
|
|
28
|
+
config.client_middleware { |chain| chain.prepend(ClientMiddleware, tracer) }
|
|
29
|
+
config.server_middleware { |chain| chain.prepend(ServerMiddleware, tracer, parent) }
|
|
30
|
+
config
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.attributes(payload, queue, operation)
|
|
34
|
+
attrs = {
|
|
35
|
+
"messaging.system" => "robin",
|
|
36
|
+
"messaging.operation.type" => operation,
|
|
37
|
+
"messaging.destination.name" => queue,
|
|
38
|
+
"messaging.message.id" => payload.fetch("jid"),
|
|
39
|
+
"robin.job.class" => payload.fetch("class")
|
|
40
|
+
}
|
|
41
|
+
attrs["robin.job.retry_count"] = payload.fetch("retry_count") if payload.key?("retry_count")
|
|
42
|
+
attrs
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
class ClientMiddleware
|
|
46
|
+
include Middleware::Stateless
|
|
47
|
+
|
|
48
|
+
def initialize(tracer)
|
|
49
|
+
@tracer = tracer
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def call(_job_class, payload, queue, _redis)
|
|
53
|
+
@tracer.in_span("publish #{queue}", kind: :producer, attributes: OpenTelemetry.attributes(payload, queue, "publish")) do
|
|
54
|
+
carrier = {} #: Hash[String, String]
|
|
55
|
+
::OpenTelemetry.propagation.inject(carrier)
|
|
56
|
+
payload["otel"] = carrier unless carrier.empty?
|
|
57
|
+
yield
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
class ServerMiddleware
|
|
63
|
+
include Middleware::Stateless
|
|
64
|
+
|
|
65
|
+
def initialize(tracer, parent)
|
|
66
|
+
@tracer = tracer
|
|
67
|
+
@parent = parent
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def call(_job, payload, queue, &)
|
|
71
|
+
pushed = ::OpenTelemetry.propagation.extract(payload.fetch("otel", {}))
|
|
72
|
+
attributes = OpenTelemetry.attributes(payload, queue, "process")
|
|
73
|
+
name = "process #{queue}"
|
|
74
|
+
return ::OpenTelemetry::Context.with_current(pushed) { @tracer.in_span(name, kind: :consumer, attributes: attributes, &) } if @parent == :child
|
|
75
|
+
|
|
76
|
+
push = ::OpenTelemetry::Trace.current_span(pushed).context
|
|
77
|
+
links = [::OpenTelemetry::Trace::Link.new(push)] if push.valid?
|
|
78
|
+
detached { @tracer.in_span(name, kind: :consumer, attributes: attributes, links: links, &) }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
# mutant:disable -- Context.current reads an attached nil as the
|
|
84
|
+
# empty context, so passing nil instead is the same.
|
|
85
|
+
def detached(&) = ::OpenTelemetry::Context.with_current(::OpenTelemetry::Context.empty, &)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|