robin 0.1.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/java/robin_resp/Blocking.java +27 -0
  13. data/ext/java/robin_resp/NativeIO.java +229 -0
  14. data/ext/java/robin_resp/NativeRESP.java +279 -0
  15. data/ext/java/robin_resp/NativeSlots.java +133 -0
  16. data/ext/java/robin_resp/RobinRespService.java +32 -0
  17. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  18. data/lib/generators/robin/install/install_generator.rb +39 -0
  19. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  20. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  21. data/lib/generators/robin/job/job_generator.rb +78 -0
  22. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  23. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  24. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  25. data/lib/puma/plugin/robin.rb +131 -0
  26. data/lib/robin/active_job.rb +144 -0
  27. data/lib/robin/api.rb +777 -0
  28. data/lib/robin/batch.rb +700 -0
  29. data/lib/robin/cancellation.rb +62 -0
  30. data/lib/robin/cli.rb +325 -0
  31. data/lib/robin/client.rb +372 -0
  32. data/lib/robin/config.rb +461 -0
  33. data/lib/robin/cron.rb +325 -0
  34. data/lib/robin/debounce.rb +108 -0
  35. data/lib/robin/deploy.rb +65 -0
  36. data/lib/robin/embedded.rb +53 -0
  37. data/lib/robin/encryption.rb +147 -0
  38. data/lib/robin/events.rb +60 -0
  39. data/lib/robin/expiry.rb +41 -0
  40. data/lib/robin/fetcher.rb +441 -0
  41. data/lib/robin/gvl.rb +62 -0
  42. data/lib/robin/health.rb +76 -0
  43. data/lib/robin/iterable_job.rb +294 -0
  44. data/lib/robin/job.rb +258 -0
  45. data/lib/robin/job_class.rb +157 -0
  46. data/lib/robin/job_logger.rb +121 -0
  47. data/lib/robin/launcher.rb +658 -0
  48. data/lib/robin/leader.rb +86 -0
  49. data/lib/robin/limiter.rb +639 -0
  50. data/lib/robin/manager.rb +125 -0
  51. data/lib/robin/metrics.rb +303 -0
  52. data/lib/robin/middleware.rb +165 -0
  53. data/lib/robin/opentelemetry.rb +88 -0
  54. data/lib/robin/periodic.rb +327 -0
  55. data/lib/robin/processor.rb +317 -0
  56. data/lib/robin/profiler.rb +140 -0
  57. data/lib/robin/progress.rb +46 -0
  58. data/lib/robin/rails/app_loader.rb +23 -0
  59. data/lib/robin/rails/current_attributes.rb +88 -0
  60. data/lib/robin/rails/error_reporter.rb +31 -0
  61. data/lib/robin/rails/notifications.rb +22 -0
  62. data/lib/robin/rails/railtie.rb +87 -0
  63. data/lib/robin/rails/transactional_push.rb +49 -0
  64. data/lib/robin/rails.rb +23 -0
  65. data/lib/robin/recovery.rb +135 -0
  66. data/lib/robin/redis/connection.rb +639 -0
  67. data/lib/robin/redis/pool.rb +190 -0
  68. data/lib/robin/redis/resp.rb +77 -0
  69. data/lib/robin/reliable_push.rb +153 -0
  70. data/lib/robin/retrier.rb +284 -0
  71. data/lib/robin/scheduler.rb +102 -0
  72. data/lib/robin/scripts.rb +331 -0
  73. data/lib/robin/statsd.rb +135 -0
  74. data/lib/robin/swarm.rb +319 -0
  75. data/lib/robin/systemd.rb +47 -0
  76. data/lib/robin/testing.rb +158 -0
  77. data/lib/robin/top.rb +178 -0
  78. data/lib/robin/unique.rb +198 -0
  79. data/lib/robin/version.rb +5 -0
  80. data/lib/robin/warmup.rb +94 -0
  81. data/lib/robin/web/api.rb +580 -0
  82. data/lib/robin/web/batch_status.rb +44 -0
  83. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  84. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  86. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  87. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  88. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  89. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  90. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  91. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  92. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  93. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  94. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  95. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  96. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  97. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  98. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  99. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  100. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  101. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  102. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  103. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  104. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  105. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  106. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  107. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  108. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  109. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  110. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  111. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  112. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  113. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  114. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  115. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  116. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  117. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  118. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  119. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  120. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  121. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  122. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  123. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  124. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  125. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  126. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  127. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  128. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  129. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  130. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  131. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  132. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  133. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  134. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  135. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  136. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  137. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  138. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  139. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  140. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  141. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  142. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  143. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  144. data/lib/robin/web/dist/index.html +19 -0
  145. data/lib/robin/web/extension.rb +193 -0
  146. data/lib/robin/web/permissions.rb +40 -0
  147. data/lib/robin/web/prometheus.rb +79 -0
  148. data/lib/robin/web/router.rb +150 -0
  149. data/lib/robin/web/static.rb +110 -0
  150. data/lib/robin/web.rb +196 -0
  151. data/lib/robin.rb +261 -0
  152. data/lib/robin_resp/robin_resp.jar +0 -0
  153. data/sig/manifest.yaml +9 -0
  154. data/sig/robin/api.rbs +442 -0
  155. data/sig/robin/batch.rbs +292 -0
  156. data/sig/robin/cancellation.rbs +25 -0
  157. data/sig/robin/cli.rbs +102 -0
  158. data/sig/robin/client.rbs +118 -0
  159. data/sig/robin/config.rbs +341 -0
  160. data/sig/robin/cron.rbs +149 -0
  161. data/sig/robin/debounce.rbs +36 -0
  162. data/sig/robin/deploy.rbs +24 -0
  163. data/sig/robin/embedded.rbs +25 -0
  164. data/sig/robin/encryption.rbs +49 -0
  165. data/sig/robin/events.rbs +10 -0
  166. data/sig/robin/expiry.rbs +17 -0
  167. data/sig/robin/fetcher.rbs +134 -0
  168. data/sig/robin/gvl.rbs +39 -0
  169. data/sig/robin/health.rbs +30 -0
  170. data/sig/robin/iterable_job.rbs +99 -0
  171. data/sig/robin/job.rbs +173 -0
  172. data/sig/robin/job_class.rbs +55 -0
  173. data/sig/robin/job_logger.rbs +56 -0
  174. data/sig/robin/launcher.rbs +191 -0
  175. data/sig/robin/leader.rbs +43 -0
  176. data/sig/robin/limiter.rbs +317 -0
  177. data/sig/robin/manager.rbs +54 -0
  178. data/sig/robin/metrics.rbs +122 -0
  179. data/sig/robin/middleware.rbs +106 -0
  180. data/sig/robin/opentelemetry.rbs +51 -0
  181. data/sig/robin/periodic.rbs +81 -0
  182. data/sig/robin/processor.rbs +107 -0
  183. data/sig/robin/profiler.rbs +60 -0
  184. data/sig/robin/progress.rbs +22 -0
  185. data/sig/robin/rails/app_loader.rbs +10 -0
  186. data/sig/robin/rails/current_attributes.rbs +44 -0
  187. data/sig/robin/rails/error_reporter.rbs +21 -0
  188. data/sig/robin/rails/notifications.rbs +9 -0
  189. data/sig/robin/rails/transactional_push.rbs +16 -0
  190. data/sig/robin/rails.rbs +8 -0
  191. data/sig/robin/recovery.rbs +29 -0
  192. data/sig/robin/redis/connection.rbs +224 -0
  193. data/sig/robin/redis/pool.rbs +89 -0
  194. data/sig/robin/redis/resp.rbs +29 -0
  195. data/sig/robin/reliable_push.rbs +77 -0
  196. data/sig/robin/retrier.rbs +72 -0
  197. data/sig/robin/scheduler.rbs +40 -0
  198. data/sig/robin/scripts.rbs +36 -0
  199. data/sig/robin/statsd.rbs +55 -0
  200. data/sig/robin/swarm.rbs +133 -0
  201. data/sig/robin/systemd.rbs +13 -0
  202. data/sig/robin/testing.rbs +80 -0
  203. data/sig/robin/top.rbs +79 -0
  204. data/sig/robin/unique.rbs +72 -0
  205. data/sig/robin/version.rbs +3 -0
  206. data/sig/robin/warmup.rbs +44 -0
  207. data/sig/robin/web/api.rbs +196 -0
  208. data/sig/robin/web/batch_status.rbs +25 -0
  209. data/sig/robin/web/extension.rbs +106 -0
  210. data/sig/robin/web/permissions.rbs +14 -0
  211. data/sig/robin/web/prometheus.rbs +33 -0
  212. data/sig/robin/web/router.rbs +53 -0
  213. data/sig/robin/web/static.rbs +42 -0
  214. data/sig/robin/web.rbs +140 -0
  215. data/sig/robin.rbs +116 -0
  216. data/sig/robin_resp.rbs +55 -0
  217. metadata +294 -0
@@ -0,0 +1,125 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Owns the worker threads of one capsule.
5
+ class Manager
6
+ # Worker deaths less than RESPAWN_WINDOW seconds apart count as a
7
+ # streak; each one in a streak doubles the wait before the replacement
8
+ # starts (1s, 2s, 4s... up to MAX_RESPAWN_DELAY), so a persistent fault
9
+ # can't spin up threads in a hot loop.
10
+ RESPAWN_WINDOW = 60
11
+ MAX_RESPAWN_DELAY = 30
12
+
13
+ attr_reader :capsule, :fetcher, :launcher
14
+
15
+ def initialize(capsule, launcher)
16
+ @capsule = capsule
17
+ @config = capsule.config
18
+ @launcher = launcher
19
+ @fetcher = Fetcher.new(capsule, launcher.identity)
20
+ @mutex = Mutex.new
21
+ @slots = concurrency # the next new worker's number
22
+ @processors = Array.new(@slots) { |i| Processor.new(self, capsule, @fetcher, i) }
23
+ end
24
+
25
+ # How many workers it runs: the capsule's concurrency, until #resize.
26
+ def concurrency = @mutex.synchronize { capsule.concurrency }
27
+
28
+ # Runs +count+ workers from now on. New ones start right away (or with
29
+ # the others, before #start); ones no longer needed finish their
30
+ # current job, then stop.
31
+ def resize(count)
32
+ @mutex.synchronize do
33
+ return if @stopping
34
+
35
+ active = @processors.reject(&:retired?)
36
+ if count > active.size
37
+ added = Array.new(count - active.size) { Processor.new(self, capsule, @fetcher, (@slots += 1) - 1) }
38
+ @processors.concat(added)
39
+ added.each { |processor| processor.start(0) } if @started
40
+ else
41
+ active.last(active.size - count).each(&:retire)
42
+ end
43
+ capsule.resized(count)
44
+ end
45
+ end
46
+
47
+ def start
48
+ @mutex.synchronize do
49
+ @started = true
50
+ @processors.each { |processor| processor.start(0) }
51
+ end
52
+ end
53
+
54
+ def processors
55
+ @mutex.synchronize { @processors.dup }
56
+ end
57
+
58
+ def quiet
59
+ @stopping = true
60
+ processors.each(&:terminate)
61
+ end
62
+
63
+ # Waits for running jobs until +deadline+ (monotonic seconds), then
64
+ # interrupts whatever is still running and waits for it to wind down
65
+ # until +hard_deadline+ (shared by every capsule).
66
+ def stop(deadline, hard_deadline)
67
+ quiet
68
+ sleep 0.05 while processors.any?(&:alive?) && monotonic < deadline
69
+ remaining = processors.select(&:alive?)
70
+ return if remaining.empty?
71
+
72
+ @config.logger.warn("interrupting #{remaining.size} job(s) still running after #{@config.timeout}s")
73
+ Events.emit(@config, "jobs.interrupted", count: remaining.size)
74
+ remaining.each(&:kill)
75
+ remaining.each { |p| p.thread&.join([hard_deadline - monotonic, 0].max) }
76
+ end
77
+
78
+ def processor_stopped(processor)
79
+ @mutex.synchronize { @processors.delete(processor) }
80
+ @launcher.work_finished_cleanup(processor)
81
+ end
82
+
83
+ def processor_died(processor, exception)
84
+ @config.handle_exception(exception, {context: "worker thread died"})
85
+ @launcher.work_finished_cleanup(processor)
86
+ delay = @mutex.synchronize do
87
+ index = @processors.index(processor)
88
+ @processors.delete(processor)
89
+ unless @stopping || processor.retired?
90
+ replacement = Processor.new(self, capsule, @fetcher, index || @processors.size)
91
+ @processors << replacement
92
+ respawn_delay.tap { |d| replacement.start(d) }
93
+ end
94
+ end
95
+ Events.emit(@config, "worker.died", error: exception, delay: delay)
96
+ end
97
+
98
+ def work_started(processor, unit)
99
+ @launcher.work_started(processor, unit)
100
+ end
101
+
102
+ def work_finished(processor, payload, elapsed_ms, ok)
103
+ @launcher.work_finished(processor, payload, elapsed_ms, ok)
104
+ end
105
+
106
+ private
107
+
108
+ # Seconds to wait before replacing a dead worker. Called with the
109
+ # mutex held.
110
+ def respawn_delay
111
+ now = monotonic
112
+ @deaths = (@last_death && now - @last_death < RESPAWN_WINDOW) ? @deaths + 1 : 0
113
+ @last_death = now
114
+ return 0 if @deaths.zero?
115
+
116
+ delay = [2**(@deaths - 1), MAX_RESPAWN_DELAY].min #: Integer
117
+ @config.logger.warn("worker thread died #{@deaths + 1} times in a row; restarting it in #{delay}s")
118
+ delay
119
+ end
120
+
121
+ def monotonic
122
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,303 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Per-job-class execution metrics.
5
+ #
6
+ # Workers aggregate counts, total time and a latency histogram in memory;
7
+ # the heartbeat flushes them to Redis once every few seconds, so recording
8
+ # costs no extra round-trips per job.
9
+ #
10
+ # Storage (UTC):
11
+ # robin:m:<YYYYMMDDHHMM> hash "<Class>|s|f|ms" counts, 8h TTL
12
+ # robin:h:<Class>:<YYYYMMDDHHMM> hash bucket index => count, 8h TTL
13
+ # robin:mh:<YYYYMMDDHH> hash hourly rollup, kept metrics_retention_days
14
+ # robin:qh:<YYYYMMDDHH> hash "<queue>|size" / "<queue>|latency" hourly maxima
15
+ # robin:qm:<YYYYMMDDHHMM> hash the same, per minute, 8h TTL
16
+ module Metrics
17
+ # Upper bounds (ms) of the histogram buckets; the last is open-ended.
18
+ BUCKETS = [5, 10, 20, 35, 50, 75, 100, 150, 250, 400, 600, 1_000, 1_500, 2_500, 4_000,
19
+ 6_000, 10_000, 15_000, 25_000, 40_000, 60_000, 120_000, 300_000, 600_000, Float::INFINITY].freeze
20
+
21
+ MINUTE_TTL = 8 * 3600
22
+
23
+ # The daily job totals (stat:processed:<YYYY-MM-DD>, stat:failed:...)
24
+ # are two small keys a day, kept five years.
25
+ DAILY_TOTALS_TTL = 5 * 365 * 86_400
26
+
27
+ class << self
28
+ def bucket_for(ms)
29
+ BUCKETS.bsearch_index { |b| ms <= b } #: Integer
30
+ end
31
+
32
+ def minute_stamp(time) = time.getutc.strftime("%Y%m%d%H%M")
33
+
34
+ def hour_stamp(time) = time.getutc.strftime("%Y%m%d%H")
35
+
36
+ # Records queue sizes and latencies (leader only, once a minute), in
37
+ # two round trips.
38
+ # mutant:disable -- Connection#evalsha splats ARGV, so passing the
39
+ # prefix bare or wrapped in an Array is identical.
40
+ def sample_queues(config, conn, time = Time.now)
41
+ found = conn.evalsha(Scripts::QUEUE_SAMPLES, [config.key("queues")], ["#{config.key("queue")}:"])
42
+ return if found.empty?
43
+
44
+ samples = found.each_slice(3).to_h { |q, size, oldest| [q, {size: size, latency: Queue.latency_of(oldest)}] }
45
+ fields = samples.flat_map { |q, s| ["#{q}|size", s.fetch(:size), "#{q}|latency", s.fetch(:latency)] }
46
+ retention = config.metrics_retention_days * 86_400
47
+ conn.pipelined do |p|
48
+ p.evalsha(Scripts::HMAX, [config.key("qm", minute_stamp(time))], [MINUTE_TTL, *fields])
49
+ p.evalsha(Scripts::HMAX, [config.key("qh", hour_stamp(time))], [retention, *fields])
50
+ end
51
+ config.statsd&.queue_metrics(samples)
52
+ end
53
+ end
54
+
55
+ # Each thread records into a stripe of its own, so workers never wait
56
+ # on each other (with real threads, as on JRuby, one shared lock would
57
+ # serialize every job); draining merges the stripes.
58
+ class Collector
59
+ # One thread's counts.
60
+ class Stripe
61
+ attr_reader :thread
62
+
63
+ def initialize(thread)
64
+ @thread = thread
65
+ @mutex = Mutex.new
66
+ reset
67
+ end
68
+
69
+ def record(klass, ms, ok)
70
+ @mutex.synchronize do
71
+ entry = Collector.entry(@jobs, klass)
72
+ # Timings cover successful runs only: a job failing fast (or
73
+ # timing out) would skew the averages and the histogram.
74
+ if ok
75
+ entry[:s] += 1
76
+ entry[:ms] += ms
77
+ entry.fetch(:hist)[Metrics.bucket_for(ms)] += 1
78
+ else
79
+ entry[:f] += 1
80
+ end
81
+ @processed += 1
82
+ @failed += 1 unless ok
83
+ end
84
+ end
85
+
86
+ # Adds counts drained earlier.
87
+ def add(drained)
88
+ jobs, processed, failed = drained
89
+ @mutex.synchronize do
90
+ Collector.merge(@jobs, jobs)
91
+ @processed += processed
92
+ @failed += failed
93
+ end
94
+ end
95
+
96
+ def drain
97
+ @mutex.synchronize do
98
+ snapshot = [@jobs, @processed, @failed] #: Metrics::drained
99
+ reset
100
+ snapshot
101
+ end
102
+ end
103
+
104
+ private
105
+
106
+ def reset
107
+ @jobs = {}
108
+ @processed = 0
109
+ @failed = 0
110
+ end
111
+ end
112
+
113
+ def self.entry(jobs, klass) = (jobs[klass] ||= {s: 0, f: 0, ms: 0.0, hist: Hash.new(0)})
114
+
115
+ # Adds +from+'s per-class counts to +into+'s.
116
+ def self.merge(into, from)
117
+ from.each do |klass, data|
118
+ entry = entry(into, klass)
119
+ entry[:s] += data.fetch(:s)
120
+ entry[:f] += data.fetch(:f)
121
+ entry[:ms] += data.fetch(:ms)
122
+ data.fetch(:hist).each { |b, c| entry.fetch(:hist)[b] += c }
123
+ end
124
+ end
125
+
126
+ def initialize
127
+ @mutex = Mutex.new
128
+ @stripes = []
129
+ @key = :"robin_metrics_#{object_id}"
130
+ end
131
+
132
+ def record(klass, ms, ok) = stripe.record(klass, ms, ok)
133
+
134
+ # [per-class counts, processed, failed] since the last drain. Stripes
135
+ # of threads that have finished go once they're drained.
136
+ def drain
137
+ stripes = @mutex.synchronize do
138
+ all = @stripes
139
+ @stripes = all.select { |s| s.thread.alive? }
140
+ all
141
+ end
142
+ jobs = {} #: Metrics::jobs
143
+ processed = failed = 0
144
+ stripes.each do |stripe|
145
+ stripe_jobs, stripe_processed, stripe_failed = stripe.drain
146
+ Collector.merge(jobs, stripe_jobs)
147
+ processed += stripe_processed
148
+ failed += stripe_failed
149
+ end
150
+ [jobs, processed, failed]
151
+ end
152
+
153
+ # Puts drained data back (used when a flush fails).
154
+ def restore(drained) = stripe.add(drained)
155
+
156
+ # Writes what was recorded since the last flush in one pipeline.
157
+ def flush(config, conn, time = Time.now)
158
+ drained = nil #: Metrics::drained?
159
+ conn.pipelined { |p| drained = queue(config, p, time) }
160
+ # (Pipelined runs the block, so drained is set.)
161
+ flushed(config, drained) # steep:ignore ArgumentTypeMismatch
162
+ rescue
163
+ restore(drained) if drained
164
+ raise
165
+ end
166
+
167
+ # Queues the commands that write what was recorded since the last
168
+ # flush on +pipe+ (a pipeline or transaction about to be sent), and
169
+ # returns what they write: pass it to #flushed once they're sent, or
170
+ # to #restore if sending them failed.
171
+ def queue(config, pipe, time = Time.now)
172
+ drained = drain
173
+ write(config, pipe, time, drained) unless drained.fetch(1).zero?
174
+ drained
175
+ end
176
+
177
+ # Reports queued metrics to StatsD once they're written.
178
+ def flushed(config, drained)
179
+ jobs, processed, _failed = drained
180
+ config.statsd&.job_metrics(jobs) unless processed.zero?
181
+ end
182
+
183
+ private
184
+
185
+ # This thread's stripe, made on first use.
186
+ def stripe
187
+ Thread.current[@key] ||= Stripe.new(Thread.current).tap do |s|
188
+ @mutex.synchronize { @stripes << s }
189
+ end
190
+ end
191
+
192
+ def write(config, p, time, drained)
193
+ jobs, processed, failed = drained
194
+ minute = Metrics.minute_stamp(time)
195
+ hour = Metrics.hour_stamp(time)
196
+ day = time.getutc.strftime("%Y-%m-%d")
197
+ mkey = config.key("m", minute)
198
+ hkey = config.key("mh", hour)
199
+ p.call("INCRBY", config.key("stat", "processed"), processed)
200
+ p.call("INCRBY", config.key("stat", "processed", day), processed)
201
+ p.call("EXPIRE", config.key("stat", "processed", day), DAILY_TOTALS_TTL)
202
+ if failed > 0
203
+ p.call("INCRBY", config.key("stat", "failed"), failed)
204
+ p.call("INCRBY", config.key("stat", "failed", day), failed)
205
+ p.call("EXPIRE", config.key("stat", "failed", day), DAILY_TOTALS_TTL)
206
+ end
207
+ jobs.each do |klass, d|
208
+ [mkey, hkey].each do |key|
209
+ if d.fetch(:s) > 0
210
+ p.call("HINCRBY", key, "#{klass}|s", d.fetch(:s))
211
+ p.call("HINCRBYFLOAT", key, "#{klass}|ms", d.fetch(:ms).round(1))
212
+ end
213
+ p.call("HINCRBY", key, "#{klass}|f", d.fetch(:f)) if d.fetch(:f) > 0
214
+ end
215
+ hist = config.key("h", klass, minute)
216
+ d.fetch(:hist).each { |bucket, count| p.call("HINCRBY", hist, bucket, count) }
217
+ p.call("EXPIRE", hist, MINUTE_TTL)
218
+ end
219
+ p.call("EXPIRE", mkey, MINUTE_TTL)
220
+ p.call("EXPIRE", hkey, config.metrics_retention_days * 86_400)
221
+ end
222
+ end
223
+
224
+ # Query helpers for the web UI and API.
225
+ class Query
226
+ GRANULARITIES = {minute: [60, "qm", :minute_stamp], hour: [3600, "qh", :hour_stamp]} #: Hash[Symbol, [Integer, String, Symbol]]
227
+ .freeze
228
+
229
+ def initialize(config = Robin.config)
230
+ @config = config
231
+ end
232
+
233
+ # Per-class totals over the last +minutes+ minutes, plus a per-minute
234
+ # series for charting. Returns {"classes" => {...}, "series" => [...]}
235
+ def recent(minutes: 60, now: Time.now)
236
+ stamps = (0...minutes).map { |i| Metrics.minute_stamp(now - i * 60) }.reverse
237
+ rows = @config.redis do |c|
238
+ c.pipelined { |p| stamps.each { |s| p.call("HGETALL", @config.key("m", s)) } }
239
+ end
240
+ summarize(stamps, rows)
241
+ end
242
+
243
+ # Hourly history over the last +hours+ hours (historical metrics).
244
+ def history(hours: 24 * 7, now: Time.now)
245
+ stamps = (0...hours).map { |i| Metrics.hour_stamp(now - i * 3600) }.reverse
246
+ rows = @config.redis do |c|
247
+ c.pipelined { |p| stamps.each { |s| p.call("HGETALL", @config.key("mh", s)) } }
248
+ end
249
+ summarize(stamps, rows)
250
+ end
251
+
252
+ # Queue size/latency maxima, per minute (granularity :minute) or per
253
+ # hour (granularity :hour). Other granularities raise KeyError.
254
+ def queue_history(granularity: :minute, points: 60, now: Time.now)
255
+ step, prefix, fmt = GRANULARITIES.fetch(granularity)
256
+ stamps = (0...points).map { |i| Metrics.public_send(fmt, now - i * step) }.reverse
257
+ rows = @config.redis do |c|
258
+ c.pipelined { |p| stamps.each { |s| p.call("HGETALL", @config.key(prefix, s)) } }
259
+ end
260
+ stamps.zip(rows).map do |stamp, row|
261
+ data = Hash.new { |h, k| h[k] = {} }
262
+ row.each_slice(2) do |field, value|
263
+ queue, _, kind = field.rpartition("|")
264
+ data[queue][kind] = value.to_i
265
+ end
266
+ {"t" => stamp, "queues" => data}
267
+ end
268
+ end
269
+
270
+ # Latency histogram for one class over the last +minutes+ minutes.
271
+ def histogram(klass, minutes: 60, now: Time.now)
272
+ stamps = (0...minutes).map { |i| Metrics.minute_stamp(now - i * 60) }
273
+ rows = @config.redis do |c|
274
+ c.pipelined { |p| stamps.each { |s| p.call("HGETALL", @config.key("h", klass, s)) } }
275
+ end
276
+ totals = Array.new(BUCKETS.size, 0)
277
+ rows.each { |row| row.each_slice(2) { |b, n| totals[Integer(b)] += Integer(n) } }
278
+ BUCKETS.each_with_index.map { |limit, i| {"le" => (limit unless limit.infinite?), "count" => totals.fetch(i)} }
279
+ end
280
+
281
+ private
282
+
283
+ def summarize(stamps, rows)
284
+ classes = Hash.new { |h, k| h[k] = {"s" => 0, "f" => 0, "ms" => 0.0} } #: Hash[String, Hash[String, Numeric]]
285
+ series = stamps.zip(rows).map do |stamp, row|
286
+ point = {"t" => stamp, "s" => 0, "f" => 0, "ms" => 0.0} #: Hash[String, untyped]
287
+ row.each_slice(2) do |field, value|
288
+ klass, _, kind = field.rpartition("|")
289
+ v = (kind == "ms") ? Float(value) : Integer(value)
290
+ classes[klass][kind] += v
291
+ point[kind] += v
292
+ end
293
+ point
294
+ end
295
+ classes.each_value do |c|
296
+ n = c.fetch("s") # "ms" only counts successful runs
297
+ c["avg_ms"] = n.zero? ? 0 : (c.fetch("ms") / n).round(1)
298
+ end
299
+ {"classes" => classes, "series" => series}
300
+ end
301
+ end
302
+ end
303
+ end
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Middleware wraps job pushes (client side) and job execution (server
5
+ # side), Rack style.
6
+ #
7
+ # Client middleware:
8
+ #
9
+ # class AddTenant
10
+ # def call(job_class, payload, queue, redis)
11
+ # payload["tenant"] ||= Current.tenant
12
+ # yield # return nil instead of yielding to stop the push
13
+ # end
14
+ # end
15
+ #
16
+ # Server middleware:
17
+ #
18
+ # class TimeIt
19
+ # def call(job, payload, queue)
20
+ # t = Time.now
21
+ # yield
22
+ # ensure
23
+ # puts "took #{Time.now - t}s"
24
+ # end
25
+ # end
26
+ #
27
+ # Middleware classes may define +initialize(*args)+ to receive arguments
28
+ # passed to Chain#add. If they include Robin::Middleware::ConfigAware
29
+ # they also get access to the Robin config.
30
+ #
31
+ # A middleware is built anew for every job, unless it includes
32
+ # Robin::Middleware::Stateless: then one instance serves every job, on
33
+ # every thread, so it must not keep per-job state in instance variables.
34
+ #
35
+ # A middleware that only matters to some jobs can say which with a class
36
+ # method +applies?+, taking +call+'s arguments: the chain skips it (and
37
+ # doesn't build it) when that's false.
38
+ #
39
+ # class Audit
40
+ # def self.applies?(_job, payload, _queue) = payload.key?("audit")
41
+ # ...
42
+ # end
43
+ module Middleware
44
+ module ConfigAware
45
+ attr_accessor :config
46
+
47
+ def redis(&) = config.redis(&)
48
+
49
+ def logger = config.logger
50
+ end
51
+
52
+ module Stateless; end
53
+
54
+ class Entry
55
+ attr_reader :klass
56
+
57
+ def initialize(config, klass, args)
58
+ @config = config
59
+ @klass = klass
60
+ @args = args
61
+ @stateless = klass.include?(Stateless)
62
+ @filtered = klass.respond_to?(:applies?)
63
+ end
64
+
65
+ def applies?(args) = !@filtered || @klass.applies?(*args)
66
+
67
+ # The instance to call. A stateless middleware's is built once (two
68
+ # threads racing to build it first is harmless).
69
+ def build
70
+ return @instance ||= instantiate if @stateless
71
+
72
+ instantiate
73
+ end
74
+
75
+ private
76
+
77
+ def instantiate
78
+ instance = @klass.new(*@args)
79
+ instance.config = @config if instance.respond_to?(:config=)
80
+ instance
81
+ end
82
+ end
83
+
84
+ class Chain
85
+ include Enumerable
86
+
87
+ def initialize(config)
88
+ @config = config
89
+ @entries = [] #: Array[Entry]
90
+ .freeze
91
+ end
92
+
93
+ # Frozen: change the chain with its methods.
94
+ attr_reader :entries
95
+
96
+ def each(&) = @entries.each(&)
97
+
98
+ # The chain is replaced on every change, never changed in place, so a
99
+ # job being invoked keeps the middleware it started with (and a dup'd
100
+ # chain changes independently).
101
+ def add(klass, *args)
102
+ @entries = (without(klass) << Entry.new(@config, klass, args)).freeze
103
+ self
104
+ end
105
+
106
+ def prepend(klass, *args)
107
+ @entries = without(klass).unshift(Entry.new(@config, klass, args)).freeze
108
+ self
109
+ end
110
+
111
+ def insert_before(target, klass, *args)
112
+ entries = without(klass)
113
+ index = entries.index { |e| e.klass.equal?(target) } || 0
114
+ @entries = entries.insert(index, Entry.new(@config, klass, args)).freeze
115
+ self
116
+ end
117
+
118
+ def insert_after(target, klass, *args)
119
+ entries = without(klass)
120
+ index = entries.index { |e| e.klass.equal?(target) } || (entries.size - 1)
121
+ @entries = entries.insert(index + 1, Entry.new(@config, klass, args)).freeze
122
+ self
123
+ end
124
+
125
+ def remove(klass)
126
+ @entries = without(klass).freeze
127
+ self
128
+ end
129
+
130
+ def exists?(klass)
131
+ any? { |e| e.klass.equal?(klass) }
132
+ end
133
+
134
+ def empty? = @entries.empty?
135
+
136
+ def clear
137
+ @entries = [] #: Array[Entry]
138
+ .freeze
139
+ self
140
+ end
141
+
142
+ # Invokes the chain around the block, outermost middleware first.
143
+ # Each middleware is instantiated as the chain reaches it. Returns the
144
+ # block's value, or whatever a middleware that halted the chain
145
+ # returned instead of yielding.
146
+ def invoke(*args, &block)
147
+ call_from(@entries, 0, args, block)
148
+ end
149
+
150
+ private
151
+
152
+ # Calls the first middleware from +index+ on that applies; yielding
153
+ # calls the next one. (No closures are built per job, so a long chain
154
+ # costs little.)
155
+ def call_from(entries, index, args, block)
156
+ index += 1 while (entry = entries.at(index)) && !entry.applies?(args)
157
+ return block.call unless entry
158
+
159
+ entry.build.call(*args) { call_from(entries, index + 1, args, block) }
160
+ end
161
+
162
+ def without(klass) = reject { |e| e.klass.equal?(klass) }
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "robin"
4
+ require "opentelemetry"
5
+
6
+ module Robin
7
+ # OpenTelemetry tracing for pushing and running jobs. Needs the
8
+ # opentelemetry-api gem (and an SDK to export spans somewhere):
9
+ #
10
+ # require "robin/opentelemetry"
11
+ #
12
+ # Robin.configure do |config|
13
+ # Robin::OpenTelemetry.install(config)
14
+ # end
15
+ #
16
+ # Pushing a job opens a "publish <queue>" producer span and stores the
17
+ # trace context in the job ("otel"). Running it opens a "process <queue>"
18
+ # consumer span, a child of the push, so a request and the jobs it pushed
19
+ # share one trace. With +parent: :link+ each run starts its own trace
20
+ # instead, linked to the push (better for jobs that run long after they
21
+ # were pushed). Spans follow the messaging semantic conventions, plus
22
+ # robin.job.class and robin.job.retry_count.
23
+ module OpenTelemetry
24
+ def self.install(config, tracer_provider: ::OpenTelemetry.tracer_provider, parent: :child)
25
+ raise ArgumentError, "parent: must be :child or :link" unless %i[child link].include?(parent)
26
+
27
+ tracer = tracer_provider.tracer("robin", VERSION)
28
+ config.client_middleware { |chain| chain.prepend(ClientMiddleware, tracer) }
29
+ config.server_middleware { |chain| chain.prepend(ServerMiddleware, tracer, parent) }
30
+ config
31
+ end
32
+
33
+ def self.attributes(payload, queue, operation)
34
+ attrs = {
35
+ "messaging.system" => "robin",
36
+ "messaging.operation.type" => operation,
37
+ "messaging.destination.name" => queue,
38
+ "messaging.message.id" => payload.fetch("jid"),
39
+ "robin.job.class" => payload.fetch("class")
40
+ }
41
+ attrs["robin.job.retry_count"] = payload.fetch("retry_count") if payload.key?("retry_count")
42
+ attrs
43
+ end
44
+
45
+ class ClientMiddleware
46
+ include Middleware::Stateless
47
+
48
+ def initialize(tracer)
49
+ @tracer = tracer
50
+ end
51
+
52
+ def call(_job_class, payload, queue, _redis)
53
+ @tracer.in_span("publish #{queue}", kind: :producer, attributes: OpenTelemetry.attributes(payload, queue, "publish")) do
54
+ carrier = {} #: Hash[String, String]
55
+ ::OpenTelemetry.propagation.inject(carrier)
56
+ payload["otel"] = carrier unless carrier.empty?
57
+ yield
58
+ end
59
+ end
60
+ end
61
+
62
+ class ServerMiddleware
63
+ include Middleware::Stateless
64
+
65
+ def initialize(tracer, parent)
66
+ @tracer = tracer
67
+ @parent = parent
68
+ end
69
+
70
+ def call(_job, payload, queue, &)
71
+ pushed = ::OpenTelemetry.propagation.extract(payload.fetch("otel", {}))
72
+ attributes = OpenTelemetry.attributes(payload, queue, "process")
73
+ name = "process #{queue}"
74
+ return ::OpenTelemetry::Context.with_current(pushed) { @tracer.in_span(name, kind: :consumer, attributes: attributes, &) } if @parent == :child
75
+
76
+ push = ::OpenTelemetry::Trace.current_span(pushed).context
77
+ links = [::OpenTelemetry::Trace::Link.new(push)] if push.valid?
78
+ detached { @tracer.in_span(name, kind: :consumer, attributes: attributes, links: links, &) }
79
+ end
80
+
81
+ private
82
+
83
+ # mutant:disable -- Context.current reads an attached nil as the
84
+ # empty context, so passing nil instead is the same.
85
+ def detached(&) = ::OpenTelemetry::Context.with_current(::OpenTelemetry::Context.empty, &)
86
+ end
87
+ end
88
+ end