robin 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/robin_resp/extconf.rb +29 -0
  13. data/ext/robin_resp/robin_resp.c +645 -0
  14. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  15. data/lib/generators/robin/install/install_generator.rb +39 -0
  16. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  17. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  18. data/lib/generators/robin/job/job_generator.rb +78 -0
  19. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  20. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  21. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  22. data/lib/puma/plugin/robin.rb +131 -0
  23. data/lib/robin/active_job.rb +144 -0
  24. data/lib/robin/api.rb +777 -0
  25. data/lib/robin/batch.rb +700 -0
  26. data/lib/robin/cancellation.rb +62 -0
  27. data/lib/robin/cli.rb +325 -0
  28. data/lib/robin/client.rb +355 -21
  29. data/lib/robin/config.rb +454 -4
  30. data/lib/robin/cron.rb +325 -0
  31. data/lib/robin/debounce.rb +108 -0
  32. data/lib/robin/deploy.rb +65 -0
  33. data/lib/robin/embedded.rb +53 -0
  34. data/lib/robin/encryption.rb +147 -0
  35. data/lib/robin/events.rb +60 -0
  36. data/lib/robin/expiry.rb +41 -0
  37. data/lib/robin/fetcher.rb +441 -0
  38. data/lib/robin/gvl.rb +62 -0
  39. data/lib/robin/health.rb +76 -0
  40. data/lib/robin/iterable_job.rb +294 -0
  41. data/lib/robin/job.rb +258 -0
  42. data/lib/robin/job_class.rb +157 -0
  43. data/lib/robin/job_logger.rb +121 -0
  44. data/lib/robin/launcher.rb +658 -0
  45. data/lib/robin/leader.rb +86 -0
  46. data/lib/robin/limiter.rb +639 -0
  47. data/lib/robin/manager.rb +125 -0
  48. data/lib/robin/metrics.rb +303 -0
  49. data/lib/robin/middleware.rb +165 -0
  50. data/lib/robin/opentelemetry.rb +88 -0
  51. data/lib/robin/periodic.rb +327 -0
  52. data/lib/robin/processor.rb +317 -0
  53. data/lib/robin/profiler.rb +140 -0
  54. data/lib/robin/progress.rb +46 -0
  55. data/lib/robin/rails/app_loader.rb +23 -0
  56. data/lib/robin/rails/current_attributes.rb +88 -0
  57. data/lib/robin/rails/error_reporter.rb +31 -0
  58. data/lib/robin/rails/notifications.rb +22 -0
  59. data/lib/robin/rails/railtie.rb +87 -0
  60. data/lib/robin/rails/transactional_push.rb +49 -0
  61. data/lib/robin/rails.rb +23 -0
  62. data/lib/robin/recovery.rb +135 -0
  63. data/lib/robin/redis/connection.rb +639 -0
  64. data/lib/robin/redis/pool.rb +190 -0
  65. data/lib/robin/redis/resp.rb +77 -0
  66. data/lib/robin/reliable_push.rb +153 -0
  67. data/lib/robin/retrier.rb +284 -0
  68. data/lib/robin/scheduler.rb +102 -0
  69. data/lib/robin/scripts.rb +331 -0
  70. data/lib/robin/statsd.rb +135 -0
  71. data/lib/robin/swarm.rb +319 -0
  72. data/lib/robin/systemd.rb +47 -0
  73. data/lib/robin/testing.rb +158 -0
  74. data/lib/robin/top.rb +178 -0
  75. data/lib/robin/unique.rb +198 -0
  76. data/lib/robin/version.rb +5 -0
  77. data/lib/robin/warmup.rb +94 -0
  78. data/lib/robin/web/api.rb +580 -0
  79. data/lib/robin/web/batch_status.rb +44 -0
  80. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  81. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  82. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  83. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  84. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  86. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  87. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  88. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  89. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  90. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  91. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  92. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  93. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  94. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  95. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  96. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  97. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  98. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  99. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  100. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  101. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  102. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  103. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  104. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  105. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  106. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  107. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  108. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  109. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  110. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  111. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  112. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  113. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  114. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  115. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  116. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  117. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  118. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  119. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  120. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  121. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  122. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  123. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  124. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  125. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  126. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  127. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  128. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  129. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  130. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  131. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  132. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  133. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  134. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  135. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  136. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  137. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  138. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  139. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  140. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  141. data/lib/robin/web/dist/index.html +19 -0
  142. data/lib/robin/web/extension.rb +193 -0
  143. data/lib/robin/web/permissions.rb +40 -0
  144. data/lib/robin/web/prometheus.rb +79 -0
  145. data/lib/robin/web/router.rb +150 -0
  146. data/lib/robin/web/static.rb +110 -0
  147. data/lib/robin/web.rb +196 -0
  148. data/lib/robin.rb +258 -8
  149. data/sig/manifest.yaml +9 -0
  150. data/sig/robin/api.rbs +442 -0
  151. data/sig/robin/batch.rbs +292 -0
  152. data/sig/robin/cancellation.rbs +25 -0
  153. data/sig/robin/cli.rbs +102 -0
  154. data/sig/robin/client.rbs +118 -0
  155. data/sig/robin/config.rbs +341 -0
  156. data/sig/robin/cron.rbs +149 -0
  157. data/sig/robin/debounce.rbs +36 -0
  158. data/sig/robin/deploy.rbs +24 -0
  159. data/sig/robin/embedded.rbs +25 -0
  160. data/sig/robin/encryption.rbs +49 -0
  161. data/sig/robin/events.rbs +10 -0
  162. data/sig/robin/expiry.rbs +17 -0
  163. data/sig/robin/fetcher.rbs +134 -0
  164. data/sig/robin/gvl.rbs +39 -0
  165. data/sig/robin/health.rbs +30 -0
  166. data/sig/robin/iterable_job.rbs +99 -0
  167. data/sig/robin/job.rbs +173 -0
  168. data/sig/robin/job_class.rbs +55 -0
  169. data/sig/robin/job_logger.rbs +56 -0
  170. data/sig/robin/launcher.rbs +191 -0
  171. data/sig/robin/leader.rbs +43 -0
  172. data/sig/robin/limiter.rbs +317 -0
  173. data/sig/robin/manager.rbs +54 -0
  174. data/sig/robin/metrics.rbs +122 -0
  175. data/sig/robin/middleware.rbs +106 -0
  176. data/sig/robin/opentelemetry.rbs +51 -0
  177. data/sig/robin/periodic.rbs +81 -0
  178. data/sig/robin/processor.rbs +107 -0
  179. data/sig/robin/profiler.rbs +60 -0
  180. data/sig/robin/progress.rbs +22 -0
  181. data/sig/robin/rails/app_loader.rbs +10 -0
  182. data/sig/robin/rails/current_attributes.rbs +44 -0
  183. data/sig/robin/rails/error_reporter.rbs +21 -0
  184. data/sig/robin/rails/notifications.rbs +9 -0
  185. data/sig/robin/rails/transactional_push.rbs +16 -0
  186. data/sig/robin/rails.rbs +8 -0
  187. data/sig/robin/recovery.rbs +29 -0
  188. data/sig/robin/redis/connection.rbs +224 -0
  189. data/sig/robin/redis/pool.rbs +89 -0
  190. data/sig/robin/redis/resp.rbs +29 -0
  191. data/sig/robin/reliable_push.rbs +77 -0
  192. data/sig/robin/retrier.rbs +72 -0
  193. data/sig/robin/scheduler.rbs +40 -0
  194. data/sig/robin/scripts.rbs +36 -0
  195. data/sig/robin/statsd.rbs +55 -0
  196. data/sig/robin/swarm.rbs +133 -0
  197. data/sig/robin/systemd.rbs +13 -0
  198. data/sig/robin/testing.rbs +80 -0
  199. data/sig/robin/top.rbs +79 -0
  200. data/sig/robin/unique.rbs +72 -0
  201. data/sig/robin/version.rbs +3 -0
  202. data/sig/robin/warmup.rbs +44 -0
  203. data/sig/robin/web/api.rbs +196 -0
  204. data/sig/robin/web/batch_status.rbs +25 -0
  205. data/sig/robin/web/extension.rbs +106 -0
  206. data/sig/robin/web/permissions.rbs +14 -0
  207. data/sig/robin/web/prometheus.rbs +33 -0
  208. data/sig/robin/web/router.rbs +53 -0
  209. data/sig/robin/web/static.rbs +42 -0
  210. data/sig/robin/web.rbs +140 -0
  211. data/sig/robin.rbs +116 -0
  212. data/sig/robin_resp.rbs +55 -0
  213. metadata +274 -44
  214. data/bin/robin +0 -5
  215. data/lib/robin/actions/authenticate.rb +0 -38
  216. data/lib/robin/actions/empty_tweet.rb +0 -7
  217. data/lib/robin/actions/followers.rb +0 -22
  218. data/lib/robin/actions/help.rb +0 -16
  219. data/lib/robin/actions/home_timeline.rb +0 -9
  220. data/lib/robin/actions/new_tweet.rb +0 -11
  221. data/lib/robin/actions/unknown_flag.rb +0 -11
  222. data/lib/robin/actions/user_timeline.rb +0 -19
  223. data/lib/robin/args_parser.rb +0 -70
  224. data/lib/robin/pin_authenticator.rb +0 -31
  225. data/lib/robin/user.rb +0 -36
@@ -0,0 +1,140 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "zlib"
4
+
5
+ module Robin
6
+ # Profiles single job runs with Vernier, a sampling profiler (add
7
+ # `gem "vernier"`; CRuby only), to find out why a job is slow:
8
+ #
9
+ # ReportJob.set(profile: true).perform_async(42)
10
+ # ReportJob.set(profile: "big export").perform_async(43) # with a label
11
+ #
12
+ # The profile, in the Firefox Profiler format, is kept in Redis for a
13
+ # week (the newest KEEP of them). The dashboard lists them and opens them
14
+ # in https://profiler.firefox.com; from Ruby:
15
+ #
16
+ # Robin::Profiler.list # newest first, without the data
17
+ # Robin::Profiler.data(jid) # the profile JSON, or nil
18
+ #
19
+ # A process profiles one job at a time; others asking meanwhile run
20
+ # unprofiled. Failed runs are profiled too.
21
+ module Profiler
22
+ TTL = 7 * 86_400
23
+ KEEP = 100
24
+ MAX_BYTES = 16 * 1024 * 1024 # compressed; bigger profiles are dropped
25
+ LOCK = Mutex.new
26
+
27
+ # Loads Vernier once, warning when it can't.
28
+ def self.available?(config)
29
+ return @available unless @available.nil?
30
+
31
+ @available = begin
32
+ require "vernier"
33
+ true
34
+ rescue LoadError
35
+ config.logger.warn("profiling a job needs the vernier gem (CRuby); running it unprofiled")
36
+ false
37
+ end
38
+ end
39
+
40
+ # Runs the block under the profiler and stores the result. Returns the
41
+ # block's value.
42
+ def self.profile(config, payload)
43
+ collector = start(config)
44
+ return yield unless collector
45
+
46
+ started = Robin.now_ms
47
+ ok = false
48
+ begin
49
+ result = yield
50
+ ok = true
51
+ result
52
+ ensure
53
+ record(config, payload, collector, started, ok)
54
+ end
55
+ end
56
+
57
+ # A running collector, or nil if one can't start (e.g. the app is
58
+ # profiling something else).
59
+ def self.start(config)
60
+ Vernier::Collector.new(:wall, {}).tap(&:start)
61
+ rescue => e
62
+ config.logger.warn("couldn't start the profiler (#{e.class}: #{e.message}); running the job unprofiled")
63
+ nil
64
+ end
65
+
66
+ def self.record(config, payload, collector, started, ok)
67
+ duration = Robin.now_ms - started
68
+ data = Zlib.gzip(collector.stop.to_gecko)
69
+ jid = payload.fetch("jid")
70
+ if data.bytesize > MAX_BYTES
71
+ config.logger.warn("profile of #{payload.fetch("class")} jid=#{jid} is too big to keep (#{data.bytesize} bytes)")
72
+ return
73
+ end
74
+
75
+ label = payload.fetch("profile")
76
+ entry = {"jid" => jid, "class" => Robin.display_class(payload), "queue" => payload["queue"],
77
+ "label" => label.equal?(true) ? nil : label.to_s, "started_at" => started,
78
+ "duration_ms" => duration, "ok" => ok, "bytes" => data.bytesize}
79
+ config.redis do |conn|
80
+ conn.multi do |tx|
81
+ tx.call("SET", config.key("profile", jid), data, "EX", TTL)
82
+ tx.call("ZADD", config.key("profiles"), started, Robin.dump_json(entry))
83
+ tx.call("ZREMRANGEBYSCORE", config.key("profiles"), "-inf", "(#{started - TTL * 1000}")
84
+ tx.call("ZREMRANGEBYRANK", config.key("profiles"), 0, -KEEP - 1)
85
+ end
86
+ end
87
+ rescue => e
88
+ config.handle_exception(e, {context: "storing a job profile", job: payload})
89
+ end
90
+
91
+ # Stored profiles, newest first (without their data).
92
+ def self.list(config: Robin.config)
93
+ entries(config).map(&:last)
94
+ end
95
+
96
+ # The profile JSON for +jid+, or nil once it has expired.
97
+ def self.data(jid, config: Robin.config)
98
+ data = config.redis { |c| c.call("GET", config.key("profile", jid)) }
99
+ data && Zlib.gunzip(data)
100
+ end
101
+
102
+ # Deletes the profile of +jid+; true if there was one.
103
+ def self.delete(jid, config: Robin.config)
104
+ raws = entries(config).filter_map { |raw, entry| raw if entry.fetch("jid") == jid }
105
+ config.redis do |c|
106
+ c.multi do |tx|
107
+ tx.call("DEL", config.key("profile", jid))
108
+ tx.call("ZREM", config.key("profiles"), *raws) unless raws.empty?
109
+ end
110
+ end
111
+ !raws.empty?
112
+ end
113
+
114
+ # [raw, parsed] pairs, newest first.
115
+ def self.entries(config)
116
+ config.redis { |c| c.call("ZRANGE", config.key("profiles"), 0, -1, "REV") }.map { |raw| [raw, Robin.load_json(raw)] }
117
+ end
118
+ private_class_method :entries
119
+
120
+ # Profiles jobs pushed with a "profile" option. Installed by default;
121
+ # a no-op for every other job.
122
+ class ServerMiddleware
123
+ include Middleware::ConfigAware
124
+ include Middleware::Stateless
125
+
126
+ def self.applies?(_job, payload, _queue) = payload.key?("profile")
127
+
128
+ def call(_job, payload, _queue, &)
129
+ return yield unless payload["profile"] && Profiler.available?(config)
130
+ return yield unless LOCK.try_lock
131
+
132
+ begin
133
+ Profiler.profile(config, payload, &)
134
+ ensure
135
+ LOCK.unlock
136
+ end
137
+ end
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Live progress reporting for long jobs, shown on the web UI's Busy page.
5
+ #
6
+ # def perform(ids)
7
+ # ids.each_with_index do |id, i|
8
+ # process(id)
9
+ # progress!((i + 1) * 100 / ids.size, "#{i + 1}/#{ids.size} rows")
10
+ # end
11
+ # end
12
+ module Progress
13
+ TTL = 86_400
14
+
15
+ class << self
16
+ def report(jid, percent, message = nil, meta = {}, config: Robin.config)
17
+ data = entry(percent, message, meta)
18
+ config.redis { |c| c.call(*set_command(config, jid, data)) }
19
+ data
20
+ end
21
+
22
+ # What #report stores. A nil +percent+ is unknown (the Busy page
23
+ # shows the message, or meta's "iterations" count, without a bar).
24
+ def entry(percent, message = nil, meta = {})
25
+ data = {"pct" => percent&.to_f&.clamp(0, 100)&.round(1), "msg" => message, "at" => Robin.now_ms} #: entry
26
+ data["meta"] = meta unless meta.empty?
27
+ data
28
+ end
29
+
30
+ # The command #report sends, for callers that batch it with others.
31
+ def set_command(config, jid, data) = ["SET", config.key("progress", jid), Robin.dump_json(data), "EX", TTL]
32
+
33
+ def fetch(jid, config: Robin.config)
34
+ raw = config.redis { |c| c.call("GET", config.key("progress", jid)) }
35
+ raw && Robin.load_json(raw)
36
+ end
37
+
38
+ def fetch_many(jids, config: Robin.config)
39
+ return {} if jids.empty?
40
+
41
+ values = config.redis { |c| c.call("MGET", *jids.map { |j| config.key("progress", j) }) }
42
+ jids.zip(values).each_with_object({}) { |(j, v), h| h[j] = Robin.load_json(v) if v }
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "robin"
4
+
5
+ module Robin
6
+ module Rails
7
+ # Boots a Rails application directory for `bundle exec robin`: the
8
+ # CLI's first app loader (see Robin::CLI.app_loaders). Other
9
+ # directories are left to the next loader.
10
+ module AppLoader
11
+ # The Railtie is loaded first so it joins the app's initialization
12
+ # even when Bundler.require hasn't loaded robin (or loaded it before
13
+ # Rails).
14
+ def self.call(dir)
15
+ return false unless File.exist?(File.join(dir, "config", "application.rb"))
16
+
17
+ require "robin/rails"
18
+ require File.expand_path(File.join(dir, "config", "environment.rb"))
19
+ true
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_job"
4
+ require "active_job/base" # what ActiveJob::Arguments needs
5
+
6
+ module Robin
7
+ module Rails
8
+ # Carries ActiveSupport::CurrentAttributes from where a job is pushed
9
+ # into the job, so that e.g. Current.user and Current.tenant are set
10
+ # while it runs:
11
+ #
12
+ # config.robin.current_attributes = ["Current"] # class names
13
+ #
14
+ # Pushing saves each class's attributes in the job, the first listed
15
+ # class's under "cattr", the next under "cattr_1" and so on, serialized
16
+ # the way ActiveJob serializes arguments, so records travel as
17
+ # GlobalIDs. Running sets them around the job, inside the Rails
18
+ # executor (which resets them afterwards). Classes are looked up by
19
+ # name on every push and run, so code reloading keeps working.
20
+ #
21
+ # The attributes are stored in Redis as they are: unlike encrypted
22
+ # arguments, they're readable there and in the dashboard.
23
+ module CurrentAttributes
24
+ # The job field holding the attributes of the class at +index+ in
25
+ # the configured list.
26
+ def self.key_at(index) = index.zero? ? "cattr" : "cattr_#{index}"
27
+
28
+ # [field, class name] for each configured class.
29
+ def self.fields(names) = Array(names).each_with_index.map { |name, i| [key_at(i), name.to_s] }
30
+
31
+ # Saves the attributes into jobs being pushed.
32
+ class Save
33
+ include Robin::Middleware::Stateless
34
+
35
+ def initialize(names)
36
+ @fields = CurrentAttributes.fields(names)
37
+ end
38
+
39
+ def call(_job_class, payload, _queue, _redis)
40
+ @fields.each do |key, name|
41
+ next if payload.key?(key)
42
+
43
+ attributes = name.constantize.attributes
44
+ payload[key] = ::ActiveJob::Arguments.serialize(attributes) unless attributes.empty?
45
+ end
46
+ yield
47
+ end
48
+ end
49
+
50
+ # Sets the saved attributes while a job runs.
51
+ class Load
52
+ include Robin::Middleware::Stateless
53
+
54
+ def initialize(names)
55
+ @fields = CurrentAttributes.fields(names)
56
+ end
57
+
58
+ def call(_job, payload, _queue, &) = set(@fields.select { |key, _| payload.key?(key) }, payload, &)
59
+
60
+ private
61
+
62
+ # Nests one CurrentAttributes.set per class around the job. A class
63
+ # that no longer exists (renamed since the push) is skipped.
64
+ def set(fields, payload, &job)
65
+ return yield if fields.empty?
66
+
67
+ key, name = fields.fetch(0)
68
+ rest = fields.drop(1)
69
+ klass = name.safe_constantize
70
+ return set(rest, payload, &job) unless klass
71
+
72
+ # An attribute removed since the push is left out.
73
+ attributes = ::ActiveJob::Arguments.deserialize(payload.fetch(key)).to_h
74
+ attributes = attributes.select { |name, _| klass.method_defined?(:"#{name}=") }
75
+ klass.set(attributes) { set(rest, payload, &job) }
76
+ end
77
+ end
78
+
79
+ def self.install(config, names)
80
+ names = Array(names)
81
+ return if names.empty?
82
+
83
+ config.client_middleware { |chain| chain.add(Save, names) }
84
+ config.server_middleware { |chain| chain.prepend(Load, names) }
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ module Rails
5
+ # A Robin error handler that reports to the Rails error reporter
6
+ # (Rails.error), so every service subscribed to it sees Robin's errors.
7
+ # A failed job is reported as unhandled with source "robin.job"; Robin's
8
+ # own errors (e.g. Redis unreachable while fetching) as handled with
9
+ # source "robin". The context names the job, never its arguments.
10
+ #
11
+ # Rails.error reports an exception only once, so an app handler that
12
+ # also calls Rails.error.report doesn't double-report.
13
+ class ErrorReporter
14
+ JOB_FAILED = "job raised"
15
+
16
+ def initialize(reporter)
17
+ @reporter = reporter
18
+ end
19
+
20
+ def call(exception, context, _config)
21
+ job = context.fetch(:job, {})
22
+ failed = context[:context] == JOB_FAILED
23
+ details = {
24
+ robin: context[:context], queue: context.fetch(:queue, job["queue"]), jid: job["jid"],
25
+ job_class: Robin.display_class(job), retry_count: job["retry_count"], bid: job["bid"]
26
+ }
27
+ @reporter.report(exception, handled: !failed, source: failed ? "robin.job" : "robin", context: details.compact)
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/notifications"
4
+
5
+ module Robin
6
+ module Rails
7
+ # Publishes Robin's events (see Robin::Events) through
8
+ # ActiveSupport::Notifications as "<event>.robin":
9
+ #
10
+ # ActiveSupport::Notifications.subscribe("job.expired.robin") do |event|
11
+ # Rails.logger.info("expired: #{event.payload[:job]["jid"]}")
12
+ # end
13
+ #
14
+ # The Railtie adds it to Robin.config.event_handlers unless
15
+ # config.robin.notifications is false.
16
+ module Notifications
17
+ def self.call(name, payload)
18
+ ActiveSupport::Notifications.instrument("#{name}.robin", payload)
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails"
4
+ require "active_support/ordered_options"
5
+
6
+ module Robin
7
+ module Rails
8
+ # Wires Robin into a Rails app:
9
+ #
10
+ # * jobs run inside the Rails reloader and executor, so code reloading,
11
+ # query caches and connection checkin work as in a request
12
+ # * ActiveJob gets the :robin adapter and +robin_options+
13
+ # * Robin.config.environment follows Rails.env
14
+ # * outside the robin process (web, console), Robin logs to Rails.logger
15
+ # * errors are reported to Rails.error (see Robin::Rails::ErrorReporter)
16
+ # * failed jobs' backtraces go through Rails.backtrace_cleaner
17
+ # * what jobs log through Rails.logger is tagged with the job's class
18
+ # and jid (when the logger supports tags)
19
+ # * Robin's events are published through ActiveSupport::Notifications
20
+ # (see Robin::Rails::Notifications)
21
+ #
22
+ # Settings, e.g. in config/application.rb:
23
+ #
24
+ # config.robin.report_errors = false # default true
25
+ # config.robin.transactional_push = true # default false; see TransactionalPush
26
+ # config.robin.current_attributes = ["Current"] # see CurrentAttributes
27
+ # config.robin.notifications = false # default true
28
+ class Railtie < ::Rails::Railtie
29
+ config.robin = ActiveSupport::OrderedOptions.new
30
+ config.robin.report_errors = true
31
+ config.robin.transactional_push = false
32
+ config.robin.current_attributes = []
33
+ config.robin.notifications = true
34
+
35
+ # Before the app's initializers, so they can still override it.
36
+ initializer "robin.environment", before: :load_config_initializers do
37
+ Robin.config.environment = ::Rails.env.to_s
38
+ end
39
+
40
+ # Before ActiveJob's own hook selects the configured queue adapter.
41
+ initializer "robin.active_job", before: "active_job.set_configs" do
42
+ ActiveSupport.on_load(:active_job) { require "robin/active_job" }
43
+ end
44
+
45
+ config.after_initialize { |app| Railtie.integrate(app) }
46
+
47
+ def self.integrate(app, config: Robin.config)
48
+ options = app.config.robin
49
+ config.job_wrapper ||= job_wrapper(app)
50
+ # (lambda, not ->: Steep splats a -> lambda's one Array parameter.)
51
+ config.backtrace_cleaner ||= lambda { |lines| ::Rails.backtrace_cleaner.clean(lines) }
52
+ config.logger = ::Rails.logger if !Robin.server? && ::Rails.logger
53
+ if options.report_errors && config.error_handlers.none?(ErrorReporter)
54
+ config.error_handlers << ErrorReporter.new(::Rails.error)
55
+ end
56
+ config.push_deferrer ||= TransactionalPush if options.transactional_push
57
+ if options.notifications && !config.event_handlers.include?(Notifications)
58
+ config.event_handlers << Notifications
59
+ end
60
+ CurrentAttributes.install(config, options.current_attributes)
61
+ end
62
+
63
+ # Runs each job inside the app's reloader (and so its executor). A
64
+ # job's exception is carried out of the reloader and raised after it,
65
+ # so the executor doesn't report it: Robin does, once, with the job's
66
+ # context, and never for Robin::Discard.
67
+ def self.job_wrapper(app)
68
+ lambda do |payload, &job|
69
+ failure = nil #: Exception?
70
+ result = app.reloader.wrap do
71
+ tagged(payload, &job)
72
+ rescue Exception => e # rubocop:disable Lint/RescueException
73
+ failure = e
74
+ end
75
+ failure ? raise(failure) : result
76
+ end
77
+ end
78
+
79
+ def self.tagged(payload, &)
80
+ logger = ::Rails.logger
81
+ return yield unless logger.respond_to?(:tagged)
82
+
83
+ logger.tagged("#{Robin.display_class(payload)} jid=#{payload.fetch("jid")}", &)
84
+ end
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ module Rails
5
+ # A Config#push_deferrer that holds jobs pushed inside Active Record
6
+ # transactions until all of them commit (rolled back transactions drop
7
+ # them), so a job never runs before the data it needs is visible. It
8
+ # needs ActiveRecord.after_all_transactions_commit (Rails 7.2+); without
9
+ # it, jobs are pushed right away.
10
+ #
11
+ # The jobs pushed inside the same transaction wait together and go in
12
+ # one round trip once it commits.
13
+ module TransactionalPush
14
+ # [the transaction, the deliveries waiting on it] for this thread.
15
+ WAITING = :robin_transactional_push
16
+
17
+ def self.call(_payloads, &deliver)
18
+ return false unless defined?(::ActiveRecord.after_all_transactions_commit)
19
+
20
+ transaction = current_transaction
21
+ waiting = Thread.current[WAITING]
22
+ if transaction && waiting&.first.equal?(transaction)
23
+ waiting.last << deliver
24
+ return true
25
+ end
26
+
27
+ deliveries = [deliver]
28
+ Thread.current[WAITING] = [transaction, deliveries] if transaction
29
+ ActiveRecord.after_all_transactions_commit do
30
+ Thread.current[WAITING] = nil if Thread.current[WAITING]&.last.equal?(deliveries)
31
+ Client.together { deliveries.each(&:call) }
32
+ end
33
+ true
34
+ end
35
+
36
+ # The innermost open transaction on the primary database, if any.
37
+ # Jobs pushed in it are delivered when the same transactions commit.
38
+ # One rolled back is never current again, so the deliveries left
39
+ # waiting on it are never added to (nor made).
40
+ def self.current_transaction
41
+ return unless defined?(ActiveRecord::Base.current_transaction)
42
+
43
+ transaction = ActiveRecord::Base.current_transaction
44
+ transaction if transaction.open?
45
+ end
46
+ private_class_method :current_transaction
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "robin"
4
+ require_relative "rails/app_loader"
5
+ require_relative "rails/error_reporter"
6
+ require_relative "rails/transactional_push"
7
+ require_relative "rails/current_attributes"
8
+ require_relative "rails/notifications"
9
+ require_relative "rails/railtie"
10
+
11
+ module Robin
12
+ # The server finds ActiveJob jobs' wrapper class by name, possibly before
13
+ # anything has loaded ActiveJob.
14
+ autoload :ActiveJob, "robin/active_job"
15
+
16
+ # Rails integration for Robin: a Railtie (see Robin::Rails::Railtie), the
17
+ # ActiveJob adapter, generators, Rails.error reporting and
18
+ # transaction-aware pushes. Robin itself works with any Ruby or Rack app:
19
+ # this is loaded with it in a Rails app (one that loaded Rails first, as
20
+ # Bundler.require does), or with `require "robin/rails"`.
21
+ module Rails
22
+ end
23
+ end
@@ -0,0 +1,135 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Puts jobs held by crashed processes back on their queues.
5
+ #
6
+ # Each server registers its in-flight set (see Robin::Fetcher) in
7
+ # robin:inflights. A process whose heartbeat key has expired is dead;
8
+ # its in-flight jobs are moved back to the front of their queues. Jobs
9
+ # that keep getting orphaned (they crash their process every time) are
10
+ # moved to the dead set.
11
+ #
12
+ # A process that is shutting down writes a stopping:<identity> marker
13
+ # first; if it dies before it finishes, its jobs were interrupted by the
14
+ # shutdown, not crashing it, so recovering them doesn't count toward the
15
+ # poison-pill limit.
16
+ module Recovery
17
+ class << self
18
+ # Recovers every dead process's in-flight jobs, in three round trips
19
+ # however many there are (four when one is a poison pill). Returns
20
+ # the number of jobs recovered. +pool+ is where the Redis connection
21
+ # comes from (a server uses its housekeeping pool).
22
+ def run(config, pool: config.redis_pool)
23
+ pool.with do |conn|
24
+ # In order, so processes are logged (and recovered) in order.
25
+ sets = conn.call("SMEMBERS", config.key("inflights")).sort
26
+ prefix = "#{config.key("inflight")}:"
27
+ identities = sets.map { |set| set.delete_prefix(prefix) }
28
+ checks = conn.pipelined do |p|
29
+ identities.each do |identity|
30
+ p.call("EXISTS", config.process_key(identity))
31
+ p.call("EXISTS", config.key("stopping", identity))
32
+ end
33
+ end
34
+ dead = sets.zip(identities, checks.each_slice(2)).filter_map do |set, identity, (alive, stopping)|
35
+ [set, identity, stopping.zero?] if alive.zero?
36
+ end
37
+ recover(config, conn, dead).sum
38
+ end
39
+ end
40
+
41
+ # Moves all jobs in a process's in-flight keys (its in-flight set
42
+ # +key+) back to their queues, in one round trip (two when one is a
43
+ # poison pill), and unregisters the keys; returns how many. With
44
+ # +count+ false (a graceful shutdown) the jobs don't count toward
45
+ # the poison-pill limit.
46
+ def drain(config, conn, key, count:)
47
+ done = queue_recover(config, conn, key, count)
48
+ settle(config, conn, [[done, count]]).sum
49
+ end
50
+
51
+ private
52
+
53
+ # Recovers the jobs of +dead+ processes ([in-flight set, identity,
54
+ # count?] each) and forgets the processes; returns how many jobs each
55
+ # had.
56
+ def recover(config, conn, dead)
57
+ replies = conn.pipelined do |p|
58
+ dead.each do |set, identity, count|
59
+ queue_recover(config, p, set, count)
60
+ forget(config, p, set, identity)
61
+ end
62
+ end
63
+ counts = settle(config, conn, replies.each_slice(2).map(&:first).zip(dead.map(&:last)))
64
+ counts.zip(dead) do |recovered, (_set, identity, _count)|
65
+ next if recovered.zero?
66
+
67
+ config.logger.warn("recovered #{recovered} in-flight job(s) from dead process #{identity}")
68
+ config.statsd&.increment("jobs.recovered", recovered)
69
+ end
70
+ counts
71
+ end
72
+
73
+ # mutant:disable -- Connection#evalsha splats ARGV, so passing the
74
+ # identity bare or wrapped in an Array is identical.
75
+ def forget(config, pipe, set, identity)
76
+ pipe.evalsha(Scripts::FORGET, [set, config.key("inflights"), config.key("stopping", identity), config.key("processes")], [identity])
77
+ end
78
+
79
+ # Runs RECOVER on +redis+ (a connection), or queues it (a pipeline).
80
+ def queue_recover(config, redis, key, count)
81
+ redis.evalsha(Scripts::RECOVER, [key, config.key("queues"), config.key("dead")],
82
+ ["#{config.key("queue")}:", config.max_recoveries, Robin.epoch_seconds(Robin.now_ms), count, "#{config.key("orphan")}:"])
83
+ end
84
+
85
+ # Takes RECOVER's replies, each with whether its jobs count: marks
86
+ # the poison pills among them (in one round trip) and runs the
87
+ # recovery handlers. Returns how many jobs each reply had.
88
+ def settle(config, conn, results)
89
+ jobs = results.flat_map do |done, count|
90
+ # (RECOVER's reply is pairs, so each_slice(2) never yields a nil raw.)
91
+ done.each_slice(2).map { |status, raw| [status == "dead", raw, count] } # steep:ignore BlockBodyTypeMismatch
92
+ end #: Array[[bool, String, bool?]]
93
+ poison = jobs.select(&:first).map { |_dead, raw, _count| raw }
94
+ mark_poison(config, conn, poison) unless poison.empty?
95
+ jobs.each { |dead, raw, count| recovered_job(config, raw, dead) if count }
96
+ results.map { |done, _count| done.size / 2 }
97
+ end
98
+
99
+ # Runs config.recovery_handlers for a job recovered after a crash.
100
+ def recovered_job(config, raw, poison)
101
+ return if config.recovery_handlers.empty? && config.event_handlers.empty?
102
+
103
+ payload = Robin.load_json(raw)
104
+ Events.emit(config, "job.recovered", job: payload, poison: poison)
105
+ config.recovery_handlers.each do |handler|
106
+ handler.call(payload, poison)
107
+ rescue => e
108
+ config.handle_exception(e, {context: "recovery handler", job: payload})
109
+ end
110
+ rescue JSON::ParserError
111
+ nil
112
+ end
113
+
114
+ def mark_poison(config, conn, raws)
115
+ payloads = raws.map do |raw|
116
+ payload = Robin.load_json(raw).except("enqueued_at")
117
+ payload["error_class"] = "Robin::PoisonPill"
118
+ payload["error_message"] = "job was running when its process died #{config.max_recoveries + 1} times"
119
+ payload["failed_at"] ||= Robin.now_ms
120
+ payload
121
+ end
122
+ conn.multi do |tx|
123
+ raws.zip(payloads) do |raw, payload|
124
+ tx.call("ZREM", config.key("dead"), raw)
125
+ tx.call("ZADD", config.key("dead"), Robin.epoch_seconds(Robin.now_ms), Robin.dump_json(payload))
126
+ end
127
+ end
128
+ payloads.each do |payload|
129
+ config.logger.error("moved poison-pill job #{payload["class"]} jid=#{payload.fetch("jid")} to the dead set")
130
+ config.statsd&.increment("jobs.poison", tags: {class: payload["class"]})
131
+ end
132
+ end
133
+ end
134
+ end
135
+ end