robin 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/robin_resp/extconf.rb +29 -0
  13. data/ext/robin_resp/robin_resp.c +645 -0
  14. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  15. data/lib/generators/robin/install/install_generator.rb +39 -0
  16. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  17. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  18. data/lib/generators/robin/job/job_generator.rb +78 -0
  19. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  20. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  21. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  22. data/lib/puma/plugin/robin.rb +131 -0
  23. data/lib/robin/active_job.rb +144 -0
  24. data/lib/robin/api.rb +777 -0
  25. data/lib/robin/batch.rb +700 -0
  26. data/lib/robin/cancellation.rb +62 -0
  27. data/lib/robin/cli.rb +325 -0
  28. data/lib/robin/client.rb +355 -21
  29. data/lib/robin/config.rb +454 -4
  30. data/lib/robin/cron.rb +325 -0
  31. data/lib/robin/debounce.rb +108 -0
  32. data/lib/robin/deploy.rb +65 -0
  33. data/lib/robin/embedded.rb +53 -0
  34. data/lib/robin/encryption.rb +147 -0
  35. data/lib/robin/events.rb +60 -0
  36. data/lib/robin/expiry.rb +41 -0
  37. data/lib/robin/fetcher.rb +441 -0
  38. data/lib/robin/gvl.rb +62 -0
  39. data/lib/robin/health.rb +76 -0
  40. data/lib/robin/iterable_job.rb +294 -0
  41. data/lib/robin/job.rb +258 -0
  42. data/lib/robin/job_class.rb +157 -0
  43. data/lib/robin/job_logger.rb +121 -0
  44. data/lib/robin/launcher.rb +658 -0
  45. data/lib/robin/leader.rb +86 -0
  46. data/lib/robin/limiter.rb +639 -0
  47. data/lib/robin/manager.rb +125 -0
  48. data/lib/robin/metrics.rb +303 -0
  49. data/lib/robin/middleware.rb +165 -0
  50. data/lib/robin/opentelemetry.rb +88 -0
  51. data/lib/robin/periodic.rb +327 -0
  52. data/lib/robin/processor.rb +317 -0
  53. data/lib/robin/profiler.rb +140 -0
  54. data/lib/robin/progress.rb +46 -0
  55. data/lib/robin/rails/app_loader.rb +23 -0
  56. data/lib/robin/rails/current_attributes.rb +88 -0
  57. data/lib/robin/rails/error_reporter.rb +31 -0
  58. data/lib/robin/rails/notifications.rb +22 -0
  59. data/lib/robin/rails/railtie.rb +87 -0
  60. data/lib/robin/rails/transactional_push.rb +49 -0
  61. data/lib/robin/rails.rb +23 -0
  62. data/lib/robin/recovery.rb +135 -0
  63. data/lib/robin/redis/connection.rb +639 -0
  64. data/lib/robin/redis/pool.rb +190 -0
  65. data/lib/robin/redis/resp.rb +77 -0
  66. data/lib/robin/reliable_push.rb +153 -0
  67. data/lib/robin/retrier.rb +284 -0
  68. data/lib/robin/scheduler.rb +102 -0
  69. data/lib/robin/scripts.rb +331 -0
  70. data/lib/robin/statsd.rb +135 -0
  71. data/lib/robin/swarm.rb +319 -0
  72. data/lib/robin/systemd.rb +47 -0
  73. data/lib/robin/testing.rb +158 -0
  74. data/lib/robin/top.rb +178 -0
  75. data/lib/robin/unique.rb +198 -0
  76. data/lib/robin/version.rb +5 -0
  77. data/lib/robin/warmup.rb +94 -0
  78. data/lib/robin/web/api.rb +580 -0
  79. data/lib/robin/web/batch_status.rb +44 -0
  80. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  81. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  82. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  83. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  84. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  86. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  87. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  88. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  89. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  90. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  91. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  92. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  93. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  94. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  95. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  96. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  97. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  98. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  99. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  100. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  101. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  102. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  103. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  104. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  105. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  106. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  107. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  108. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  109. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  110. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  111. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  112. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  113. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  114. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  115. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  116. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  117. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  118. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  119. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  120. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  121. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  122. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  123. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  124. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  125. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  126. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  127. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  128. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  129. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  130. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  131. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  132. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  133. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  134. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  135. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  136. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  137. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  138. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  139. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  140. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  141. data/lib/robin/web/dist/index.html +19 -0
  142. data/lib/robin/web/extension.rb +193 -0
  143. data/lib/robin/web/permissions.rb +40 -0
  144. data/lib/robin/web/prometheus.rb +79 -0
  145. data/lib/robin/web/router.rb +150 -0
  146. data/lib/robin/web/static.rb +110 -0
  147. data/lib/robin/web.rb +196 -0
  148. data/lib/robin.rb +258 -8
  149. data/sig/manifest.yaml +9 -0
  150. data/sig/robin/api.rbs +442 -0
  151. data/sig/robin/batch.rbs +292 -0
  152. data/sig/robin/cancellation.rbs +25 -0
  153. data/sig/robin/cli.rbs +102 -0
  154. data/sig/robin/client.rbs +118 -0
  155. data/sig/robin/config.rbs +341 -0
  156. data/sig/robin/cron.rbs +149 -0
  157. data/sig/robin/debounce.rbs +36 -0
  158. data/sig/robin/deploy.rbs +24 -0
  159. data/sig/robin/embedded.rbs +25 -0
  160. data/sig/robin/encryption.rbs +49 -0
  161. data/sig/robin/events.rbs +10 -0
  162. data/sig/robin/expiry.rbs +17 -0
  163. data/sig/robin/fetcher.rbs +134 -0
  164. data/sig/robin/gvl.rbs +39 -0
  165. data/sig/robin/health.rbs +30 -0
  166. data/sig/robin/iterable_job.rbs +99 -0
  167. data/sig/robin/job.rbs +173 -0
  168. data/sig/robin/job_class.rbs +55 -0
  169. data/sig/robin/job_logger.rbs +56 -0
  170. data/sig/robin/launcher.rbs +191 -0
  171. data/sig/robin/leader.rbs +43 -0
  172. data/sig/robin/limiter.rbs +317 -0
  173. data/sig/robin/manager.rbs +54 -0
  174. data/sig/robin/metrics.rbs +122 -0
  175. data/sig/robin/middleware.rbs +106 -0
  176. data/sig/robin/opentelemetry.rbs +51 -0
  177. data/sig/robin/periodic.rbs +81 -0
  178. data/sig/robin/processor.rbs +107 -0
  179. data/sig/robin/profiler.rbs +60 -0
  180. data/sig/robin/progress.rbs +22 -0
  181. data/sig/robin/rails/app_loader.rbs +10 -0
  182. data/sig/robin/rails/current_attributes.rbs +44 -0
  183. data/sig/robin/rails/error_reporter.rbs +21 -0
  184. data/sig/robin/rails/notifications.rbs +9 -0
  185. data/sig/robin/rails/transactional_push.rbs +16 -0
  186. data/sig/robin/rails.rbs +8 -0
  187. data/sig/robin/recovery.rbs +29 -0
  188. data/sig/robin/redis/connection.rbs +224 -0
  189. data/sig/robin/redis/pool.rbs +89 -0
  190. data/sig/robin/redis/resp.rbs +29 -0
  191. data/sig/robin/reliable_push.rbs +77 -0
  192. data/sig/robin/retrier.rbs +72 -0
  193. data/sig/robin/scheduler.rbs +40 -0
  194. data/sig/robin/scripts.rbs +36 -0
  195. data/sig/robin/statsd.rbs +55 -0
  196. data/sig/robin/swarm.rbs +133 -0
  197. data/sig/robin/systemd.rbs +13 -0
  198. data/sig/robin/testing.rbs +80 -0
  199. data/sig/robin/top.rbs +79 -0
  200. data/sig/robin/unique.rbs +72 -0
  201. data/sig/robin/version.rbs +3 -0
  202. data/sig/robin/warmup.rbs +44 -0
  203. data/sig/robin/web/api.rbs +196 -0
  204. data/sig/robin/web/batch_status.rbs +25 -0
  205. data/sig/robin/web/extension.rbs +106 -0
  206. data/sig/robin/web/permissions.rbs +14 -0
  207. data/sig/robin/web/prometheus.rbs +33 -0
  208. data/sig/robin/web/router.rbs +53 -0
  209. data/sig/robin/web/static.rbs +42 -0
  210. data/sig/robin/web.rbs +140 -0
  211. data/sig/robin.rbs +116 -0
  212. data/sig/robin_resp.rbs +55 -0
  213. metadata +274 -44
  214. data/bin/robin +0 -5
  215. data/lib/robin/actions/authenticate.rb +0 -38
  216. data/lib/robin/actions/empty_tweet.rb +0 -7
  217. data/lib/robin/actions/followers.rb +0 -22
  218. data/lib/robin/actions/help.rb +0 -16
  219. data/lib/robin/actions/home_timeline.rb +0 -9
  220. data/lib/robin/actions/new_tweet.rb +0 -11
  221. data/lib/robin/actions/unknown_flag.rb +0 -11
  222. data/lib/robin/actions/user_timeline.rb +0 -19
  223. data/lib/robin/args_parser.rb +0 -70
  224. data/lib/robin/pin_authenticator.rb +0 -31
  225. data/lib/robin/user.rb +0 -36
@@ -0,0 +1,327 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest/sha1"
4
+
5
+ module Robin
6
+ # Periodic (cron) jobs.
7
+ #
8
+ # Robin.configure_server do |config|
9
+ # config.periodic do |cron|
10
+ # cron.register "*/5 * * * *", "CleanupJob"
11
+ # cron.register "0 3 * * *", "NightlyReport", queue: "low", args: ["daily"],
12
+ # tz: "America/Chicago", description: "Email the nightly report"
13
+ # end
14
+ # end
15
+ #
16
+ # Every server knows the schedule, but only the cluster leader enqueues.
17
+ # Each tick is claimed with an atomic SET NX, so a tick runs exactly once
18
+ # even while leadership changes hands; after a failover, the most recent
19
+ # missed tick (only) is caught up.
20
+ #
21
+ # Storage:
22
+ # robin:periodic hash id => definition JSON (the registry)
23
+ # robin:periodic:armed hash id => ms when first registered
24
+ # robin:periodic:paused set paused ids
25
+ # robin:periodic:<id>:h list recent runs, newest first
26
+ # robin:periodic:<id>:t:<ms> claim key per tick
27
+ module Periodic
28
+ HISTORY = 100
29
+ MAX_SLEEP = 5
30
+ CLAIM_TTL = 7 * 86_400
31
+
32
+ # Claims the ticks due now, of entries neither paused nor registered
33
+ # after their tick, each with an atomic SET NX. Returns, for each, 1
34
+ # (won), 0 (lost) or -1 (skipped).
35
+ #
36
+ # KEYS: armed hash, paused set
37
+ # ARGV: claim key prefix ("<prefix>periodic:"), then per tick its
38
+ # entry's id, due time (ms) and claim TTL (seconds)
39
+ CLAIM = Redis::Script.new(<<~LUA)
40
+ local results = {}
41
+ for i = 2, #ARGV, 3 do
42
+ local id, due = ARGV[i], tonumber(ARGV[i + 1])
43
+ local result = -1
44
+ if redis.call('SISMEMBER', KEYS[2], id) == 0 and due >= tonumber(redis.call('HGET', KEYS[1], id) or '0') then
45
+ result = redis.call('SET', ARGV[1] .. id .. ':t:' .. ARGV[i + 1], '1', 'NX', 'EX', ARGV[i + 2]) and 1 or 0
46
+ end
47
+ results[#results + 1] = result
48
+ end
49
+ return results
50
+ LUA
51
+
52
+ # Raised by Periodic.verify! with every problem it found.
53
+ class Invalid < Robin::Error; end
54
+
55
+ # Collects registrations from config.periodic blocks.
56
+ class Registry
57
+ attr_reader :entries
58
+
59
+ # With a +problems+ array, a bad registration is noted there instead
60
+ # of raising, so every one can be reported.
61
+ def initialize(problems = nil)
62
+ @entries = {}
63
+ @problems = problems
64
+ end
65
+
66
+ # Returns the entry's id: a digest of its schedule, class, queue,
67
+ # arguments and zone, so it stays the same across deploys.
68
+ def register(cron, klass, queue: nil, args: [], tz: nil, description: nil, **options)
69
+ # @type ivar @problems: Array[String]
70
+ begin
71
+ Cron.new(cron, tz: tz) # validate early
72
+ rescue Cron::ParseError => e
73
+ raise unless @problems
74
+
75
+ @problems << "#{klass}: #{e}"
76
+ return
77
+ end
78
+ klass = klass.to_s
79
+ queue = queue&.to_s
80
+ id = Digest::SHA1.hexdigest(Robin.dump_json([cron, klass, queue, args, tz]))[0, 12] #: String
81
+ @entries[id] = {"cron" => cron, "class" => klass, "queue" => queue, "args" => args, "tz" => tz,
82
+ "description" => description, "options" => options.transform_keys(&:to_s)}
83
+ id
84
+ end
85
+ end
86
+
87
+ class << self
88
+ # Checks periodic registrations, for an app's test suite. Pass the
89
+ # block given to config.periodic, or a config to check all of its
90
+ # blocks (those registered by configure_server only run in servers):
91
+ #
92
+ # SCHEDULE = ->(cron) { cron.register "0 3 * * *", "NightlyReport" }
93
+ # config.periodic(&SCHEDULE) # the initializer
94
+ # Robin::Periodic.verify!(&SCHEDULE) # a test
95
+ #
96
+ # Raises Invalid listing every bad cron expression or time zone,
97
+ # class that doesn't exist or isn't a job, and arguments that JSON
98
+ # would change. Returns the entries otherwise.
99
+ def verify!(config = Robin.config, &block)
100
+ problems = [] #: Array[String]
101
+ registry = Registry.new(problems)
102
+ (block ? [block] : config.periodic_blocks).each { |b| b.call(registry) }
103
+ registry.entries.each_value { |entry| problems.concat(entry_problems(entry)) }
104
+ raise Invalid, "periodic jobs have problems:\n#{problems.map { |p| " #{p}" }.join("\n")}" unless problems.empty?
105
+
106
+ registry.entries
107
+ end
108
+
109
+ # Called by Launcher#run when config.periodic blocks exist.
110
+ def attach(launcher)
111
+ config = launcher.config
112
+ registry = Registry.new
113
+ config.periodic_blocks.each { |block| block.call(registry) }
114
+ entries = registry.entries
115
+ return if entries.empty?
116
+
117
+ write_registry(config, entries)
118
+ crons = entries.transform_values { |e| Cron.new(e.fetch("cron"), tz: e.fetch("tz")) }
119
+ Thread.new do
120
+ Thread.current.name = "robin.periodic"
121
+ run_loop(launcher, entries, crons)
122
+ end
123
+ end
124
+
125
+ # Every periodic job, in two round trips. With +history+ (a count),
126
+ # each also has its job options and up to that many of its latest
127
+ # runs, newest first (see #history).
128
+ def list(config: Robin.config, history: nil)
129
+ defs, armed, paused = config.redis do |c|
130
+ c.pipelined do |p|
131
+ p.call("HGETALL", config.key("periodic"))
132
+ p.call("HGETALL", config.key("periodic", "armed"))
133
+ p.call("SMEMBERS", config.key("periodic", "paused"))
134
+ end
135
+ end #: Array[untyped]
136
+ defs = defs.each_slice(2).to_h
137
+ armed = armed.each_slice(2).to_h
138
+ ids = defs.keys.sort
139
+ runs = config.redis do |c|
140
+ c.pipelined do |p|
141
+ ids.each do |id|
142
+ key = config.key("periodic", id, "h")
143
+ history ? p.call("LRANGE", key, 0, history - 1) : p.call("LINDEX", key, 0)
144
+ end
145
+ end
146
+ end
147
+ ids.zip(runs).map do |id, reply|
148
+ d = Robin.load_json(defs.fetch(id))
149
+ # The latest runs, or the latest run (or nil).
150
+ recent = Array(reply).map { |raw| Robin.load_json(raw) }
151
+ last = recent.first
152
+ cron = parse(d)
153
+ next_time = cron&.next_time
154
+ {"id" => id, "cron" => d.fetch("cron"), "class" => d.fetch("class"), "queue" => d["queue"] || "default",
155
+ "args" => d.fetch("args"), "tz" => d["tz"], "description" => d["description"] || cron&.description || d.fetch("cron"),
156
+ "next_at" => next_time && Integer(next_time) * 1000, "last_at" => last&.fetch("at"), "last_jid" => last&.fetch("jid"),
157
+ "paused" => paused.include?(id), "armed_at" => armed[id]&.to_i} #: listing
158
+ .tap { |job| job.merge!({"options" => d.fetch("options"), "history" => recent}) if history }
159
+ end
160
+ end
161
+
162
+ def history(id, config: Robin.config, limit: 25)
163
+ raw = config.redis { |c| c.call("LRANGE", config.key("periodic", id, "h"), 0, limit - 1) }
164
+ raw.map { |r| Robin.load_json(r) }
165
+ end
166
+
167
+ # Enqueues the job right away (outside its schedule). Returns the jid.
168
+ def enqueue_now(id, config: Robin.config)
169
+ raw = config.redis { |c| c.call("HGET", config.key("periodic"), id) }
170
+ raise ArgumentError, "no periodic job #{id.inspect}" unless raw
171
+
172
+ jid = push_job(config, id, Robin.load_json(raw))
173
+ record(config, [[id, Robin.now_ms, jid]], manual: true)
174
+ jid
175
+ end
176
+
177
+ def pause!(id, config: Robin.config)
178
+ config.redis { |c| c.call("SADD", config.key("periodic", "paused"), id) }
179
+ true
180
+ end
181
+
182
+ def unpause!(id, config: Robin.config)
183
+ config.redis { |c| c.call("SREM", config.key("periodic", "paused"), id) }
184
+ true
185
+ end
186
+
187
+ # Checks every entry and enqueues the ones whose latest tick hasn't
188
+ # been claimed yet. Safe to call from several processes at once.
189
+ #
190
+ # +claimed+ remembers, per entry, the latest tick this process tried
191
+ # to claim (won or lost): that tick's claim key outlives the time it
192
+ # stays the latest, so trying again could only fail. Passing the same
193
+ # hash on every call spares those round trips; the claim in Redis
194
+ # still decides who enqueues.
195
+ def tick(config, entries, crons, now = Time.now, claimed = {})
196
+ # The ticks due, minus those tried already.
197
+ dues = entries.each_key.filter_map do |id|
198
+ cron = crons.fetch(id)
199
+ due = cron.match?(now) ? Integer(now) : cron.previous_time(now)&.to_i
200
+ [id, due * 1000] if due
201
+ end - claimed.to_a #: Array[tick]
202
+ return [] if dues.empty?
203
+
204
+ won = claim(config, dues, crons, now, claimed)
205
+ pushed = Client.together { won.map { |id, due_ms| [id, due_ms, push_job(config, id, entries.fetch(id))] } } #: Array[[String, Integer, String?]]
206
+ record(config, pushed)
207
+ pushed.map(&:last)
208
+ end
209
+
210
+ # Writes +entries+ (Registry#entries) to Redis as the periodic jobs,
211
+ # arming the new ones from now: what #attach does as a server starts.
212
+ def write_registry(config, entries)
213
+ now = Robin.now_ms
214
+ config.redis do |c|
215
+ c.multi do |tx|
216
+ tx.call("DEL", config.key("periodic"))
217
+ tx.call("HSET", config.key("periodic"), *entries.flat_map { |id, e| [id, Robin.dump_json(e)] })
218
+ entries.each_key { |id| tx.call("HSETNX", config.key("periodic", "armed"), id, now) }
219
+ end
220
+ end
221
+ end
222
+
223
+ private
224
+
225
+ # Claims the ticks +dues+ ([id, due (ms)] each) of entries not paused
226
+ # or registered after them, in one round trip; returns the ones won.
227
+ # A claim outlives the tick's time as the entry's latest (until its
228
+ # next tick) by CLAIM_TTL, so a tick long ago is never claimed twice.
229
+ def claim(config, dues, crons, now, claimed)
230
+ args = dues.flat_map { |id, due_ms| [id, due_ms, CLAIM_TTL + Integer(crons.fetch(id).next_time(now) || now) - Integer(now)] }
231
+ results = config.redis do |c|
232
+ c.evalsha(CLAIM, [config.key("periodic", "armed"), config.key("periodic", "paused")], ["#{config.key("periodic")}:", *args])
233
+ end
234
+ dues.zip(results).filter_map do |(id, due_ms), result|
235
+ next if result.negative?
236
+
237
+ claimed[id] = due_ms
238
+ [id, due_ms] if result == 1
239
+ end
240
+ end
241
+
242
+ def run_loop(launcher, entries, crons)
243
+ config = launcher.config
244
+ claimed = {} #: Hash[String, Integer]
245
+ until launcher.stopping?
246
+ begin
247
+ tick(config, entries, crons, Time.now, claimed) if launcher.leader.leader?
248
+ rescue => e
249
+ config.handle_exception(e, {context: "periodic jobs"})
250
+ end
251
+ sleep(seconds_until_next(crons, Time.now))
252
+ end
253
+ end
254
+
255
+ # Sleeps until the next tick is due, waking at least every MAX_SLEEP
256
+ # seconds to notice leadership changes and shutdown.
257
+ def seconds_until_next(crons, now)
258
+ soonest = crons.each_value.filter_map { |c| c.next_time(now) }.min
259
+ soonest ? (soonest - now).clamp(0.05, MAX_SLEEP) : MAX_SLEEP
260
+ end
261
+
262
+ # A loaded class contributes its robin_options (queue, retry...);
263
+ # otherwise the name is pushed as is.
264
+ # mutant:disable -- `const_get` on Robin::Periodic instead of Object
265
+ # differs only for constants nested in Robin::Periodic, none of which
266
+ # are job classes.
267
+ def entry_problems(entry)
268
+ name = entry.fetch("class")
269
+ args = entry.fetch("args")
270
+ klass = begin
271
+ Object.const_get(name)
272
+ rescue NameError
273
+ nil
274
+ end
275
+ problems = [] #: Array[String]
276
+ if klass.nil?
277
+ problems << "#{name}: no such class"
278
+ elsif !(klass.instance_of?(Class) && (klass.method_defined?(:perform) || klass.respond_to?(:perform_later)))
279
+ problems << "#{name}: not a job class"
280
+ end
281
+ problems << "#{name}: args must be an Array" unless args.instance_of?(Array)
282
+ problems << "#{name}: args #{args.inspect} change in JSON" unless Robin.load_json(Robin.dump_json(args)) == args
283
+ problems
284
+ end
285
+
286
+ # mutant:disable -- `const_get` on Robin::Periodic instead of Object
287
+ # differs only for constants nested in Robin::Periodic, none of which
288
+ # are job classes.
289
+ def job_class(name)
290
+ Object.const_get(name)
291
+ rescue NameError
292
+ name
293
+ end
294
+
295
+ def push_job(config, id, entry)
296
+ klass = job_class(entry.fetch("class"))
297
+ item = entry.fetch("options").merge({"class" => klass, "args" => entry.fetch("args"), "periodic" => id})
298
+ queue = entry.fetch("queue")
299
+ item["queue"] = queue if queue
300
+ Client.new(config: config).push(item)
301
+ end
302
+
303
+ # Adds the pushes of ticks ([id, at (ms), jid] each) to their
304
+ # entries' histories, in one round trip.
305
+ def record(config, pushed, manual: false)
306
+ config.redis do |c|
307
+ c.pipelined do |p|
308
+ pushed.each do |id, at_ms, jid|
309
+ entry = {"at" => at_ms, "jid" => jid} #: Hash[String, untyped]
310
+ entry["manual"] = true if manual
311
+ key = config.key("periodic", id, "h")
312
+ p.call("LPUSH", key, Robin.dump_json(entry))
313
+ p.call("LTRIM", key, 0, HISTORY - 1)
314
+ end
315
+ end
316
+ end
317
+ end
318
+
319
+ # nil for definitions this version can't parse.
320
+ def parse(entry)
321
+ Cron.new(entry.fetch("cron"), tz: entry["tz"])
322
+ rescue Cron::ParseError
323
+ nil
324
+ end
325
+ end
326
+ end
327
+ end
@@ -0,0 +1,317 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "timeout"
4
+
5
+ module Robin
6
+ # A worker thread: fetch a job, run it through middleware, record the
7
+ # outcome, repeat.
8
+ class Processor
9
+ # Seconds between fetch attempts while fetching fails (Redis is down,
10
+ # failing over, loading, out of memory...): doubles from 0.2s up to 5s.
11
+ MIN_BACKOFF = 0.1
12
+ MAX_BACKOFF = 5
13
+
14
+ attr_reader :capsule, :thread, :job
15
+
16
+ # The in-flight key of the last finished job, until its
17
+ # acknowledgement goes out with the next fetch (see Fetcher#fetch).
18
+ attr_reader :unacked
19
+
20
+ # [queue, raw payload, started at (ms), in-flight key] of the running
21
+ # job, or nil; kept up to date by the launcher.
22
+ attr_accessor :work
23
+
24
+ # When (monotonic seconds) it last finished a job, or nil, and how
25
+ # many it has finished; kept up to date by the launcher.
26
+ attr_accessor :finished_at, :finished
27
+
28
+ def initialize(manager, capsule, fetcher, slot)
29
+ @manager = manager
30
+ @capsule = capsule
31
+ @config = capsule.config
32
+ @fetcher = fetcher
33
+ @slot = slot
34
+ @retrier = Retrier.new(@config)
35
+ @job_logger = JobLogger.new(@config)
36
+ @wrapper = @config.job_wrapper
37
+ @failures = 0
38
+ @backoff = MIN_BACKOFF
39
+ @retired = false
40
+ @finished = 0
41
+ end
42
+
43
+ # Starts the worker thread, after +delay+ seconds.
44
+ def start(delay)
45
+ @thread = Thread.new do
46
+ Thread.current.name = "robin.#{capsule.name}.#{@slot}"
47
+ Thread.current.report_on_exception = false
48
+ Thread.current[:robin_capsule] = capsule
49
+ Events.emit(@config, "worker.started", capsule: capsule.name)
50
+ # Robin::Shutdown (sent by #kill) only interrupts the job itself or
51
+ # a sleep, never fetching, acknowledging or recording a failure.
52
+ Thread.handle_interrupt({Shutdown => :never}) { run(delay) }
53
+ rescue Shutdown
54
+ nil # delivered after the loop ended; nothing left to interrupt
55
+ end
56
+ self
57
+ end
58
+
59
+ # Stop after the current job.
60
+ def terminate
61
+ @done = true
62
+ @job&.interrupt_requested = true
63
+ end
64
+
65
+ # Stop after the current job, without interrupting it: the capsule
66
+ # needs fewer workers (see Manager#resize).
67
+ def retire
68
+ @retired = true
69
+ @done = true
70
+ end
71
+
72
+ def retired? = @retired
73
+
74
+ def alive?
75
+ @thread&.alive?
76
+ end
77
+
78
+ # Hard stop: interrupt the running job.
79
+ def kill
80
+ @done = true
81
+ @thread&.raise(Shutdown) # a no-op for a thread that already finished
82
+ end
83
+
84
+ # Fetches the next job and runs it, on the calling thread, as the
85
+ # worker thread does each time round (its acknowledgement goes out
86
+ # with the next fetch, as there); what a benchmark measures. Returns
87
+ # whether there was one.
88
+ def work_once
89
+ unit = fetch
90
+ process(unit) if unit
91
+ !unit.nil?
92
+ end
93
+
94
+ private
95
+
96
+ def run(delay)
97
+ interruptible { sleep(delay) } if delay.positive?
98
+ until @done
99
+ unit = fetch
100
+ next unless unit
101
+
102
+ # Quieted while blocked waiting for work: hand the job back.
103
+ @done ? unit.requeue : process(unit)
104
+ end
105
+ rescue Shutdown
106
+ nil
107
+ rescue Exception => e # rubocop:disable Lint/RescueException
108
+ died = e
109
+ ensure
110
+ acknowledge_unacked
111
+ release_fetched if @done
112
+ died ? @manager.processor_died(self, died) : @manager.processor_stopped(self)
113
+ end
114
+
115
+ # The next job, or nil. A failed fetch is reported (the first failure
116
+ # of an outage, then every 30th) and followed by a backoff sleep, so a
117
+ # Redis outage or error state never kills or spins the worker.
118
+ def fetch
119
+ ack = @unacked
120
+ @unacked = nil
121
+ unit = @fetcher.fetch(@slot, ack: ack)
122
+ if (started = @outage_started)
123
+ @config.logger.info(format("Redis is back after %.1fs", monotonic - started))
124
+ @outage_started = nil
125
+ @failures = 0
126
+ @backoff = MIN_BACKOFF
127
+ end
128
+ unit
129
+ rescue => e
130
+ @outage_started ||= monotonic
131
+ @failures += 1
132
+ @config.handle_exception(e, {context: "fetch"}) if @failures == 1 || (@failures % 30).zero?
133
+ @backoff = [@backoff * 2, MAX_BACKOFF].min
134
+ interruptible { sleep @backoff } unless @done
135
+ nil
136
+ end
137
+
138
+ def process(unit)
139
+ payload = begin
140
+ Robin.remember_raw(Robin.load_json(unit.raw), unit.raw)
141
+ rescue JSON::ParserError => e
142
+ @config.handle_exception(e, {context: "invalid JSON job payload", queue: unit.queue, raw: unit.raw[0, 200]})
143
+ DeadSet.new(config: @config).kill(unit.raw, ack: unit)
144
+ return
145
+ end
146
+
147
+ if Cancellation.skip?(@config, payload["jid"], payload["bid"])
148
+ @config.logger.info { "skipping cancelled job #{payload["class"]} jid=#{payload.fetch("jid")}" }
149
+ Events.emit(@config, "job.skipped", job: payload, reason: "cancelled")
150
+ Batch.job_succeeded(@config, payload.fetch("bid"), payload.fetch("jid")) if payload["bid"]
151
+ complete(unit, payload)
152
+ return
153
+ end
154
+
155
+ if (paused = JobClass.paused_class(@config, payload)) && JobClass.park(@config, unit, paused)
156
+ @config.logger.info { "parked #{paused} jid=#{payload.fetch("jid")}: its class is paused" }
157
+ Events.emit(@config, "job.parked", job: payload, class: paused)
158
+ complete(unit, payload)
159
+ return
160
+ end
161
+
162
+ @manager.work_started(self, unit)
163
+ start = monotonic
164
+ ok = false
165
+ job_class = nil
166
+ begin
167
+ job_class = constantize(payload["class"])
168
+ execute(job_class, payload, unit)
169
+ ok = true
170
+ complete(unit, payload)
171
+ rescue Robin::Discard => e
172
+ @config.logger.info { "discarded #{payload.fetch("class")} jid=#{payload["jid"]}: #{Retrier.message_of(e)}" }
173
+ Events.emit(@config, "job.discarded", job: payload, error: e)
174
+ ok = true
175
+ complete(unit, payload)
176
+ rescue Exception => e # rubocop:disable Lint/RescueException
177
+ raise if e.is_a?(SystemExit) || e.is_a?(NoMemoryError)
178
+ # Interrupted by a shutdown (directly, or a job turned the interrupt
179
+ # into its own error): leave the job in flight for the manager to
180
+ # requeue.
181
+ raise Shutdown if Robin.shutdown?(e)
182
+
183
+ handle_failure(unit, payload, job_class, e)
184
+ # Errors a job may raise (and its timeout) are recorded; anything
185
+ # more severe (e.g. a signal) is recorded and then takes the worker
186
+ # down with it.
187
+ raise unless e.is_a?(StandardError) || e.is_a?(ScriptError) || e.is_a?(Job::Timeout)
188
+ ensure
189
+ elapsed_ms = ((monotonic - start) * 1000).round(1)
190
+ @manager.work_finished(self, payload, elapsed_ms, ok)
191
+ end
192
+ end
193
+
194
+ # Acknowledges a finished job: with the next fetch, which saves a
195
+ # round trip. One that was recovered from a crashed process also has
196
+ # its poison-pill counter cleared, so it's acknowledged right away, in
197
+ # the same round trip as that.
198
+ def complete(unit, payload)
199
+ return @unacked = unit.key unless payload.key?("recovered")
200
+
201
+ @config.redis do |conn|
202
+ conn.pipelined do |p|
203
+ unit.acknowledge(p)
204
+ p.call("DEL", @config.key("orphan", payload.fetch("jid")))
205
+ end
206
+ end
207
+ end
208
+
209
+ # A stopping (or dying) worker won't fetch again, so its last finished
210
+ # job is acknowledged on its own. If that fails, the job stays in
211
+ # flight and is put back later (run again: at least once).
212
+ def acknowledge_unacked
213
+ unacked = @unacked or return
214
+
215
+ @config.redis { |conn| @fetcher.acknowledge(conn, [unacked]) }
216
+ @unacked = nil
217
+ rescue => e
218
+ @config.handle_exception(e, {context: "acknowledging a finished job"})
219
+ end
220
+
221
+ # A stopping worker hands back the jobs its capsule fetched for other
222
+ # workers (they're stopping too), so they run elsewhere instead of
223
+ # waiting for this process to stop. A job fetched afterwards by a
224
+ # worker still finishing up is handed back when that one stops.
225
+ def release_fetched
226
+ @fetcher.release
227
+ rescue => e
228
+ @config.handle_exception(e, {context: "handing back fetched jobs"})
229
+ end
230
+
231
+ def execute(job_class, payload, unit)
232
+ # @type ivar @batch_failure: Batch::HeldFailure
233
+ job = job_class.new
234
+ job.jid = payload["jid"] if job.respond_to?(:jid=)
235
+ job.payload = payload if job.respond_to?(:payload=)
236
+ @job = job
237
+ run_job = lambda do
238
+ @job_logger.call(payload) do
239
+ @config.server_middleware.invoke(job, payload, unit.queue) do
240
+ perform(job, payload)
241
+ end
242
+ end
243
+ end
244
+ return wrapped(payload, run_job) unless payload.key?("bid")
245
+
246
+ # A batch job's failure is recorded with its retry (see
247
+ # #handle_failure), or else on the way out, as its middleware would.
248
+ @batch_failure = Batch::HeldFailure.new(payload)
249
+ Batch.holding_failure(@batch_failure) { wrapped(payload, run_job) }
250
+ record_batch_failure
251
+ rescue Exception => e # rubocop:disable Lint/RescueException
252
+ record_batch_failure unless retried?(e)
253
+ raise
254
+ ensure
255
+ @job = nil
256
+ end
257
+
258
+ def wrapped(payload, run_job) = (wrapper = @wrapper) ? wrapper.call(payload, &run_job) : run_job.call
259
+
260
+ # Does #process hand +exception+ to the retrier?
261
+ def retried?(exception)
262
+ !(exception.is_a?(Discard) || exception.is_a?(SystemExit) || exception.is_a?(NoMemoryError) || Robin.shutdown?(exception))
263
+ end
264
+
265
+ # The batch failure the retrier didn't take: the job's error was
266
+ # swallowed, or turned into a discard or a shutdown, by an outer
267
+ # middleware.
268
+ def record_batch_failure
269
+ held = take_batch_failure
270
+ Batch.job_failed(@config, held.payload.fetch("bid"), held.payload.fetch("jid"), held.info) if held
271
+ end
272
+
273
+ # The held batch failure, if the job's batch middleware recorded one.
274
+ def take_batch_failure
275
+ held = @batch_failure
276
+ @batch_failure = nil
277
+ held if held&.info
278
+ end
279
+
280
+ def perform(job, payload)
281
+ args = payload["args"]
282
+ interruptible do
283
+ if (limit = payload["timeout"])
284
+ Timeout.timeout(limit.to_f, Job::Timeout) { job.perform(*args) }
285
+ else
286
+ job.perform(*args)
287
+ end
288
+ end
289
+ end
290
+
291
+ # Lets a pending or later Robin::Shutdown interrupt the block.
292
+ def interruptible(&)
293
+ Thread.handle_interrupt({Shutdown => :immediate}, &)
294
+ end
295
+
296
+ def handle_failure(unit, payload, job_class, exception)
297
+ batch_failure = take_batch_failure&.info
298
+ # In the job's logging context, like the job's own log lines.
299
+ Context.with(JobLogger.context_for(payload)) do
300
+ @config.handle_exception(exception, {context: "job raised", job: payload, queue: unit.queue})
301
+ end
302
+ @retrier.handle(unit, payload, job_class || Object, exception, batch_failure: batch_failure)
303
+ rescue => e
304
+ # Couldn't record the failure (e.g. Redis down). The job stays in
305
+ # flight and will be put back later.
306
+ @config.handle_exception(e, {context: "recording job failure", job: payload})
307
+ end
308
+
309
+ def constantize(name)
310
+ Object.const_get(name)
311
+ end
312
+
313
+ def monotonic
314
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
315
+ end
316
+ end
317
+ end