robin 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/robin_resp/extconf.rb +29 -0
  13. data/ext/robin_resp/robin_resp.c +645 -0
  14. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  15. data/lib/generators/robin/install/install_generator.rb +39 -0
  16. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  17. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  18. data/lib/generators/robin/job/job_generator.rb +78 -0
  19. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  20. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  21. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  22. data/lib/puma/plugin/robin.rb +131 -0
  23. data/lib/robin/active_job.rb +144 -0
  24. data/lib/robin/api.rb +777 -0
  25. data/lib/robin/batch.rb +700 -0
  26. data/lib/robin/cancellation.rb +62 -0
  27. data/lib/robin/cli.rb +325 -0
  28. data/lib/robin/client.rb +355 -21
  29. data/lib/robin/config.rb +454 -4
  30. data/lib/robin/cron.rb +325 -0
  31. data/lib/robin/debounce.rb +108 -0
  32. data/lib/robin/deploy.rb +65 -0
  33. data/lib/robin/embedded.rb +53 -0
  34. data/lib/robin/encryption.rb +147 -0
  35. data/lib/robin/events.rb +60 -0
  36. data/lib/robin/expiry.rb +41 -0
  37. data/lib/robin/fetcher.rb +441 -0
  38. data/lib/robin/gvl.rb +62 -0
  39. data/lib/robin/health.rb +76 -0
  40. data/lib/robin/iterable_job.rb +294 -0
  41. data/lib/robin/job.rb +258 -0
  42. data/lib/robin/job_class.rb +157 -0
  43. data/lib/robin/job_logger.rb +121 -0
  44. data/lib/robin/launcher.rb +658 -0
  45. data/lib/robin/leader.rb +86 -0
  46. data/lib/robin/limiter.rb +639 -0
  47. data/lib/robin/manager.rb +125 -0
  48. data/lib/robin/metrics.rb +303 -0
  49. data/lib/robin/middleware.rb +165 -0
  50. data/lib/robin/opentelemetry.rb +88 -0
  51. data/lib/robin/periodic.rb +327 -0
  52. data/lib/robin/processor.rb +317 -0
  53. data/lib/robin/profiler.rb +140 -0
  54. data/lib/robin/progress.rb +46 -0
  55. data/lib/robin/rails/app_loader.rb +23 -0
  56. data/lib/robin/rails/current_attributes.rb +88 -0
  57. data/lib/robin/rails/error_reporter.rb +31 -0
  58. data/lib/robin/rails/notifications.rb +22 -0
  59. data/lib/robin/rails/railtie.rb +87 -0
  60. data/lib/robin/rails/transactional_push.rb +49 -0
  61. data/lib/robin/rails.rb +23 -0
  62. data/lib/robin/recovery.rb +135 -0
  63. data/lib/robin/redis/connection.rb +639 -0
  64. data/lib/robin/redis/pool.rb +190 -0
  65. data/lib/robin/redis/resp.rb +77 -0
  66. data/lib/robin/reliable_push.rb +153 -0
  67. data/lib/robin/retrier.rb +284 -0
  68. data/lib/robin/scheduler.rb +102 -0
  69. data/lib/robin/scripts.rb +331 -0
  70. data/lib/robin/statsd.rb +135 -0
  71. data/lib/robin/swarm.rb +319 -0
  72. data/lib/robin/systemd.rb +47 -0
  73. data/lib/robin/testing.rb +158 -0
  74. data/lib/robin/top.rb +178 -0
  75. data/lib/robin/unique.rb +198 -0
  76. data/lib/robin/version.rb +5 -0
  77. data/lib/robin/warmup.rb +94 -0
  78. data/lib/robin/web/api.rb +580 -0
  79. data/lib/robin/web/batch_status.rb +44 -0
  80. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  81. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  82. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  83. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  84. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  86. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  87. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  88. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  89. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  90. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  91. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  92. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  93. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  94. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  95. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  96. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  97. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  98. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  99. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  100. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  101. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  102. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  103. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  104. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  105. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  106. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  107. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  108. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  109. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  110. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  111. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  112. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  113. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  114. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  115. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  116. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  117. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  118. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  119. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  120. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  121. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  122. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  123. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  124. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  125. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  126. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  127. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  128. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  129. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  130. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  131. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  132. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  133. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  134. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  135. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  136. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  137. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  138. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  139. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  140. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  141. data/lib/robin/web/dist/index.html +19 -0
  142. data/lib/robin/web/extension.rb +193 -0
  143. data/lib/robin/web/permissions.rb +40 -0
  144. data/lib/robin/web/prometheus.rb +79 -0
  145. data/lib/robin/web/router.rb +150 -0
  146. data/lib/robin/web/static.rb +110 -0
  147. data/lib/robin/web.rb +196 -0
  148. data/lib/robin.rb +258 -8
  149. data/sig/manifest.yaml +9 -0
  150. data/sig/robin/api.rbs +442 -0
  151. data/sig/robin/batch.rbs +292 -0
  152. data/sig/robin/cancellation.rbs +25 -0
  153. data/sig/robin/cli.rbs +102 -0
  154. data/sig/robin/client.rbs +118 -0
  155. data/sig/robin/config.rbs +341 -0
  156. data/sig/robin/cron.rbs +149 -0
  157. data/sig/robin/debounce.rbs +36 -0
  158. data/sig/robin/deploy.rbs +24 -0
  159. data/sig/robin/embedded.rbs +25 -0
  160. data/sig/robin/encryption.rbs +49 -0
  161. data/sig/robin/events.rbs +10 -0
  162. data/sig/robin/expiry.rbs +17 -0
  163. data/sig/robin/fetcher.rbs +134 -0
  164. data/sig/robin/gvl.rbs +39 -0
  165. data/sig/robin/health.rbs +30 -0
  166. data/sig/robin/iterable_job.rbs +99 -0
  167. data/sig/robin/job.rbs +173 -0
  168. data/sig/robin/job_class.rbs +55 -0
  169. data/sig/robin/job_logger.rbs +56 -0
  170. data/sig/robin/launcher.rbs +191 -0
  171. data/sig/robin/leader.rbs +43 -0
  172. data/sig/robin/limiter.rbs +317 -0
  173. data/sig/robin/manager.rbs +54 -0
  174. data/sig/robin/metrics.rbs +122 -0
  175. data/sig/robin/middleware.rbs +106 -0
  176. data/sig/robin/opentelemetry.rbs +51 -0
  177. data/sig/robin/periodic.rbs +81 -0
  178. data/sig/robin/processor.rbs +107 -0
  179. data/sig/robin/profiler.rbs +60 -0
  180. data/sig/robin/progress.rbs +22 -0
  181. data/sig/robin/rails/app_loader.rbs +10 -0
  182. data/sig/robin/rails/current_attributes.rbs +44 -0
  183. data/sig/robin/rails/error_reporter.rbs +21 -0
  184. data/sig/robin/rails/notifications.rbs +9 -0
  185. data/sig/robin/rails/transactional_push.rbs +16 -0
  186. data/sig/robin/rails.rbs +8 -0
  187. data/sig/robin/recovery.rbs +29 -0
  188. data/sig/robin/redis/connection.rbs +224 -0
  189. data/sig/robin/redis/pool.rbs +89 -0
  190. data/sig/robin/redis/resp.rbs +29 -0
  191. data/sig/robin/reliable_push.rbs +77 -0
  192. data/sig/robin/retrier.rbs +72 -0
  193. data/sig/robin/scheduler.rbs +40 -0
  194. data/sig/robin/scripts.rbs +36 -0
  195. data/sig/robin/statsd.rbs +55 -0
  196. data/sig/robin/swarm.rbs +133 -0
  197. data/sig/robin/systemd.rbs +13 -0
  198. data/sig/robin/testing.rbs +80 -0
  199. data/sig/robin/top.rbs +79 -0
  200. data/sig/robin/unique.rbs +72 -0
  201. data/sig/robin/version.rbs +3 -0
  202. data/sig/robin/warmup.rbs +44 -0
  203. data/sig/robin/web/api.rbs +196 -0
  204. data/sig/robin/web/batch_status.rbs +25 -0
  205. data/sig/robin/web/extension.rbs +106 -0
  206. data/sig/robin/web/permissions.rbs +14 -0
  207. data/sig/robin/web/prometheus.rbs +33 -0
  208. data/sig/robin/web/router.rbs +53 -0
  209. data/sig/robin/web/static.rbs +42 -0
  210. data/sig/robin/web.rbs +140 -0
  211. data/sig/robin.rbs +116 -0
  212. data/sig/robin_resp.rbs +55 -0
  213. metadata +274 -44
  214. data/bin/robin +0 -5
  215. data/lib/robin/actions/authenticate.rb +0 -38
  216. data/lib/robin/actions/empty_tweet.rb +0 -7
  217. data/lib/robin/actions/followers.rb +0 -22
  218. data/lib/robin/actions/help.rb +0 -16
  219. data/lib/robin/actions/home_timeline.rb +0 -9
  220. data/lib/robin/actions/new_tweet.rb +0 -11
  221. data/lib/robin/actions/unknown_flag.rb +0 -11
  222. data/lib/robin/actions/user_timeline.rb +0 -19
  223. data/lib/robin/args_parser.rb +0 -70
  224. data/lib/robin/pin_authenticator.rb +0 -31
  225. data/lib/robin/user.rb +0 -36
@@ -0,0 +1,580 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ class Web
5
+ # JSON endpoint handlers. One instance per request.
6
+ class API
7
+ SETS = JobLocator::SETS
8
+ SET_ACTIONS = {"retry" => :retry, "delete" => :delete, "kill" => :kill}.freeze #: Hash[String, :retry | :delete | :kill]
9
+ SIGNALS = {"quiet" => :quiet!, "stop" => :stop!, "dump" => :dump_threads}.freeze #: Hash[String, :quiet! | :stop! | :dump_threads]
10
+ MAX_CONCURRENCY = 1000
11
+
12
+ # Every page is built in; kept in the session for older UIs.
13
+ FEATURES = {"batches" => true, "limiters" => true, "cron" => true}.freeze
14
+
15
+ # Integer query parameters: name => [default, minimum, maximum].
16
+ INT_PARAMS = {
17
+ "days" => [30, 1, 180], "minutes" => [60, 5, 480], "hours" => [24 * 7, 1, 24 * 90],
18
+ "points" => [60, 5, 24 * 90], "page" => [1, 1, 1_000_000], "per_page" => [25, 1, 200],
19
+ "limit" => [100, 1, 1000]
20
+ }.freeze #: Hash[String, [Integer, Integer, Integer]]
21
+
22
+ INFO_FIELDS = %w[redis_version redis_mode os uptime_in_seconds connected_clients used_memory_human
23
+ used_memory_peak_human maxmemory_human maxmemory_policy mem_fragmentation_ratio total_commands_processed
24
+ instantaneous_ops_per_sec keyspace_hits keyspace_misses role tcp_port].freeze
25
+
26
+ # Searches return at most this many matches.
27
+ SEARCH_LIMIT = 1000
28
+
29
+ # Links job_info rows may have: http(s) URLs and paths on this host.
30
+ INFO_LINK = %r{\A(?:https?://|/(?!/))}i
31
+
32
+ # Fields describing a failure, dropped when a job is edited and re-run.
33
+ FAILURE_FIELDS = %w[at enqueued_at error_class error_message error_backtrace failed_at retried_at retry_count].freeze
34
+
35
+ def self.json(status, body)
36
+ [status, {"content-type" => "application/json; charset=utf-8", "cache-control" => "no-store"}, [Robin.dump_json(body)]]
37
+ end
38
+
39
+ def self.parse_info(raw)
40
+ raw.each_line.each_with_object({}) do |line, info|
41
+ key, colon, value = line.chomp.partition(":")
42
+ info[key] = value unless colon.empty?
43
+ end
44
+ end
45
+
46
+ attr_reader :config
47
+
48
+ def initialize(env, request, cluster, params)
49
+ @env = env
50
+ @request = request
51
+ @cluster = cluster
52
+ @config = Web.clusters.fetch(cluster)
53
+ @params = params
54
+ end
55
+
56
+ def authorize!(action, resource)
57
+ return if Permissions.allowed?(@env, action, resource)
58
+
59
+ raise Router::HttpError.new(403, "You don't have permission to #{Permissions::ACTIONS.fetch(action).downcase}")
60
+ end
61
+
62
+ # -- session & overview ---------------------------------------------
63
+
64
+ def session
65
+ ok(
66
+ {"title" => Web.title, "brand_color" => Web.brand_color, "version" => VERSION,
67
+ "read_only" => !!Web.read_only, "permissions" => Permissions.for(@env),
68
+ "clusters" => Web.clusters.keys, "cluster" => @cluster,
69
+ "features" => FEATURES, "stream_interval" => Web.stream_interval,
70
+ "live_updates" => Web.live_updates, "extensions" => Web.extensions.values.map(&:to_h)}
71
+ )
72
+ end
73
+
74
+ def dashboard
75
+ stats = Stats.new(config: config)
76
+ ok(
77
+ {"stats" => stats.to_h, "queues" => queue_rows(stats.queues),
78
+ "history" => stats.history(days: int_param("days")),
79
+ "realtime" => Metrics::Query.new(config).recent(minutes: int_param("minutes")).fetch("series"),
80
+ "leader" => Leader.current(config)}
81
+ )
82
+ end
83
+
84
+ def stats
85
+ ok(live_stats)
86
+ end
87
+
88
+ # Server-sent events: a "stats" event now and every
89
+ # Web.stream_interval seconds after, for Web.stream_lifetime seconds
90
+ # (the browser then reconnects).
91
+ def stream
92
+ interval = Web.stream_interval
93
+ repeats = (Web.stream_lifetime / interval).floor
94
+ body = Enumerator.new do |out|
95
+ out << "retry: 3000\n\n" << stats_event
96
+ repeats.times do
97
+ sleep interval
98
+ out << stats_event
99
+ end
100
+ end
101
+ [200, {"content-type" => "text/event-stream", "cache-control" => "no-cache", "x-accel-buffering" => "no"}, body]
102
+ end
103
+
104
+ def live_stats
105
+ stats = Stats.new(config: config)
106
+ {"stats" => stats.to_h, "queues" => stats.queues, "at" => Robin.now_ms}
107
+ end
108
+
109
+ def info
110
+ raw, url = config.redis { |c| [c.call("INFO"), c.id] }
111
+ ok(
112
+ {"redis" => API.parse_info(raw).slice(*INFO_FIELDS), "redis_url" => url,
113
+ "robin" => {"version" => VERSION, "ruby" => RUBY_DESCRIPTION, "key_prefix" => config.key_prefix,
114
+ "max_retries" => config.max_retries, "dead_max_jobs" => config.dead_max_jobs,
115
+ "dead_timeout_days" => config.dead_timeout / 86_400, "native_resp" => !Redis::PARSER.eql?(Redis::RESP)},
116
+ "processes" => ProcessSet.new(config: config).size, "leader" => Leader.current(config)}
117
+ )
118
+ end
119
+
120
+ # -- queues ---------------------------------------------------------
121
+
122
+ def queues
123
+ ok({"queues" => queue_rows(Stats.new(config: config).queues), "paused_classes" => JobClass.list(config: config)})
124
+ end
125
+
126
+ def queue
127
+ q = named_queue
128
+ page, per = paging
129
+ info = q.to_h
130
+ ok({"queue" => info, "page" => page, "per_page" => per, "total" => info.fetch("size"),
131
+ "jobs" => q.page(page, per_page: per).map { |j| job_json(j) }})
132
+ end
133
+
134
+ def pause_queue
135
+ named_queue.pause!
136
+ ok({"paused" => true})
137
+ end
138
+
139
+ def unpause_queue
140
+ named_queue.unpause!
141
+ ok({"paused" => false})
142
+ end
143
+
144
+ def pause_class
145
+ named_class.pause!
146
+ ok({"paused" => true})
147
+ end
148
+
149
+ # With how many parked jobs went back on their queues.
150
+ def unpause_class = ok({"paused" => false, "requeued" => named_class.unpause!})
151
+
152
+ def clear_queue
153
+ named_queue.clear
154
+ ok({"cleared" => true})
155
+ end
156
+
157
+ # Deletes the given jids, walking the queue only until all are found,
158
+ # then deleting them together.
159
+ def delete_queue_jobs
160
+ wanted = Array(body["jids"]).to_set #: Set[String]
161
+ queue = named_queue
162
+ found = [] #: Array[JobRecord]
163
+ queue.each do |job|
164
+ found << job if wanted.delete?(job.jid)
165
+ break if wanted.empty?
166
+ end
167
+ ok({"deleted" => queue.delete_jobs(found)})
168
+ end
169
+
170
+ # Deletes every job of one class, wherever it is in the queue.
171
+ def delete_queue_class
172
+ klass = body["class"]
173
+ raise Router::HttpError.new(400, "A class name is required") unless klass.instance_of?(String) && !klass.empty?
174
+
175
+ ok({"deleted" => named_queue.delete_by_class(klass)})
176
+ end
177
+
178
+ # -- processes ------------------------------------------------------
179
+
180
+ def busy
181
+ work = WorkSet.new(config: config).map do |identity, tid, w|
182
+ [identity, tid, w, JobRecord.new(w.fetch("payload"), w.fetch("queue"))]
183
+ end #: Array[[String, String, Hash[String, json], JobRecord]]
184
+ progress = Progress.fetch_many(work.map { |_identity, _tid, _w, record| record.jid }, config: config)
185
+ rows = work.map do |identity, tid, w, record|
186
+ {"identity" => identity, "thread" => tid, "queue" => w.fetch("queue"), "run_at" => w.fetch("run_at"),
187
+ "job" => job_json(record), "progress" => progress[record.jid]}
188
+ end
189
+ ok({"processes" => ProcessSet.new(config: config).map(&:to_h), "work" => rows, "leader" => Leader.current(config)})
190
+ end
191
+
192
+ def signal_process
193
+ signal = signal_method
194
+ live_process.public_send(signal)
195
+ ok({"signalled" => 1})
196
+ end
197
+
198
+ # Body: {"concurrency": workers, "capsule": name (default "default")}.
199
+ def resize_process
200
+ process = live_process
201
+ count = body["concurrency"]
202
+ unless count.instance_of?(Integer) && count.between?(1, MAX_CONCURRENCY)
203
+ raise Router::HttpError.new(400, "concurrency must be a whole number from 1 to #{MAX_CONCURRENCY}")
204
+ end
205
+
206
+ capsule = body.fetch("capsule", "default")
207
+ unless process["capsules"]&.key?(capsule)
208
+ raise Router::HttpError.new(400, "Process #{process.identity} has no capsule #{capsule.inspect}")
209
+ end
210
+
211
+ process.resize!(count, capsule: capsule)
212
+ ok({"resized" => 1})
213
+ end
214
+
215
+ def signal_all_processes
216
+ signal = signal_method
217
+ ok({"signalled" => ProcessSet.new(config: config).count { |p| p.public_send(signal) }})
218
+ end
219
+
220
+ # -- scheduled / retries / dead ------------------------------------------
221
+
222
+ def set
223
+ set = job_set
224
+ page, per = paging
225
+ query = @params["q"].to_s.strip
226
+ newest_first = set.instance_of?(DeadSet)
227
+ if query.empty?
228
+ entries, total = set.page_and_size(page, per_page: per, reverse: newest_first)
229
+ else
230
+ matches = set.scan(query, limit: SEARCH_LIMIT)
231
+ matches.reverse! if newest_first
232
+ total = matches.size
233
+ entries = matches.drop((page - 1) * per).first(per)
234
+ end
235
+ ok({"set" => @params.fetch("set"), "page" => page, "per_page" => per, "total" => total, "query" => query,
236
+ "jobs" => entries.map { |e| job_json(e) }})
237
+ end
238
+
239
+ def set_entry
240
+ ok({"job" => job_json(find_entry!, full: true)})
241
+ end
242
+
243
+ def set_bulk
244
+ action = set_action!
245
+ set = job_set
246
+ ok({"affected" => apply(set, set.find_all_by_id(Array(body["ids"])), action)})
247
+ end
248
+
249
+ def set_all
250
+ action = set_action!
251
+ set = job_set
252
+ count = if action == :delete
253
+ set.size.tap { set.clear }
254
+ else
255
+ set.drain_pages { |page| apply(set, page, action) }
256
+ end
257
+ ok({"affected" => count})
258
+ end
259
+
260
+ # Replace a failed job's arguments and run it again right away.
261
+ def edit_entry
262
+ entry = find_entry!
263
+ raise Router::HttpError.new(422, "Encrypted jobs can't be edited") if entry["encrypted"]
264
+
265
+ args = body["args"]
266
+ raise Router::HttpError.new(422, "args must be a JSON array") unless args.instance_of?(Array)
267
+ raise Router::HttpError.new(409, "That job is no longer in the set") unless entry.delete
268
+
269
+ item = entry.item.except("jid", *FAILURE_FIELDS).merge({"args" => args, "edited_from" => entry.jid})
270
+ ok({"jid" => Client.new(config: config).push(item)})
271
+ end
272
+
273
+ def find_job
274
+ jid = @params.fetch("jid")
275
+ found = JobLocator.new(config: config).locate(jid).map { |r| r.merge({"job" => job_json(r.fetch("job"), full: true)}) }
276
+ ok({"jid" => jid, "results" => found, "progress" => Progress.fetch(jid, config: config),
277
+ "cancelled" => Cancellation.cancelled?(jid, config: config)})
278
+ end
279
+
280
+ def cancel_job
281
+ Cancellation.cancel!(@params.fetch("jid"), config: config)
282
+ ok({"cancelled" => true})
283
+ end
284
+
285
+ # -- metrics --------------------------------------------------------------
286
+
287
+ def metrics
288
+ minutes = int_param("minutes")
289
+ ok(Metrics::Query.new(config).recent(minutes: minutes).merge({"minutes" => minutes, "deploys" => deploys(minutes * 60)}))
290
+ end
291
+
292
+ def metrics_history
293
+ hours = int_param("hours")
294
+ ok(Metrics::Query.new(config).history(hours: hours).merge({"hours" => hours, "deploys" => deploys(hours * 3600)}))
295
+ end
296
+
297
+ def metrics_queues
298
+ granularity = (@params["granularity"] == "hour") ? :hour : :minute
299
+ points = int_param("points")
300
+ ok({"granularity" => granularity, "points" => Metrics::Query.new(config).queue_history(granularity: granularity, points: points)})
301
+ end
302
+
303
+ def metrics_job
304
+ klass = @params.fetch("klass")
305
+ minutes = int_param("minutes")
306
+ now = Time.now
307
+ stamps = Array.new(minutes) { |i| Metrics.minute_stamp(now - i * 60) }.reverse
308
+ fields = %w[s f ms].map { |kind| "#{klass}|#{kind}" }
309
+ *rows, paused, parked = config.redis do |c|
310
+ c.pipelined do |p|
311
+ stamps.each { |s| p.call("HMGET", config.key("m", s), *fields) }
312
+ p.call("SISMEMBER", config.key("classes", "paused"), klass)
313
+ p.call("LLEN", config.key("classes", "parked", klass))
314
+ end
315
+ end
316
+ series = stamps.zip(rows).map { |t, (s, f, ms)| {"t" => t, "s" => s.to_i, "f" => f.to_i, "ms" => ms.to_f} }
317
+ totals = %w[s f ms].to_h { |kind| [kind, series.sum { |point| point.fetch(kind) }] }
318
+ succeeded = totals.fetch("s") # "ms" only counts successful runs
319
+ totals["avg_ms"] = succeeded.zero? ? 0 : (totals.fetch("ms") / succeeded).round(1)
320
+ ok({"class" => klass, "minutes" => minutes, "series" => series, "totals" => totals, "deploys" => deploys(minutes * 60),
321
+ "histogram" => Metrics::Query.new(config).histogram(klass, minutes: minutes), "paused" => paused == 1, "parked" => parked})
322
+ end
323
+
324
+ # Deploys in the last +seconds+.
325
+ def deploys(seconds) = Deploy.list(from: Time.now - seconds, config: config)
326
+
327
+ # -- batches, limiters, cron ------------------------------------------------
328
+
329
+ def batches
330
+ tag = @params["tag"].to_s
331
+ ok({"available" => true, "batches" => Batch.list(config: config, limit: int_param("limit"), tag: (tag unless tag.empty?))})
332
+ end
333
+
334
+ def batch
335
+ status = Batch::Status.new(@params.fetch("bid"), config: config)
336
+ raise Router::HttpError.new(404, "No batch #{@params.fetch("bid")}") unless status.exists?
337
+
338
+ ok({"batch" => status.to_h})
339
+ end
340
+
341
+ def cancel_batch
342
+ bid = @params.fetch("bid")
343
+ ok({"cancelled" => Batch.new(bid, config: config).cancel!})
344
+ rescue Batch::NoSuchBatch
345
+ raise Router::HttpError.new(404, "No batch #{bid}")
346
+ end
347
+
348
+ def delete_batch
349
+ Batch::Status.new(@params.fetch("bid"), config: config).delete!
350
+ ok({"deleted" => true})
351
+ end
352
+
353
+ def limiters
354
+ ok({"available" => true, "limiters" => Limiter.list(config: config)})
355
+ end
356
+
357
+ def reset_limiter
358
+ Limiter.reset!(@params.fetch("name"), config: config)
359
+ ok({"reset" => true})
360
+ end
361
+
362
+ def delete_limiter
363
+ Limiter.delete!(@params.fetch("name"), config: config)
364
+ ok({"deleted" => true})
365
+ end
366
+
367
+ def cron
368
+ ok({"available" => true, "jobs" => Periodic.list(config: config), "leader" => Leader.current(config)})
369
+ end
370
+
371
+ def cron_entry
372
+ job = Periodic.list(config: config).find { |j| j.fetch("id") == @params.fetch("id") }
373
+ raise Router::HttpError.new(404, "No cron job #{@params.fetch("id")}") unless job
374
+
375
+ ok({"job" => job, "history" => Periodic.history(@params.fetch("id"), config: config, limit: 50)})
376
+ end
377
+
378
+ def cron_enqueue
379
+ ok({"jid" => Periodic.enqueue_now(@params.fetch("id"), config: config)})
380
+ rescue ArgumentError
381
+ raise Router::HttpError.new(404, "No cron job #{@params.fetch("id")}")
382
+ end
383
+
384
+ def cron_pause
385
+ Periodic.pause!(@params.fetch("id"), config: config)
386
+ ok({"paused" => true})
387
+ end
388
+
389
+ def cron_unpause
390
+ Periodic.unpause!(@params.fetch("id"), config: config)
391
+ ok({"paused" => false})
392
+ end
393
+
394
+ # -- profiles -------------------------------------------------------------
395
+
396
+ def profiles
397
+ ok({"profiles" => Profiler.list(config: config)})
398
+ end
399
+
400
+ # The profile itself, as a download for the Firefox Profiler.
401
+ def profile_data
402
+ jid = @params.fetch("jid")
403
+ data = Profiler.data(jid, config: config) or raise Router::HttpError.new(404, "That profile has expired")
404
+ [200, {"content-type" => "application/json", "content-disposition" => %(attachment; filename="robin-#{jid.delete("^a-zA-Z0-9_-")}.json"),
405
+ "cache-control" => "no-store"}, [data]]
406
+ end
407
+
408
+ def delete_profile
409
+ ok({"deleted" => Profiler.delete(@params.fetch("jid"), config: config)})
410
+ end
411
+
412
+ # -- extensions -----------------------------------------------------------
413
+
414
+ def extension_page
415
+ extension = extension!
416
+ request = extension_request(extension)
417
+ blocks = guarded(extension, "page") { extension.new.blocks(request) }
418
+ ok(extension.to_h.merge({"blocks" => blocks, "messages" => extension.messages(request.locale) || {}}))
419
+ end
420
+
421
+ def extension_data
422
+ extension = extension!
423
+ name = @params.fetch("name")
424
+ raise Router::HttpError.new(404, "#{extension.title} has no data #{name.inspect}") unless extension.data_sources.key?(name)
425
+
426
+ ok(guarded(extension, name) { extension.new.fetch(name, extension_request(extension)) })
427
+ end
428
+
429
+ def extension_action
430
+ extension = extension!
431
+ name = @params.fetch("action")
432
+ raise Router::HttpError.new(404, "#{extension.title} has no action #{name.inspect}") unless extension.actions.key?(name)
433
+
434
+ authorize!(:extensions, "extension:#{extension.slug}/#{name}")
435
+ ok({"message" => guarded(extension, name) { extension.new.run(name, extension_request(extension, body)) }})
436
+ end
437
+
438
+ private
439
+
440
+ def ok(body) = API.json(200, body)
441
+
442
+ def stats_event = "event: stats\ndata: #{Robin.dump_json(live_stats)}\n\n"
443
+
444
+ def extension!
445
+ slug = @params.fetch("slug")
446
+ Web.extensions.fetch(slug) { raise Router::HttpError.new(404, "No dashboard extension #{slug.inspect}") }
447
+ end
448
+
449
+ def extension_request(extension, body = {})
450
+ params = @params.except("slug", "action", "name", "cluster", "locale")
451
+ Extension::Request.new(config, params, @env, @params.fetch("locale", "en"), body, extension)
452
+ end
453
+
454
+ # An extension that raises is reported, and the user sees a 500.
455
+ def guarded(extension, what)
456
+ yield
457
+ rescue => e
458
+ config.handle_exception(e, {context: "dashboard extension #{extension.slug} (#{what})"})
459
+ raise Router::HttpError.new(500, "#{extension.title} failed: #{e.class}: #{e}")
460
+ end
461
+
462
+ def body
463
+ @body ||= begin
464
+ raw = @request.body&.read.to_s
465
+ parsed = raw.empty? ? {} : JSON.parse(raw) #: Hash[String, json]
466
+ raise Router::HttpError.new(400, "Request body must be a JSON object") unless parsed.instance_of?(Hash)
467
+
468
+ parsed
469
+ end
470
+ end
471
+
472
+ def int_param(name)
473
+ default, min, max = INT_PARAMS.fetch(name)
474
+ (Integer(@params[name], exception: false) || default).clamp(min, max)
475
+ end
476
+
477
+ def paging = [int_param("page"), int_param("per_page")]
478
+
479
+ def named_queue = Queue.new(@params.fetch("name"), config: config)
480
+
481
+ def named_class = JobClass.new(@params.fetch("name"), config: config)
482
+
483
+ def job_set
484
+ klass = SETS.fetch(@params.fetch("set")) { raise Router::HttpError.new(404, "Unknown set #{@params.fetch("set").inspect}") }
485
+ klass.new(config: config)
486
+ end
487
+
488
+ def find_entry!
489
+ job_set.find_by_id(@params.fetch("id")) or
490
+ raise Router::HttpError.new(404, "That job is no longer in the #{@params.fetch("set")} set")
491
+ end
492
+
493
+ def set_action!
494
+ name = body["action"]
495
+ action = SET_ACTIONS.fetch(name) { raise Router::HttpError.new(400, "Unknown action #{name.inspect}") }
496
+ authorize!(action, "set:#{@params.fetch("set")}")
497
+ action
498
+ end
499
+
500
+ def live_process
501
+ process = ProcessSet.new(config: config).find { |p| p.identity == @params.fetch("identity") }
502
+ raise Router::HttpError.new(404, "No live process #{@params.fetch("identity")}") unless process
503
+
504
+ process
505
+ end
506
+
507
+ def signal_method
508
+ SIGNALS.fetch(@params.fetch("signal")) { raise Router::HttpError.new(400, "Unknown signal #{@params.fetch("signal").inspect}") }
509
+ end
510
+
511
+ # Applies +action+ to +entries+ of +set+, in a few round trips;
512
+ # returns how many it applied to. (The dead set's jobs are dead
513
+ # already.)
514
+ def apply(set, entries, action)
515
+ case action
516
+ when :retry then set.retry_entries(entries)
517
+ when :delete then set.delete_entries(entries)
518
+ else set.instance_of?(DeadSet) ? 0 : set.kill_entries(entries)
519
+ end
520
+ end
521
+
522
+ # The queues' rows, in one round trip.
523
+ def queue_rows(sizes)
524
+ names = sizes.keys
525
+ paused, *oldest = config.redis do |c|
526
+ c.pipelined do |p|
527
+ p.call("SMEMBERS", config.key("paused"))
528
+ names.each { |n| p.call("LINDEX", config.key("queue", n), -1) }
529
+ end
530
+ end
531
+ names.zip(oldest).map do |name, raw|
532
+ {"name" => name, "size" => sizes.fetch(name), "latency" => Queue.latency_of(raw), "paused" => paused.include?(name)}
533
+ end
534
+ end
535
+
536
+ def job_json(record, full: false)
537
+ item = record.item
538
+ wrapper = record.klass unless record.display_class == record.klass
539
+ h = {
540
+ "jid" => record.jid, "class" => record.display_class, "wrapper" => wrapper,
541
+ "queue" => record.queue, "args" => record.display_args, "created_at" => item["created_at"],
542
+ "enqueued_at" => item["enqueued_at"], "tags" => record.tags, "bid" => record.bid,
543
+ "error_class" => record.error_class, "error_message" => record.error_message,
544
+ "failures" => record.failures.nonzero?, "failed_at" => item["failed_at"], "retried_at" => item["retried_at"],
545
+ "encrypted" => item.key?("encrypted")
546
+ }
547
+ if record.instance_of?(SortedEntry)
548
+ h["id"] = record.id
549
+ h["at"] = Robin.epoch_ms(record.score) # the dashboard's times are epoch ms
550
+ end
551
+ if full
552
+ h["payload"] = record.to_h.fetch("item")
553
+ h["backtrace"] = record.error_backtrace
554
+ h["info"] = job_info(item)
555
+ end
556
+ h
557
+ end
558
+
559
+ # The Robin::Web.job_info rows for a job. A failing block is reported
560
+ # and its row left out, so one bug can't break the job page.
561
+ def job_info(item)
562
+ Web.job_info_rows.filter_map do |label, block|
563
+ value = block.call(item)
564
+ info_row(label, value) unless value.nil?
565
+ rescue => e
566
+ config.handle_exception(e, {context: "Robin::Web.job_info #{label.inspect}", job: item})
567
+ nil
568
+ end
569
+ end
570
+
571
+ def info_row(label, value)
572
+ return {"label" => label, "text" => value.to_s} unless value.is_a?(Hash)
573
+
574
+ href = (value[:href] || value["href"]).to_s
575
+ text = value[:text] || value["text"] || href
576
+ {"label" => label, "text" => text.to_s, "href" => (href if INFO_LINK.match?(href))}.compact
577
+ end
578
+ end
579
+ end
580
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ class Web
5
+ # A small JSON endpoint that an app's own pages can poll for a batch's
6
+ # progress (to draw a progress bar for an import, say). Mount it apart
7
+ # from the dashboard: it doesn't ask the dashboard's authorization.
8
+ #
9
+ # # config/routes.rb
10
+ # mount Robin::Web::BatchStatus.new => "/batch_status"
11
+ #
12
+ # GET /batch_status/<bid>.json
13
+ # {"bid": "...", "total": 120, "pending": 30, "failures": 2,
14
+ # "complete": false, "success": false, "dead": false, "cancelled": false}
15
+ #
16
+ # A bid is 80 random bits, so knowing one is what lets you read it. It
17
+ # shows counts and states only, never jobs, arguments, errors or the
18
+ # description. Unknown and expired batches are 404s.
19
+ class BatchStatus
20
+ PATH = %r{\A/([\w-]{1,64})(?:\.json)?\z}
21
+ HEADERS = {"content-type" => "application/json; charset=utf-8", "cache-control" => "no-store"}.freeze
22
+
23
+ # +config+: the Robin config whose batches it shows (Robin.config
24
+ # when nil).
25
+ def initialize(config: nil) = @config = config
26
+
27
+ def call(env)
28
+ unless %w[GET HEAD].include?(env.fetch("REQUEST_METHOD"))
29
+ return reply(405, {"error" => "Only GET is allowed"}, {"allow" => "GET, HEAD"})
30
+ end
31
+
32
+ bid = env.fetch("PATH_INFO")[PATH, 1] or return not_found
33
+ status = Batch::Status.new(bid, config: @config || Robin.config)
34
+ status.exists? ? reply(200, status.progress) : not_found
35
+ end
36
+
37
+ private
38
+
39
+ def not_found = reply(404, {"error" => "No such batch"})
40
+
41
+ def reply(code, body, headers = {}) = [code, HEADERS.merge(headers), [Robin.dump_json(body)]]
42
+ end
43
+ end
44
+ end