robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
data/lib/robin/web.rb
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "robin"
|
|
4
|
+
require "rack"
|
|
5
|
+
|
|
6
|
+
module Robin
|
|
7
|
+
# The Robin dashboard: a Rack app serving a single-page UI and the JSON
|
|
8
|
+
# API behind it.
|
|
9
|
+
#
|
|
10
|
+
# # config/routes.rb
|
|
11
|
+
# mount Robin::Web => "/robin"
|
|
12
|
+
#
|
|
13
|
+
# # config.ru
|
|
14
|
+
# run Robin::Web
|
|
15
|
+
#
|
|
16
|
+
# Settings:
|
|
17
|
+
#
|
|
18
|
+
# Robin::Web.title = "Billing jobs"
|
|
19
|
+
# Robin::Web.read_only = true
|
|
20
|
+
# Robin::Web.live_updates = :stream # server-sent events instead of polling
|
|
21
|
+
# Robin::Web.clusters = {"us-east" => east_config, "eu-west" => eu_config}
|
|
22
|
+
# Robin::Web.authorize do |env, action, resource|
|
|
23
|
+
# user = env["warden"].user
|
|
24
|
+
# action == :read || user.admin?
|
|
25
|
+
# end
|
|
26
|
+
class Web
|
|
27
|
+
DIST = File.expand_path("web/dist", __dir__)
|
|
28
|
+
ASSETS_PATH = %r{\A(?:https?://[^/]+(?:/|\z)|/(?!/))}i
|
|
29
|
+
|
|
30
|
+
class << self
|
|
31
|
+
attr_accessor :title, :brand_color, :read_only, :stream_interval, :stream_lifetime
|
|
32
|
+
|
|
33
|
+
# :poll (default) or :stream. Streaming pushes stats over server-sent
|
|
34
|
+
# events, but each open dashboard tab then holds a web server thread,
|
|
35
|
+
# so only enable it where threads are plentiful (e.g. a dedicated
|
|
36
|
+
# dashboard process).
|
|
37
|
+
attr_accessor :live_updates
|
|
38
|
+
|
|
39
|
+
# Where browsers load the dashboard's scripts, stylesheets, fonts and
|
|
40
|
+
# images from, e.g. a CDN serving a copy of lib/robin/web/dist/assets
|
|
41
|
+
# (Robin::Web::DIST + "/assets") for each Robin version:
|
|
42
|
+
#
|
|
43
|
+
# Robin::Web.assets_path = "https://cdn.example.com/robin/#{Robin::VERSION}/"
|
|
44
|
+
#
|
|
45
|
+
# An http(s) URL or a path on this host; nil (the default) serves
|
|
46
|
+
# them from the dashboard itself. A CDN must send CORS headers
|
|
47
|
+
# (Access-Control-Allow-Origin), since the scripts are modules.
|
|
48
|
+
attr_reader :assets_path
|
|
49
|
+
|
|
50
|
+
def assets_path=(value)
|
|
51
|
+
path = value&.to_s
|
|
52
|
+
raise ArgumentError, "assets_path must be an http(s) URL or a path starting with /: #{path.inspect}" if path && !ASSETS_PATH.match?(path)
|
|
53
|
+
|
|
54
|
+
@assets_path = path&.sub(%r{/*\z}, "/")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Named Robin configs, one per Redis cluster. Defaults to the global
|
|
58
|
+
# Robin.config.
|
|
59
|
+
def clusters
|
|
60
|
+
@clusters || {"default" => Robin.config}
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def clusters=(hash)
|
|
64
|
+
raise ArgumentError, "clusters must be a non-empty Hash of name => Robin::Config" unless hash.is_a?(Hash) && !hash.empty?
|
|
65
|
+
|
|
66
|
+
@clusters = hash.transform_keys(&:to_s)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Register an authorization callback. It receives the Rack env, the
|
|
70
|
+
# action (see Robin::Web::Permissions::ACTIONS) and a resource string
|
|
71
|
+
# (e.g. "queue:default"); return truthy to allow.
|
|
72
|
+
def authorize(&block)
|
|
73
|
+
@authorizer = block
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
attr_reader :authorizer
|
|
77
|
+
|
|
78
|
+
# Adds a page of your own (a Robin::Web::Extension subclass) to the
|
|
79
|
+
# dashboard. Registering another with the same slug replaces it.
|
|
80
|
+
def register(extension)
|
|
81
|
+
raise ArgumentError, "#{extension.inspect} isn't a Robin::Web::Extension" unless extension.instance_of?(Class) && extension < Extension
|
|
82
|
+
|
|
83
|
+
extensions[extension.slug] = extension
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Registered extensions by slug.
|
|
87
|
+
def extensions = (@extensions ||= {})
|
|
88
|
+
|
|
89
|
+
# Adds a row to every job's detail page. The block gets the job's
|
|
90
|
+
# payload (a Hash) and returns the row's text, a Hash with :text and
|
|
91
|
+
# :href to make it a link (http(s) URLs and paths only), or nil to
|
|
92
|
+
# leave the row out for that job:
|
|
93
|
+
#
|
|
94
|
+
# Robin::Web.job_info("Customer") do |job|
|
|
95
|
+
# id = job["args"].first
|
|
96
|
+
# {text: "##{id}", href: "/admin/customers/#{id}"} if job["class"] == "InvoiceJob"
|
|
97
|
+
# end
|
|
98
|
+
#
|
|
99
|
+
# Adding another row with the same label replaces it.
|
|
100
|
+
def job_info(label, &block)
|
|
101
|
+
raise ArgumentError, "job_info needs a block" unless block
|
|
102
|
+
|
|
103
|
+
job_info_rows[label.to_s] = block
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# The job_info blocks by label.
|
|
107
|
+
def job_info_rows = (@job_info_rows ||= {})
|
|
108
|
+
|
|
109
|
+
def call(env)
|
|
110
|
+
app.call(env)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def app
|
|
114
|
+
@app ||= new
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Restore default settings (used by tests).
|
|
118
|
+
def reset!
|
|
119
|
+
@clusters = nil
|
|
120
|
+
@authorizer = nil
|
|
121
|
+
@extensions = nil
|
|
122
|
+
@job_info_rows = nil
|
|
123
|
+
@app = nil
|
|
124
|
+
self.title = "Robin"
|
|
125
|
+
self.assets_path = nil
|
|
126
|
+
self.brand_color = nil
|
|
127
|
+
self.read_only = false
|
|
128
|
+
self.stream_interval = 2
|
|
129
|
+
self.stream_lifetime = 60
|
|
130
|
+
self.live_updates = :poll
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
reset!
|
|
135
|
+
|
|
136
|
+
def initialize
|
|
137
|
+
@router = Router.new
|
|
138
|
+
@static = Static.new(DIST)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def call(env)
|
|
142
|
+
path = env.fetch("PATH_INFO")
|
|
143
|
+
# /metrics is both the Prometheus endpoint and a dashboard page; a
|
|
144
|
+
# browser asking for HTML gets the page.
|
|
145
|
+
if path == "/metrics" && !env["HTTP_ACCEPT"].to_s.include?("text/html")
|
|
146
|
+
Prometheus.new(env).call
|
|
147
|
+
elsif path == "/health"
|
|
148
|
+
health
|
|
149
|
+
elsif path.start_with?("/api/")
|
|
150
|
+
@router.call(env)
|
|
151
|
+
elsif path.start_with?("/extensions/")
|
|
152
|
+
extension_asset(env, path)
|
|
153
|
+
else
|
|
154
|
+
@static.call(env)
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# GET /health: pings every cluster's Redis, for load balancer and uptime
|
|
159
|
+
# checks. 200 when all of them answer, else 503. Needs no permission:
|
|
160
|
+
# it reveals nothing but reachability and round-trip times.
|
|
161
|
+
def health
|
|
162
|
+
clusters = Web.clusters.to_h do |name, config|
|
|
163
|
+
started = monotonic_ms
|
|
164
|
+
begin
|
|
165
|
+
config.redis { |c| c.call("PING") }
|
|
166
|
+
[name, {"ok" => true, "latency_ms" => (monotonic_ms - started).round(2)}]
|
|
167
|
+
rescue => e
|
|
168
|
+
[name, {"ok" => false, "error" => "#{e.class}: #{e}"}]
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
healthy = clusters.each_value.all? { |c| c.fetch("ok") }
|
|
172
|
+
body = Robin.dump_json({"healthy" => healthy, "redis" => clusters})
|
|
173
|
+
[healthy ? 200 : 503, {"content-type" => "application/json", "cache-control" => "no-store"}, [body]]
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# GET /extensions/<slug>/<file>: a file from an extension's assets.
|
|
177
|
+
def extension_asset(env, path)
|
|
178
|
+
return API.json(403, {"error" => "You don't have access to this dashboard"}) unless Permissions.allowed?(env, :read, path)
|
|
179
|
+
|
|
180
|
+
_, _, slug, file = path.split("/", 4)
|
|
181
|
+
@static.call_file(env, Web.extensions[slug]&.asset_file(file.to_s))
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
private
|
|
185
|
+
|
|
186
|
+
def monotonic_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
require_relative "web/permissions"
|
|
191
|
+
require_relative "web/extension"
|
|
192
|
+
require_relative "web/api"
|
|
193
|
+
require_relative "web/router"
|
|
194
|
+
require_relative "web/static"
|
|
195
|
+
require_relative "web/prometheus"
|
|
196
|
+
require_relative "web/batch_status"
|
data/lib/robin.rb
CHANGED
|
@@ -1,11 +1,261 @@
|
|
|
1
|
-
|
|
2
|
-
require_relative 'robin/args_parser'
|
|
1
|
+
# frozen_string_literal: true
|
|
3
2
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
require "json"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require "logger"
|
|
6
|
+
require "socket"
|
|
7
|
+
|
|
8
|
+
require_relative "robin/version"
|
|
9
|
+
require_relative "robin/redis/connection"
|
|
10
|
+
require_relative "robin/redis/pool"
|
|
11
|
+
|
|
12
|
+
# Robin: fast, reliable background jobs for Ruby.
|
|
13
|
+
#
|
|
14
|
+
# class SendReceipt
|
|
15
|
+
# include Robin::Job
|
|
16
|
+
# robin_options queue: "mailers", retry: 5
|
|
17
|
+
#
|
|
18
|
+
# def perform(order_id)
|
|
19
|
+
# ...
|
|
20
|
+
# end
|
|
21
|
+
# end
|
|
22
|
+
#
|
|
23
|
+
# SendReceipt.perform_async(42)
|
|
24
|
+
module Robin
|
|
25
|
+
NAME = "Robin"
|
|
26
|
+
|
|
27
|
+
# Job ids generated per read of random bytes (see Robin.generate_jid).
|
|
28
|
+
JID_BATCH = 64
|
|
29
|
+
|
|
30
|
+
# A thread's random hex digits for job ids, and how far it has used them.
|
|
31
|
+
JidBatch = Struct.new(:hex, :offset, :pid)
|
|
32
|
+
|
|
33
|
+
class Error < StandardError; end
|
|
34
|
+
|
|
35
|
+
# Raised by a job to stop immediately without retrying.
|
|
36
|
+
class Discard < Error; end
|
|
37
|
+
|
|
38
|
+
# Raised inside worker threads when the server's shutdown timeout
|
|
39
|
+
# elapses. It descends from Interrupt so that application code's
|
|
40
|
+
# `rescue => e` doesn't swallow it.
|
|
41
|
+
class Shutdown < Interrupt; end
|
|
42
|
+
|
|
43
|
+
autoload :Config, "robin/config"
|
|
44
|
+
autoload :Capsule, "robin/config"
|
|
45
|
+
autoload :Middleware, "robin/middleware"
|
|
46
|
+
autoload :Job, "robin/job"
|
|
47
|
+
autoload :Client, "robin/client"
|
|
48
|
+
autoload :Scripts, "robin/scripts"
|
|
49
|
+
autoload :Fetcher, "robin/fetcher"
|
|
50
|
+
autoload :Processor, "robin/processor"
|
|
51
|
+
autoload :Manager, "robin/manager"
|
|
52
|
+
autoload :Launcher, "robin/launcher"
|
|
53
|
+
autoload :HealthServer, "robin/health"
|
|
54
|
+
autoload :Systemd, "robin/systemd"
|
|
55
|
+
autoload :Scheduler, "robin/scheduler"
|
|
56
|
+
autoload :Warmup, "robin/warmup"
|
|
57
|
+
autoload :Retrier, "robin/retrier"
|
|
58
|
+
autoload :JobLogger, "robin/job_logger"
|
|
59
|
+
autoload :Context, "robin/job_logger"
|
|
60
|
+
autoload :Stats, "robin/api"
|
|
61
|
+
autoload :Queue, "robin/api"
|
|
62
|
+
autoload :JobRecord, "robin/api"
|
|
63
|
+
autoload :SortedEntry, "robin/api"
|
|
64
|
+
autoload :JobSet, "robin/api"
|
|
65
|
+
autoload :ScheduledSet, "robin/api"
|
|
66
|
+
autoload :RetrySet, "robin/api"
|
|
67
|
+
autoload :DeadSet, "robin/api"
|
|
68
|
+
autoload :ProcessSet, "robin/api"
|
|
69
|
+
autoload :ProcessRecord, "robin/api"
|
|
70
|
+
autoload :WorkSet, "robin/api"
|
|
71
|
+
autoload :JobLocator, "robin/api"
|
|
72
|
+
autoload :Metrics, "robin/metrics"
|
|
73
|
+
autoload :Deploy, "robin/deploy"
|
|
74
|
+
autoload :Recovery, "robin/recovery"
|
|
75
|
+
autoload :Leader, "robin/leader"
|
|
76
|
+
autoload :Batch, "robin/batch"
|
|
77
|
+
autoload :Limiter, "robin/limiter"
|
|
78
|
+
autoload :Periodic, "robin/periodic"
|
|
79
|
+
autoload :Cron, "robin/cron"
|
|
80
|
+
autoload :Unique, "robin/unique"
|
|
81
|
+
autoload :Events, "robin/events"
|
|
82
|
+
autoload :GVL, "robin/gvl"
|
|
83
|
+
autoload :Encryption, "robin/encryption"
|
|
84
|
+
autoload :Expiry, "robin/expiry"
|
|
85
|
+
autoload :Debounce, "robin/debounce"
|
|
86
|
+
autoload :Cancellation, "robin/cancellation"
|
|
87
|
+
autoload :JobClass, "robin/job_class"
|
|
88
|
+
autoload :Progress, "robin/progress"
|
|
89
|
+
autoload :Profiler, "robin/profiler"
|
|
90
|
+
autoload :IterableJob, "robin/iterable_job"
|
|
91
|
+
autoload :ReliablePush, "robin/reliable_push"
|
|
92
|
+
autoload :Swarm, "robin/swarm"
|
|
93
|
+
autoload :Statsd, "robin/statsd"
|
|
94
|
+
autoload :Embedded, "robin/embedded"
|
|
95
|
+
autoload :CLI, "robin/cli"
|
|
96
|
+
autoload :Top, "robin/top"
|
|
97
|
+
autoload :Testing, "robin/testing"
|
|
98
|
+
|
|
99
|
+
class << self
|
|
100
|
+
def config
|
|
101
|
+
@config ||= Config.new
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Loads every autoloaded part of Robin up front, e.g. before forking so
|
|
105
|
+
# children share the code via copy-on-write. Robin::Testing is skipped:
|
|
106
|
+
# loading it switches pushes to test mode.
|
|
107
|
+
def eager_load!
|
|
108
|
+
constants.each { |name| const_get(name) unless name.equal?(:Testing) }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
attr_writer :config
|
|
112
|
+
|
|
113
|
+
# Configure Robin for every process (client and server).
|
|
114
|
+
def configure
|
|
115
|
+
yield config
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Configure Robin only inside a job-processing server.
|
|
119
|
+
def configure_server
|
|
120
|
+
yield config if server?
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Configure Robin only inside non-server processes (web, console, rake).
|
|
124
|
+
def configure_client
|
|
125
|
+
yield config unless server?
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def server?
|
|
129
|
+
!!@server
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def server!
|
|
133
|
+
@server = true
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Run a Robin server inside the current process (e.g. alongside Puma).
|
|
137
|
+
def configure_embed(&)
|
|
138
|
+
Embedded.new(config).configure(&)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def redis(&)
|
|
142
|
+
config.redis(&)
|
|
143
|
+
end
|
|
8
144
|
|
|
9
|
-
|
|
10
|
-
|
|
145
|
+
# This machine's name for process identities and the web UI: Heroku's
|
|
146
|
+
# dyno name (e.g. "worker.1") when there is one, which means more than
|
|
147
|
+
# its random hostnames.
|
|
148
|
+
def hostname(env: ENV)
|
|
149
|
+
env["DYNO"] || Socket.gethostname
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def logger
|
|
153
|
+
config.logger
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def dump_json(obj)
|
|
157
|
+
JSON.generate(obj)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def load_json(str)
|
|
161
|
+
JSON.parse(str)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# The job as it was stored in Redis, before middleware or the job itself
|
|
165
|
+
# changed anything (encrypted arguments are still encrypted). Use it to
|
|
166
|
+
# write a running job back to Redis. Payloads that didn't come from
|
|
167
|
+
# Redis are returned unchanged.
|
|
168
|
+
def pristine(payload)
|
|
169
|
+
raw = payload.instance_variable_get(:@robin_raw)
|
|
170
|
+
raw ? load_json(raw) : payload
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Whether +exception+ is Robin::Shutdown or was caused by it (a job that
|
|
174
|
+
# rescued the shutdown interrupt and raised its own error instead).
|
|
175
|
+
def shutdown?(exception)
|
|
176
|
+
# @type var exception: Exception?
|
|
177
|
+
seen = Set[]
|
|
178
|
+
while exception && seen.add?(exception)
|
|
179
|
+
return true if exception.is_a?(Shutdown)
|
|
180
|
+
|
|
181
|
+
exception = exception.cause
|
|
182
|
+
end
|
|
183
|
+
false
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Marks +payload+ as parsed from +raw+ (see #pristine).
|
|
187
|
+
def remember_raw(payload, raw)
|
|
188
|
+
payload.instance_variable_set(:@robin_raw, raw)
|
|
189
|
+
payload
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Returns +string+ as valid UTF-8, ready for JSON: text in other
|
|
193
|
+
# encodings is transcoded; bytes that aren't valid UTF-8 (including
|
|
194
|
+
# binary data) become U+FFFD.
|
|
195
|
+
def utf8(string)
|
|
196
|
+
# Binary data is read as UTF-8; encoding to UTF-8 then replaces
|
|
197
|
+
# invalid bytes (it validates even when nothing needs converting).
|
|
198
|
+
string = String.new(string, encoding: Encoding::UTF_8) if string.encoding == Encoding::BINARY
|
|
199
|
+
string.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
|
|
200
|
+
rescue Encoding::ConverterNotFoundError
|
|
201
|
+
String.new(string, encoding: Encoding::UTF_8).scrub
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def now_ms
|
|
205
|
+
Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# When a job runs ("at") and the scores of the schedule, retry and dead
|
|
209
|
+
# sets are epoch seconds, as Floats to the millisecond; every other
|
|
210
|
+
# timestamp is epoch milliseconds. These convert between the two.
|
|
211
|
+
def epoch_seconds(ms) = ms / 1000.0
|
|
212
|
+
|
|
213
|
+
def epoch_ms(seconds) = (Float(seconds) * 1000).round
|
|
214
|
+
|
|
215
|
+
# The class a job shows as: its display_class, the class it wraps (an
|
|
216
|
+
# ActiveJob job's, named in "wrapped"), or its class.
|
|
217
|
+
def display_class(payload)
|
|
218
|
+
wrapped = payload["wrapped"]
|
|
219
|
+
payload["display_class"] || (wrapped if wrapped.instance_of?(String)) || payload["class"]
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# A job's epoch-ms timestamp (created_at, enqueued_at...) as epoch ms,
|
|
223
|
+
# also when it's from a job pushed by an older client that wrote epoch
|
|
224
|
+
# seconds as a Float.
|
|
225
|
+
def ms_of(time) = (time.instance_of?(Float) && time < 100_000_000_000) ? epoch_ms(time) : time
|
|
226
|
+
|
|
227
|
+
# A job id: 24 random hex digits. Random bytes are read JID_BATCH ids
|
|
228
|
+
# at a time (a syscall each read), per thread; a forked child reads its
|
|
229
|
+
# own rather than reusing its parent's.
|
|
230
|
+
def generate_jid
|
|
231
|
+
batch = Thread.current[:robin_jids]
|
|
232
|
+
unless batch && batch.offset < batch.hex.bytesize && batch.pid == Process.pid
|
|
233
|
+
batch = Thread.current[:robin_jids] = JidBatch.new(SecureRandom.hex(12 * JID_BATCH), 0, Process.pid)
|
|
234
|
+
end
|
|
235
|
+
offset = batch.offset
|
|
236
|
+
batch.offset = offset + 24
|
|
237
|
+
batch.hex.byteslice(offset, 24)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Is this process currently holding cluster leadership?
|
|
241
|
+
def leader?
|
|
242
|
+
Leader.leader?
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Requests cancellation of jobs (one or more jids); running jobs
|
|
246
|
+
# observe it through #cancelled?, and pending jobs are skipped when
|
|
247
|
+
# fetched.
|
|
248
|
+
def cancel!(*jids, ttl: 86_400)
|
|
249
|
+
Cancellation.cancel!(jids, ttl: ttl)
|
|
250
|
+
end
|
|
251
|
+
end
|
|
11
252
|
end
|
|
253
|
+
|
|
254
|
+
require_relative "robin/config"
|
|
255
|
+
require_relative "robin/job"
|
|
256
|
+
|
|
257
|
+
# In a Rails app, which loads Rails before Bundler.require loads its gems,
|
|
258
|
+
# the Rails integration (see Robin::Rails) joins the app's boot. Elsewhere
|
|
259
|
+
# it stays off; `require "robin/rails"` turns it on, e.g. when robin is
|
|
260
|
+
# required before Rails.
|
|
261
|
+
require_relative "robin/rails" if defined?(Rails::Railtie)
|