robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "robin/web"
|
|
4
|
+
|
|
5
|
+
# Every process: the web app, consoles and `bundle exec robin`.
|
|
6
|
+
Robin.configure do |config|
|
|
7
|
+
# Robin reads ROBIN_REDIS_URL or REDIS_URL by default.
|
|
8
|
+
# config.redis = {url: "redis://localhost:6379/0"}
|
|
9
|
+
|
|
10
|
+
# Raise (the default), :warn or false when job arguments aren't JSON-native.
|
|
11
|
+
# config.strict_args = :raise
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Only inside `bundle exec robin`.
|
|
15
|
+
Robin.configure_server do |config|
|
|
16
|
+
# config.on(:startup) { Rails.logger.info("Robin started") }
|
|
17
|
+
# config.death_handlers << ->(job, exception) { Rails.logger.error("#{job["class"]} died: #{exception}") }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Who may use the dashboard mounted in config/routes.rb. Reading is allowed
|
|
21
|
+
# here and changes need an admin; adapt it to your authentication.
|
|
22
|
+
# Robin::Web.authorize do |env, action, _resource|
|
|
23
|
+
# user = env["warden"]&.user
|
|
24
|
+
# action == :read || user&.admin?
|
|
25
|
+
# end
|
|
26
|
+
|
|
27
|
+
# Rails settings go in config/application.rb:
|
|
28
|
+
#
|
|
29
|
+
# config.active_job.queue_adapter = :robin # run ActiveJob jobs on Robin
|
|
30
|
+
# config.robin.report_errors = true # report errors to Rails.error (default)
|
|
31
|
+
# config.robin.transactional_push = true # push jobs after transactions commit
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Settings for `bundle exec robin`; command-line options win over these.
|
|
2
|
+
# The file is ERB, and a section named after the environment (RAILS_ENV)
|
|
3
|
+
# overrides the top-level values.
|
|
4
|
+
|
|
5
|
+
# Worker threads per process.
|
|
6
|
+
concurrency: <%%= ENV.fetch("ROBIN_CONCURRENCY", 5) %>
|
|
7
|
+
|
|
8
|
+
# Seconds running jobs get to finish when the process stops.
|
|
9
|
+
timeout: 25
|
|
10
|
+
|
|
11
|
+
# Queues in strict priority order. Use "name,weight" entries (e.g.
|
|
12
|
+
# "critical,3") to check queues in proportion to their weights instead.
|
|
13
|
+
queues:
|
|
14
|
+
- critical
|
|
15
|
+
- default
|
|
16
|
+
- low
|
|
17
|
+
|
|
18
|
+
# Capsules run some queues on their own threads, e.g. one at a time:
|
|
19
|
+
# capsules:
|
|
20
|
+
# serial:
|
|
21
|
+
# concurrency: 1
|
|
22
|
+
# queues: [ledger]
|
|
23
|
+
|
|
24
|
+
production:
|
|
25
|
+
concurrency: <%%= ENV.fetch("ROBIN_CONCURRENCY", 10) %>
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators/named_base"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
module Generators
|
|
7
|
+
# rails generate robin:job NAME [option:value ...]
|
|
8
|
+
#
|
|
9
|
+
# rails generate robin:job report queue:low retry:5
|
|
10
|
+
#
|
|
11
|
+
# Creates app/jobs/report_job.rb, a Robin::Job with those robin_options
|
|
12
|
+
# (queue defaults to "default"), and a test: spec/jobs/report_job_spec.rb
|
|
13
|
+
# when the app generates RSpec tests, else test/jobs/report_job_test.rb.
|
|
14
|
+
class JobGenerator < ::Rails::Generators::NamedBase
|
|
15
|
+
# The job to generate. (Kept out of the generator, where every public
|
|
16
|
+
# method is a step of the generator.)
|
|
17
|
+
class Blueprint
|
|
18
|
+
attr_reader :path, :class_name, :robin_options
|
|
19
|
+
|
|
20
|
+
# A trailing "_job" is implied, so "report" and "report_job" match.
|
|
21
|
+
# "queue:low retry:5 encrypt" means {"queue" => "low", "retry" =>
|
|
22
|
+
# "5", "encrypt" => nil}.
|
|
23
|
+
def initialize(name, pairs)
|
|
24
|
+
base = name.underscore.delete_suffix("_job") # "Admin::Report" is "admin/report"
|
|
25
|
+
@path = "#{base}_job"
|
|
26
|
+
@class_name = "#{base.camelize}Job"
|
|
27
|
+
options = pairs.to_h do |pair|
|
|
28
|
+
key, colon, value = pair.partition(":")
|
|
29
|
+
[key, (value unless colon.empty?)]
|
|
30
|
+
end
|
|
31
|
+
@robin_options = {"queue" => "default"}.merge(options)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def queue = robin_options.fetch("queue")
|
|
35
|
+
|
|
36
|
+
# As Ruby source: `queue: "low", retry: 5, encrypt: true`.
|
|
37
|
+
def robin_options_source
|
|
38
|
+
robin_options.map { |key, value| "#{key}: #{literal(value)}" }.join(", ")
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def literal(value)
|
|
44
|
+
case value
|
|
45
|
+
when nil then "true"
|
|
46
|
+
when /\A-?\d+\z/, "true", "false" then value
|
|
47
|
+
else value.dump
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
source_root File.expand_path("templates", __dir__)
|
|
53
|
+
desc "Creates a Robin job and its test."
|
|
54
|
+
|
|
55
|
+
argument :options_list, type: :array, default: [], banner: "option:value option:value"
|
|
56
|
+
class_option :test_framework, type: :string, desc: "Test framework for the job's test (rspec or test_unit)"
|
|
57
|
+
|
|
58
|
+
def initialize(*)
|
|
59
|
+
super
|
|
60
|
+
@job = Blueprint.new(name, options_list)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def create_job
|
|
64
|
+
template "job.rb", "app/jobs/#{@job.path}.rb"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Follows the app's generator settings (config.generators.test_framework);
|
|
68
|
+
# none when it's unset.
|
|
69
|
+
def create_test
|
|
70
|
+
case options[:test_framework].to_s
|
|
71
|
+
when "" then nil
|
|
72
|
+
when "rspec" then template "job_spec.rb", "spec/jobs/#{@job.path}_spec.rb"
|
|
73
|
+
else template "job_test.rb", "test/jobs/#{@job.path}_test.rb"
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails_helper"
|
|
4
|
+
require "robin/testing"
|
|
5
|
+
|
|
6
|
+
RSpec.describe <%= @job.class_name %> do
|
|
7
|
+
before { described_class.clear }
|
|
8
|
+
|
|
9
|
+
it "is pushed to the <%= @job.queue %> queue" do
|
|
10
|
+
described_class.perform_async
|
|
11
|
+
|
|
12
|
+
expect(described_class.jobs.map { |job| job["queue"] }).to eq(["<%= @job.queue %>"])
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
it "performs" do
|
|
16
|
+
described_class.perform_async
|
|
17
|
+
described_class.drain
|
|
18
|
+
|
|
19
|
+
expect(described_class.jobs).to be_empty
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
require "robin/testing"
|
|
5
|
+
|
|
6
|
+
class <%= @job.class_name %>Test < ActiveSupport::TestCase
|
|
7
|
+
setup { <%= @job.class_name %>.clear }
|
|
8
|
+
|
|
9
|
+
test "is pushed to the <%= @job.queue %> queue" do
|
|
10
|
+
<%= @job.class_name %>.perform_async
|
|
11
|
+
|
|
12
|
+
assert_equal ["<%= @job.queue %>"], <%= @job.class_name %>.jobs.map { |job| job["queue"] }
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
test "performs" do
|
|
16
|
+
<%= @job.class_name %>.perform_async
|
|
17
|
+
<%= @job.class_name %>.drain
|
|
18
|
+
|
|
19
|
+
assert_empty <%= @job.class_name %>.jobs
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "puma/plugin"
|
|
4
|
+
require "puma/dsl"
|
|
5
|
+
|
|
6
|
+
module Puma
|
|
7
|
+
# Runs an embedded Robin server in every Puma process that serves
|
|
8
|
+
# requests: each worker in cluster mode, the server itself in single mode.
|
|
9
|
+
#
|
|
10
|
+
# # config/puma.rb
|
|
11
|
+
# plugin :robin
|
|
12
|
+
# robin_config do |config| # optional, like Robin.configure_embed's block
|
|
13
|
+
# config.concurrency = 2
|
|
14
|
+
# end
|
|
15
|
+
#
|
|
16
|
+
# Robin is only required once the application has loaded in the process
|
|
17
|
+
# that runs it, never in Puma's master: requiring it there would activate
|
|
18
|
+
# gems in the master that every worker inherits, so after a deploy that
|
|
19
|
+
# changes their versions, phased restarts would boot workers that can't
|
|
20
|
+
# load the new bundle. Starting after the application has loaded also
|
|
21
|
+
# means the application's own Robin configuration (Redis, middleware)
|
|
22
|
+
# is in place.
|
|
23
|
+
#
|
|
24
|
+
# The server starts when Puma's server starts accepting requests, and
|
|
25
|
+
# stops (finishing running jobs or putting them back on their queues)
|
|
26
|
+
# while Puma stops: on shutdown, on hot and phased restarts and when
|
|
27
|
+
# a worker is replaced.
|
|
28
|
+
class RobinPlugin < Plugin
|
|
29
|
+
# Server states Puma::Server reports when it stops accepting requests.
|
|
30
|
+
STOPPING = %i[stop restart halt].freeze
|
|
31
|
+
|
|
32
|
+
# Puma::DSL methods: the blocks, run with Robin's config before the
|
|
33
|
+
# first server starts, are kept in Puma's options.
|
|
34
|
+
module DSL
|
|
35
|
+
def robin_config(&block)
|
|
36
|
+
raise ArgumentError, "robin_config needs a block" unless block
|
|
37
|
+
|
|
38
|
+
(@options[:robin_config] ||= []) << block
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def start(launcher)
|
|
43
|
+
@launcher = launcher
|
|
44
|
+
warn_about_master if ENV.key?("PUMA_BUNDLER_PRUNED")
|
|
45
|
+
launcher.events.register(:state) { |state| changed(state) }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Puma's server is :booting, :running, stopping (STOPPING) or :done.
|
|
49
|
+
def changed(state)
|
|
50
|
+
# A worker forked from another (fork_worker) inherits this object.
|
|
51
|
+
forget_parent unless @pid == Process.pid
|
|
52
|
+
case state
|
|
53
|
+
when :running then run
|
|
54
|
+
when *STOPPING then stop
|
|
55
|
+
when :done then stop&.join
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def forget_parent
|
|
62
|
+
@pid = Process.pid
|
|
63
|
+
@robin = nil
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def run
|
|
67
|
+
return if @robin
|
|
68
|
+
|
|
69
|
+
require "robin"
|
|
70
|
+
blocks = @configured ? [] : Array(@launcher.options[:robin_config]) #: Array[^(Robin::Config) -> void]
|
|
71
|
+
embed = Robin.configure_embed { |config| blocks.each { |block| block.call(config) } }
|
|
72
|
+
@configured = true
|
|
73
|
+
warn_about_timeout
|
|
74
|
+
@robin = embed.run
|
|
75
|
+
log "* Robin #{Robin::VERSION} running #{Robin.config.total_concurrency} worker thread(s)"
|
|
76
|
+
rescue => e
|
|
77
|
+
log_error("start", e)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Stops the server in the background, so Puma can finish its requests
|
|
81
|
+
# meanwhile, and returns the thread doing it (or that did, if any).
|
|
82
|
+
def stop
|
|
83
|
+
robin = @robin
|
|
84
|
+
@robin = nil
|
|
85
|
+
@stopper = Thread.new { stop_now(robin) } if robin
|
|
86
|
+
@stopper
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def stop_now(robin)
|
|
90
|
+
robin.stop
|
|
91
|
+
log "* Robin stopped"
|
|
92
|
+
rescue => e
|
|
93
|
+
log_error("stop", e)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# In cluster mode Puma kills a worker that's still stopping after
|
|
97
|
+
# worker_shutdown_timeout.
|
|
98
|
+
def warn_about_timeout
|
|
99
|
+
options = @launcher.options
|
|
100
|
+
return unless options.fetch(:workers).positive?
|
|
101
|
+
|
|
102
|
+
needed = Robin.config.timeout + Robin::Launcher::HARD_SHUTDOWN_GRACE
|
|
103
|
+
limit = options.fetch(:worker_shutdown_timeout)
|
|
104
|
+
return if needed < limit
|
|
105
|
+
|
|
106
|
+
log "! Robin may need #{needed}s to stop (config.timeout plus #{Robin::Launcher::HARD_SHUTDOWN_GRACE}s), " \
|
|
107
|
+
"but Puma kills workers after worker_shutdown_timeout (#{limit}s)"
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# The master of a pruned bundle must not have Robin's gems activated.
|
|
111
|
+
def warn_about_master
|
|
112
|
+
return unless defined?(Robin.configure_embed) || Gem.loaded_specs.key?("robin")
|
|
113
|
+
|
|
114
|
+
log "! Robin is loaded in Puma's master process, so workers inherit the gems it activated and phased " \
|
|
115
|
+
"restarts can fail after a deploy changes their versions. Don't require robin in config/puma.rb, " \
|
|
116
|
+
"and with prune_bundler add extra_runtime_dependencies [\"robin\"]"
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def log(message) = @launcher.log_writer.log(message)
|
|
120
|
+
|
|
121
|
+
def log_error(what, error)
|
|
122
|
+
log "! Robin failed to #{what}: #{error} (#{error.class})"
|
|
123
|
+
# (A rescued error has a backtrace.)
|
|
124
|
+
backtrace = error.backtrace #: Array[String]
|
|
125
|
+
backtrace.each { |line| log " #{line}" }
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
DSL.include(RobinPlugin::DSL)
|
|
130
|
+
Plugins.register("robin", RobinPlugin)
|
|
131
|
+
end
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_job"
|
|
4
|
+
require "robin"
|
|
5
|
+
|
|
6
|
+
module Robin
|
|
7
|
+
# ActiveJob integration.
|
|
8
|
+
#
|
|
9
|
+
# # config/application.rb
|
|
10
|
+
# config.active_job.queue_adapter = :robin
|
|
11
|
+
#
|
|
12
|
+
# class ReportJob < ApplicationJob
|
|
13
|
+
# robin_options retry: 5, expires_in: 3600, tags: ["reports"]
|
|
14
|
+
# end
|
|
15
|
+
module ActiveJob
|
|
16
|
+
# A Robin::Job inclusion check: ActiveJob classes run on Robin through
|
|
17
|
+
# the adapter, not by including Robin::Job.
|
|
18
|
+
REJECT_ACTIVE_JOB_CLASSES = lambda do |base|
|
|
19
|
+
raise ArgumentError, "Robin::Job can't be included in ActiveJob classes" if base <= ::ActiveJob::Base
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# robin_options that the adapter sets itself.
|
|
23
|
+
IGNORED = %w[queue class args jid at].freeze
|
|
24
|
+
|
|
25
|
+
# A Robin::Client rewriter: a job hash naming an ActiveJob class (as
|
|
26
|
+
# periodic jobs may) becomes the wrapper's payload for
|
|
27
|
+
# Class.new(*args), on the hash's queue or the job's own. Other keys
|
|
28
|
+
# (e.g. "retry") override the class's robin_options. ActiveJob's
|
|
29
|
+
# enqueue callbacks don't run, as with ActiveJob.perform_all_later.
|
|
30
|
+
WRAP_ACTIVE_JOB_CLASSES = lambda do |item|
|
|
31
|
+
klass = item["class"]
|
|
32
|
+
next item unless klass.instance_of?(Class) && klass <= ::ActiveJob::Base
|
|
33
|
+
|
|
34
|
+
job = klass.new(*item.fetch("args"))
|
|
35
|
+
job.queue_name = item["queue"] if item["queue"]
|
|
36
|
+
ActiveJob.payload(job, item["at"], item.except(*IGNORED))
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# The Robin job hash that runs +job+ through Wrapper.
|
|
40
|
+
def self.payload(job, timestamp, extra = {})
|
|
41
|
+
options = job.class.respond_to?(:robin_options) ? job.class.robin_options.except(*IGNORED) : {} #: job_hash
|
|
42
|
+
# The client defaults "retry" to true, accepts Time or epoch "at"
|
|
43
|
+
# values, and drops a nil "at".
|
|
44
|
+
options.merge(extra).merge(
|
|
45
|
+
{"class" => Wrapper, "wrapped" => job.class.name, "display_class" => display_class(job),
|
|
46
|
+
"queue" => job.queue_name, "args" => [job.serialize], "at" => timestamp}
|
|
47
|
+
)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Mail deliveries show as the mail they send ("UserMailer#welcome")
|
|
51
|
+
# rather than all as ActionMailer::MailDeliveryJob.
|
|
52
|
+
def self.display_class(job)
|
|
53
|
+
return job.class.name unless job.class.ancestors.any? { |c| c.name == "ActionMailer::MailDeliveryJob" }
|
|
54
|
+
|
|
55
|
+
mailer, mail = job.arguments
|
|
56
|
+
"#{mailer}##{mail}"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Runs ActiveJob jobs inside Robin.
|
|
60
|
+
class Wrapper
|
|
61
|
+
include Robin::Job
|
|
62
|
+
|
|
63
|
+
def perform(job_data)
|
|
64
|
+
Thread.current[:robin_active_job] = self
|
|
65
|
+
::ActiveJob::Base.execute(job_data.merge({"provider_job_id" => jid}))
|
|
66
|
+
ensure
|
|
67
|
+
Thread.current[:robin_active_job] = nil
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Adds +robin_options+ to ActiveJob classes.
|
|
72
|
+
module Options
|
|
73
|
+
def self.included(base)
|
|
74
|
+
base.extend(ClassMethods)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
module ClassMethods
|
|
78
|
+
def robin_options(opts = nil)
|
|
79
|
+
if opts
|
|
80
|
+
@robin_options = robin_options.merge(opts.to_h { |k, v| [k.to_s, v.instance_of?(Symbol) ? v.to_s : v] })
|
|
81
|
+
end
|
|
82
|
+
@robin_options || (superclass.respond_to?(:robin_options) ? superclass.robin_options : {})
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
module ActiveJob
|
|
90
|
+
module QueueAdapters
|
|
91
|
+
# The ActiveJob adapter, selected with +queue_adapter = :robin+.
|
|
92
|
+
class RobinAdapter < AbstractAdapter
|
|
93
|
+
def enqueue(job)
|
|
94
|
+
enqueue_at(job, nil)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def enqueue_at(job, timestamp)
|
|
98
|
+
jid = Robin::Client.push(payload(job, timestamp))
|
|
99
|
+
job.provider_job_id = jid
|
|
100
|
+
job.successfully_enqueued = !jid.nil? if job.respond_to?(:successfully_enqueued=)
|
|
101
|
+
jid
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Bulk enqueue (ActiveJob.perform_all_later): a bulk push per class,
|
|
105
|
+
# queue and time, all in one round trip.
|
|
106
|
+
def enqueue_all(jobs)
|
|
107
|
+
pushed = Robin::Client.together do
|
|
108
|
+
jobs.group_by do |j|
|
|
109
|
+
[j.class, j.queue_name, j.scheduled_at] #: [Class, String, Time | Float | nil]
|
|
110
|
+
end.map do |(_klass, _queue, scheduled_at), group|
|
|
111
|
+
base = payload(group.first, scheduled_at)
|
|
112
|
+
base["args"] = group.map { |j| [j.serialize] }
|
|
113
|
+
[group, Robin::Client.push_bulk(base)] #: [Array[::ActiveJob::Base], Array[String?]]
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
pushed.sum do |group, jids|
|
|
117
|
+
group.zip(jids).count do |job, jid|
|
|
118
|
+
job.provider_job_id = jid
|
|
119
|
+
job.successfully_enqueued = !jid.nil? if job.respond_to?(:successfully_enqueued=)
|
|
120
|
+
jid
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Lets ActiveJob::Continuation (Rails 8.1+) checkpoint when the server
|
|
126
|
+
# shuts down. Rails 8.1's adapters keep +stopping+ the same way; older
|
|
127
|
+
# ones have no such attribute, so it's defined here for all of them.
|
|
128
|
+
attr_writer :stopping
|
|
129
|
+
|
|
130
|
+
def stopping?
|
|
131
|
+
job = Thread.current[:robin_active_job]
|
|
132
|
+
job ? job.interrupted? : !!@stopping
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
private
|
|
136
|
+
|
|
137
|
+
def payload(job, timestamp) = Robin::ActiveJob.payload(job, timestamp)
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
Robin::Job.inclusion_checks << Robin::ActiveJob::REJECT_ACTIVE_JOB_CLASSES
|
|
143
|
+
Robin::Client.rewriters << Robin::ActiveJob::WRAP_ACTIVE_JOB_CLASSES
|
|
144
|
+
ActiveSupport.on_load(:active_job) { include Robin::ActiveJob::Options }
|