robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
data/lib/robin/cron.rb
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# A cron expression.
|
|
7
|
+
#
|
|
8
|
+
# cron = Robin::Cron.new("*/15 9-17 * * MON-FRI", tz: "America/New_York")
|
|
9
|
+
# cron.next_time # => next matching Time (UTC)
|
|
10
|
+
# cron.previous_time
|
|
11
|
+
# cron.description # => "every 15 minutes from 09:00 to 17:59 on weekdays (America/New_York)"
|
|
12
|
+
#
|
|
13
|
+
# Five fields (minute hour day-of-month month day-of-week), or six with a
|
|
14
|
+
# leading seconds field. Supports *, lists (1,2), ranges (1-5), steps
|
|
15
|
+
# (*/5, 10-40/10, 5/15), month and weekday names, ? (same as *), L in the
|
|
16
|
+
# day-of-month field (last day of the month), 7 as Sunday, and the
|
|
17
|
+
# @yearly, @annually, @monthly, @weekly, @daily, @midnight and @hourly
|
|
18
|
+
# shortcuts. When both day fields are restricted, a day matches if
|
|
19
|
+
# either does.
|
|
20
|
+
#
|
|
21
|
+
# Time zones: nil (the system zone), "UTC", fixed offsets ("+05:30"), or
|
|
22
|
+
# IANA names ("Europe/Berlin", needs the tzinfo gem). Wall-clock times
|
|
23
|
+
# skipped by a daylight-saving change don't run; repeated ones run once.
|
|
24
|
+
class Cron
|
|
25
|
+
class ParseError < ArgumentError; end
|
|
26
|
+
|
|
27
|
+
SHORTCUTS = {
|
|
28
|
+
"@yearly" => "0 0 1 1 *", "@annually" => "0 0 1 1 *", "@monthly" => "0 0 1 * *",
|
|
29
|
+
"@weekly" => "0 0 * * 0", "@daily" => "0 0 * * *", "@midnight" => "0 0 * * *", "@hourly" => "0 * * * *"
|
|
30
|
+
}.freeze
|
|
31
|
+
MONTHS = %w[JAN FEB MAR APR MAY JUN JUL AUG SEP OCT NOV DEC].each_with_index.to_h { |m, i| [m, i + 1] }.freeze
|
|
32
|
+
DAYS = %w[SUN MON TUE WED THU FRI SAT].each_with_index.to_h.freeze
|
|
33
|
+
DAY_NAMES = %w[Sunday Monday Tuesday Wednesday Thursday Friday Saturday].freeze
|
|
34
|
+
MONTH_NAMES = %w[January February March April May June July August September October November December].freeze
|
|
35
|
+
ANY = %w[* ?].freeze
|
|
36
|
+
|
|
37
|
+
# Upper bound on search steps. Every step moves to the next candidate
|
|
38
|
+
# month, day, hour, minute or second, so this spans decades even for
|
|
39
|
+
# the sparsest expressions; impossible ones (Feb 30) give up with nil.
|
|
40
|
+
MAX_STEPS = 20_000
|
|
41
|
+
|
|
42
|
+
attr_reader :expression, :tz, :zone, :seconds, :minutes, :hours, :days, :months, :weekdays
|
|
43
|
+
|
|
44
|
+
def initialize(expression, tz: nil)
|
|
45
|
+
@expression = expression.to_s.strip
|
|
46
|
+
@tz = tz
|
|
47
|
+
@zone = Zone.for(tz)
|
|
48
|
+
parse(SHORTCUTS.fetch(@expression.downcase, @expression))
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def to_s = @expression
|
|
52
|
+
|
|
53
|
+
def inspect = "#<Robin::Cron #{@expression.inspect}#{" tz=#{@tz}" if @tz}>"
|
|
54
|
+
|
|
55
|
+
# Whether +time+ (to the second) matches.
|
|
56
|
+
def match?(time)
|
|
57
|
+
w = @zone.to_wall(time)
|
|
58
|
+
@months.include?(w.month) && day_match?(w) && @hours.include?(w.hour) &&
|
|
59
|
+
@minutes.include?(w.min) && @seconds.include?(w.sec)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The first matching time strictly after +from+, in UTC.
|
|
63
|
+
def next_time(from = Time.now)
|
|
64
|
+
w = @zone.to_wall(from)
|
|
65
|
+
MAX_STEPS.times do
|
|
66
|
+
if !@months.include?(w.month)
|
|
67
|
+
w = next_month(w)
|
|
68
|
+
elsif !day_match?(w)
|
|
69
|
+
w = next_day(w)
|
|
70
|
+
elsif !(h = @hours.find { |x| x >= w.hour })
|
|
71
|
+
w = next_day(w)
|
|
72
|
+
elsif h > w.hour
|
|
73
|
+
w = Time.utc(w.year, w.month, w.day, h)
|
|
74
|
+
elsif !(m = @minutes.find { |x| x >= w.min })
|
|
75
|
+
w = Time.utc(w.year, w.month, w.day, w.hour) + 3_600
|
|
76
|
+
elsif m > w.min
|
|
77
|
+
w = Time.utc(w.year, w.month, w.day, w.hour, m)
|
|
78
|
+
elsif !(s = @seconds.find { |x| x >= w.sec })
|
|
79
|
+
w = Time.utc(w.year, w.month, w.day, w.hour, w.min) + 60
|
|
80
|
+
else
|
|
81
|
+
w = Time.utc(w.year, w.month, w.day, w.hour, w.min, s)
|
|
82
|
+
utc = @zone.to_utc(w)
|
|
83
|
+
return utc if utc && utc > from
|
|
84
|
+
|
|
85
|
+
w += 1
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The last matching time strictly before +from+, in UTC.
|
|
92
|
+
def previous_time(from = Time.now)
|
|
93
|
+
w = @zone.to_wall(from)
|
|
94
|
+
MAX_STEPS.times do
|
|
95
|
+
if !@months.include?(w.month)
|
|
96
|
+
w = Time.utc(w.year, w.month) - 1
|
|
97
|
+
elsif !day_match?(w)
|
|
98
|
+
w = Time.utc(w.year, w.month, w.day) - 1
|
|
99
|
+
elsif !(h = @hours.reverse_each.find { |x| x <= w.hour })
|
|
100
|
+
w = Time.utc(w.year, w.month, w.day) - 1
|
|
101
|
+
elsif h < w.hour
|
|
102
|
+
w = Time.utc(w.year, w.month, w.day, h, 59, 59)
|
|
103
|
+
elsif !(m = @minutes.reverse_each.find { |x| x <= w.min })
|
|
104
|
+
w = Time.utc(w.year, w.month, w.day, w.hour) - 1
|
|
105
|
+
elsif m < w.min
|
|
106
|
+
w = Time.utc(w.year, w.month, w.day, w.hour, m, 59)
|
|
107
|
+
elsif !(s = @seconds.reverse_each.find { |x| x <= w.sec })
|
|
108
|
+
w = Time.utc(w.year, w.month, w.day, w.hour, w.min) - 1
|
|
109
|
+
else
|
|
110
|
+
w = Time.utc(w.year, w.month, w.day, w.hour, w.min, s)
|
|
111
|
+
utc = @zone.to_utc(w)
|
|
112
|
+
return utc if utc && utc < from
|
|
113
|
+
|
|
114
|
+
w -= 1
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
nil
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Best-effort English description, e.g. "every 5 minutes" or
|
|
121
|
+
# "at 03:30 on Monday". Falls back to the expression itself.
|
|
122
|
+
def description
|
|
123
|
+
text = describe_time
|
|
124
|
+
return @expression unless text
|
|
125
|
+
|
|
126
|
+
parts = [text, describe_days, describe_months].compact
|
|
127
|
+
label = @zone.label
|
|
128
|
+
parts << "(#{label})" if label
|
|
129
|
+
parts.join(" ")
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
private
|
|
133
|
+
|
|
134
|
+
def next_month(w) = (w.month == 12) ? Time.utc(w.year + 1) : Time.utc(w.year, w.month + 1)
|
|
135
|
+
|
|
136
|
+
def next_day(w) = Time.utc(w.year, w.month, w.day) + 86_400
|
|
137
|
+
|
|
138
|
+
def describe_time
|
|
139
|
+
minute_step = step_of(@minutes, 59)
|
|
140
|
+
hour_step = step_of(@hours, 23)
|
|
141
|
+
second_step = step_of(@seconds, 59)
|
|
142
|
+
if !@seconds.eql?([0])
|
|
143
|
+
every(second_step, "second") if second_step
|
|
144
|
+
elsif minute_step && hour_step == 1
|
|
145
|
+
every(minute_step, "minute")
|
|
146
|
+
elsif minute_step && contiguous?(@hours)
|
|
147
|
+
format("%s from %02d:00 to %02d:59", every(minute_step, "minute"), @hours.first, @hours.last)
|
|
148
|
+
elsif @minutes.one? && hour_step
|
|
149
|
+
format("%s at :%02d", every(hour_step, "hour"), @minutes.first)
|
|
150
|
+
elsif @minutes.one? && @hours.one?
|
|
151
|
+
format("at %02d:%02d", @hours.first, @minutes.first)
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# 1 for every value, n for "*/n", nil for anything else.
|
|
156
|
+
def step_of(values, max) = (1..max).find { |n| values == 0.step(max, n).to_a }
|
|
157
|
+
|
|
158
|
+
def contiguous?(values) = values == (values.first..values.last).to_a
|
|
159
|
+
|
|
160
|
+
def every(step, unit) = (step == 1) ? "every #{unit}" : "every #{step} #{unit}s"
|
|
161
|
+
|
|
162
|
+
def describe_days
|
|
163
|
+
days = [describe_dom, describe_dow].compact
|
|
164
|
+
days.join(" or ") unless days.empty?
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def describe_dom
|
|
168
|
+
return "on the last day of the month" if @last_dom
|
|
169
|
+
|
|
170
|
+
"on day #{@days.join(", ")} of the month" unless @dom_any
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def describe_dow
|
|
174
|
+
return if @dow_any
|
|
175
|
+
|
|
176
|
+
case @weekdays
|
|
177
|
+
when [1, 2, 3, 4, 5] then "on weekdays"
|
|
178
|
+
when [0, 6] then "on weekends"
|
|
179
|
+
else "on #{@weekdays.map { |d| DAY_NAMES.fetch(d) }.join(", ")}"
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def describe_months
|
|
184
|
+
"in #{@months.map { |m| MONTH_NAMES.fetch(m - 1) }.join(", ")}" if @months.size < 12
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def parse(expr)
|
|
188
|
+
parts = expr.split
|
|
189
|
+
parts.unshift("0") if parts.size == 5
|
|
190
|
+
raise ParseError, "cron expression must have 5 or 6 fields: #{@expression.inspect}" unless parts.size == 6
|
|
191
|
+
|
|
192
|
+
sec, min, hour, dom, month, dow = parts #: [String, String, String, String, String, String]
|
|
193
|
+
@seconds = field(sec, 0, 59)
|
|
194
|
+
@minutes = field(min, 0, 59)
|
|
195
|
+
@hours = field(hour, 0, 23)
|
|
196
|
+
@last_dom = dom.casecmp?("L")
|
|
197
|
+
@days = @last_dom ? [] : field(dom, 1, 31)
|
|
198
|
+
@months = field(month, 1, 12, MONTHS)
|
|
199
|
+
weekdays = field(dow, 0, 7, DAYS)
|
|
200
|
+
@weekdays = weekdays.include?(7) ? ((weekdays - [7]) | [0]).sort : weekdays
|
|
201
|
+
@dom_any = ANY.include?(dom)
|
|
202
|
+
@dow_any = ANY.include?(dow)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def field(text, min, max, names = {})
|
|
206
|
+
values = text.split(",").flat_map do |part|
|
|
207
|
+
range, step = part.split("/", 2) #: [String, String] | [String]
|
|
208
|
+
step = step ? Integer(step, 10, exception: false) : 1
|
|
209
|
+
raise ParseError, "bad step in #{part.inspect}" unless step&.positive?
|
|
210
|
+
|
|
211
|
+
first, last =
|
|
212
|
+
if ANY.include?(range)
|
|
213
|
+
[min, max]
|
|
214
|
+
elsif range.include?("-")
|
|
215
|
+
range.split("-", 2).map { |v| value(v, names) }
|
|
216
|
+
else
|
|
217
|
+
v = value(range, names)
|
|
218
|
+
[v, part.include?("/") ? max : v]
|
|
219
|
+
end #: [Integer, Integer]
|
|
220
|
+
raise ParseError, "#{part.inspect} is out of range #{min}-#{max}" if first < min || last > max || first > last
|
|
221
|
+
|
|
222
|
+
first.step(last, step).to_a #: Array[Integer]
|
|
223
|
+
end
|
|
224
|
+
values.uniq.sort
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def value(text, names)
|
|
228
|
+
names.fetch(text.upcase) { Integer(text, 10) }
|
|
229
|
+
rescue ArgumentError
|
|
230
|
+
raise ParseError, "invalid cron value #{text.inspect} in #{@expression.inspect}"
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def day_match?(w)
|
|
234
|
+
dom = @last_dom ? Date.new(w.year, w.month, -1).day == w.day : @days.include?(w.day)
|
|
235
|
+
dow = @weekdays.include?(w.wday)
|
|
236
|
+
if @dom_any then dow
|
|
237
|
+
elsif @dow_any then dom
|
|
238
|
+
else dom || dow
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# Converts between UTC instants and "wall clock" times, represented as
|
|
243
|
+
# UTC Time objects holding the local fields.
|
|
244
|
+
module Zone
|
|
245
|
+
OFFSET = /\A(?<sign>[+-])(?<hours>\d{2}):?(?<minutes>\d{2})\z/
|
|
246
|
+
|
|
247
|
+
def self.for(tz)
|
|
248
|
+
case tz
|
|
249
|
+
when nil, "" then Local.new
|
|
250
|
+
when "UTC", "Etc/UTC", "GMT", "Z" then Fixed.new(0, "UTC")
|
|
251
|
+
when OFFSET
|
|
252
|
+
m = Regexp.last_match #: MatchData
|
|
253
|
+
seconds = Integer(m[:hours], 10) * 3600 + Integer(m[:minutes], 10) * 60
|
|
254
|
+
Fixed.new((m[:sign] == "-") ? -seconds : seconds, tz)
|
|
255
|
+
else IANA.new(tz)
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
class Fixed
|
|
260
|
+
attr_reader :label
|
|
261
|
+
|
|
262
|
+
def initialize(offset, label)
|
|
263
|
+
@offset = offset
|
|
264
|
+
@label = label
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def to_wall(time) = (time + @offset).utc
|
|
268
|
+
|
|
269
|
+
def to_utc(wall) = wall - @offset
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
class Local
|
|
273
|
+
attr_reader :label
|
|
274
|
+
|
|
275
|
+
def to_wall(time)
|
|
276
|
+
t = time.getlocal
|
|
277
|
+
Time.utc(t.year, t.month, t.day, t.hour, t.min, t.sec)
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# nil for wall times skipped by a daylight-saving change.
|
|
281
|
+
def to_utc(wall)
|
|
282
|
+
t = Time.local(wall.year, wall.month, wall.day, wall.hour, wall.min, wall.sec)
|
|
283
|
+
t.getutc if t.hour == wall.hour && t.min == wall.min
|
|
284
|
+
end
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
class IANA
|
|
288
|
+
attr_reader :label
|
|
289
|
+
|
|
290
|
+
def self.lookup(name)
|
|
291
|
+
load_tzinfo(name)
|
|
292
|
+
begin
|
|
293
|
+
TZInfo::Timezone.get(name)
|
|
294
|
+
rescue TZInfo::InvalidTimezoneIdentifier
|
|
295
|
+
raise ParseError, "unknown time zone #{name.inspect}"
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def self.load_tzinfo(name)
|
|
300
|
+
require "tzinfo"
|
|
301
|
+
rescue LoadError
|
|
302
|
+
raise ParseError, "time zone #{name.inspect} needs the tzinfo gem (add it to your Gemfile)"
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def initialize(name)
|
|
306
|
+
@tz = self.class.lookup(name)
|
|
307
|
+
@label = name
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
def to_wall(time)
|
|
311
|
+
t = @tz.utc_to_local(time.getutc)
|
|
312
|
+
Time.utc(t.year, t.month, t.day, t.hour, t.min, t.sec)
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
# nil for wall times skipped by a daylight-saving change; the
|
|
316
|
+
# first occurrence for repeated ones.
|
|
317
|
+
def to_utc(wall)
|
|
318
|
+
@tz.local_to_utc(wall, true)
|
|
319
|
+
rescue TZInfo::PeriodNotFound
|
|
320
|
+
nil
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
end
|
|
325
|
+
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest/sha2"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Debounced jobs: a burst of pushes with the same arguments collapses
|
|
7
|
+
# into a single run, +debounce+ seconds after the last push.
|
|
8
|
+
#
|
|
9
|
+
# class ReindexDocument
|
|
10
|
+
# include Robin::Job
|
|
11
|
+
# robin_options debounce: 30
|
|
12
|
+
# end
|
|
13
|
+
#
|
|
14
|
+
# 10.times { ReindexDocument.perform_async(doc.id) } # runs once, ~30s later
|
|
15
|
+
#
|
|
16
|
+
# Each push schedules the job and atomically removes the previously
|
|
17
|
+
# scheduled one, in the round trip that sends it. If two pushes race, both may land in the schedule, but
|
|
18
|
+
# ServerMiddleware only runs the job recorded as the latest push and
|
|
19
|
+
# skips any superseded one, so the job still runs once.
|
|
20
|
+
module Debounce
|
|
21
|
+
# Records this push as the latest and removes the previous scheduled
|
|
22
|
+
# job. KEYS: debounce key, schedule set. ARGV: "<at (its score)>|<jid>",
|
|
23
|
+
# expiry (epoch ms).
|
|
24
|
+
CLAIM = Redis::Script.new(<<~LUA)
|
|
25
|
+
local prev = redis.call('GET', KEYS[1])
|
|
26
|
+
if prev then
|
|
27
|
+
local sep = string.find(prev, '|', 1, true)
|
|
28
|
+
local at, jid = string.sub(prev, 1, sep - 1), string.sub(prev, sep + 1)
|
|
29
|
+
local needle = '"jid":"' .. jid .. '"'
|
|
30
|
+
for _, job in ipairs(redis.call('ZRANGE', KEYS[2], at, at, 'BYSCORE')) do
|
|
31
|
+
if string.find(job, needle, 1, true) then redis.call('ZREM', KEYS[2], job) end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
redis.call('SET', KEYS[1], ARGV[1], 'PXAT', ARGV[2])
|
|
35
|
+
return 1
|
|
36
|
+
LUA
|
|
37
|
+
|
|
38
|
+
# A push's claim waits on its payload (as an instance variable, which
|
|
39
|
+
# JSON leaves out) until the push sends it: see Debounce.queue_claims.
|
|
40
|
+
CLAIM_IVAR = :@robin_debounce_claim
|
|
41
|
+
|
|
42
|
+
# Queues the claims of +payloads+ on +pipe+, ahead of the jobs, so
|
|
43
|
+
# each is made in the round trip that pushes its job, and only if it
|
|
44
|
+
# is pushed. A bulk push's claims are made in order before its jobs
|
|
45
|
+
# are sent; nothing between a claim and the next job's reads them, so
|
|
46
|
+
# the outcome is the same.
|
|
47
|
+
def self.queue_claims(pipe, payloads)
|
|
48
|
+
payloads.each do |payload|
|
|
49
|
+
next unless payload.key?("debounce_key")
|
|
50
|
+
|
|
51
|
+
keys, argv = payload.instance_variable_get(CLAIM_IVAR)
|
|
52
|
+
pipe.evalsha(CLAIM, keys, argv) if keys
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Moves the claim of +from+, a payload as a client middleware got it,
|
|
57
|
+
# to +to+, the payload the middleware chain returned instead.
|
|
58
|
+
def self.carry(from, to)
|
|
59
|
+
return if to.nil? || !from.instance_variable_defined?(CLAIM_IVAR)
|
|
60
|
+
|
|
61
|
+
to.instance_variable_set(CLAIM_IVAR, from.instance_variable_get(CLAIM_IVAR))
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
class ClientMiddleware
|
|
65
|
+
include Middleware::ConfigAware
|
|
66
|
+
include Middleware::Stateless
|
|
67
|
+
|
|
68
|
+
def self.applies?(_job_class, payload, _queue, _redis) = payload.key?("debounce")
|
|
69
|
+
|
|
70
|
+
def call(job_class, payload, queue, _redis)
|
|
71
|
+
window = payload["debounce"]
|
|
72
|
+
return yield unless window
|
|
73
|
+
|
|
74
|
+
at = [payload["at"], Robin.epoch_seconds(Robin.now_ms + Integer(window * 1000))].compact.max
|
|
75
|
+
payload["at"] = at
|
|
76
|
+
digest = Digest::SHA256.hexdigest(Robin.dump_json([job_class, queue, Unique.canonical(payload.fetch("args"))]))
|
|
77
|
+
# The run time, as the schedule's score, finds the job again.
|
|
78
|
+
token = "#{at}|#{payload.fetch("jid")}"
|
|
79
|
+
# Keep the claim ten minutes past the run time.
|
|
80
|
+
payload.instance_variable_set(CLAIM_IVAR,
|
|
81
|
+
[[config.key("debounce", digest), config.key("schedule")], [token, Robin.epoch_ms(at) + 600_000]])
|
|
82
|
+
payload["debounce_key"] = digest
|
|
83
|
+
yield
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Skips debounced jobs superseded by a later push.
|
|
88
|
+
class ServerMiddleware
|
|
89
|
+
include Middleware::ConfigAware
|
|
90
|
+
include Middleware::Stateless
|
|
91
|
+
|
|
92
|
+
def self.applies?(_job, payload, _queue) = payload.key?("debounce_key")
|
|
93
|
+
|
|
94
|
+
def call(_job, payload, _queue)
|
|
95
|
+
digest = payload["debounce_key"]
|
|
96
|
+
return yield unless digest
|
|
97
|
+
|
|
98
|
+
latest = redis { |c| c.call("GET", config.key("debounce", digest)) }
|
|
99
|
+
if latest && !latest.split("|", 2).last.eql?(payload.fetch("jid"))
|
|
100
|
+
logger.info { "skipping #{payload.fetch("class")} jid=#{payload.fetch("jid")}: superseded by a later push" }
|
|
101
|
+
Events.emit(config, "job.skipped", job: payload, reason: "superseded")
|
|
102
|
+
return
|
|
103
|
+
end
|
|
104
|
+
yield
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
data/lib/robin/deploy.rb
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest/sha2"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Deploy markers: the dashboard's metrics charts draw a line at each, so
|
|
7
|
+
# a change in throughput or failures can be matched to the deploy that
|
|
8
|
+
# caused it.
|
|
9
|
+
#
|
|
10
|
+
# Robin::Deploy.mark!("v42 #{`git rev-parse --short HEAD`.strip}")
|
|
11
|
+
#
|
|
12
|
+
# or `bundle exec robin deploy "v42"` from a deploy script. Every process
|
|
13
|
+
# may mark the same deploy (e.g. at boot): a label marked in the last
|
|
14
|
+
# hour isn't marked again. Markers are kept as long as hourly metrics
|
|
15
|
+
# (Config#metrics_retention_days).
|
|
16
|
+
module Deploy
|
|
17
|
+
DEDUP_SECONDS = 3600
|
|
18
|
+
MAX_LABEL = 200
|
|
19
|
+
|
|
20
|
+
# Adds a deploy mark, unless its label was marked in the last hour,
|
|
21
|
+
# and drops marks older than the retention. Returns 1, or 0 when the
|
|
22
|
+
# label was marked already.
|
|
23
|
+
#
|
|
24
|
+
# KEYS: the marks' sorted set, the label's dedup key (none for no
|
|
25
|
+
# label). ARGV: when (epoch ms), the mark, the oldest time kept (ms),
|
|
26
|
+
# the dedup TTL (seconds).
|
|
27
|
+
MARK = Redis::Script.new(<<~LUA)
|
|
28
|
+
if KEYS[2] and not redis.call('SET', KEYS[2], ARGV[1], 'NX', 'EX', ARGV[4]) then return 0 end
|
|
29
|
+
redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2])
|
|
30
|
+
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', '(' .. ARGV[3])
|
|
31
|
+
return 1
|
|
32
|
+
LUA
|
|
33
|
+
|
|
34
|
+
class << self
|
|
35
|
+
# Records a deploy at +at+, in one round trip. Returns false when
|
|
36
|
+
# +label+ was already marked in the last hour, true otherwise.
|
|
37
|
+
def mark!(label = nil, at: Time.now, config: Robin.config)
|
|
38
|
+
label = label&.to_s&.[](0, MAX_LABEL)
|
|
39
|
+
at_ms = ms(at)
|
|
40
|
+
retention_ms = config.metrics_retention_days * 86_400_000
|
|
41
|
+
keys = [key(config), *(dedup_key(config, label) if label)]
|
|
42
|
+
mark = Robin.dump_json({"at" => at_ms, "label" => label})
|
|
43
|
+
# EVAL, not EVALSHA: the connection may be another library's (a
|
|
44
|
+
# compatibility layer's, say), and deploys are rare.
|
|
45
|
+
config.redis { |c| c.call("EVAL", MARK.source, keys.size, *keys, at_ms, mark, at_ms - retention_ms, DEDUP_SECONDS) } == 1
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Deploys from +from+ to +to+ (Times), oldest first: hashes with
|
|
49
|
+
# "at" (epoch ms) and "label" (or nil).
|
|
50
|
+
def list(from:, to: Time.now, config: Robin.config)
|
|
51
|
+
range = [ms(from), ms(to)]
|
|
52
|
+
config.redis { |c| c.call("ZRANGE", key(config), *range, "BYSCORE") }.map { |raw| Robin.load_json(raw) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def key(config) = config.key("deploys")
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# mutant:disable -- to_i, to_int and Integer() agree on Rationals.
|
|
60
|
+
def ms(time) = (time.to_r * 1000).to_i
|
|
61
|
+
|
|
62
|
+
def dedup_key(config, label) = config.key("deploy", Digest::SHA256.hexdigest(label))
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Runs a Robin server inside another process. With Puma, use the plugin
|
|
5
|
+
# (lib/puma/plugin/robin.rb), which starts a server in every worker once
|
|
6
|
+
# the application has loaded and stops it when the worker stops:
|
|
7
|
+
#
|
|
8
|
+
# # config/puma.rb
|
|
9
|
+
# plugin :robin
|
|
10
|
+
#
|
|
11
|
+
# Elsewhere:
|
|
12
|
+
#
|
|
13
|
+
# robin = Robin.configure_embed do |config|
|
|
14
|
+
# config.queues = %w[critical default]
|
|
15
|
+
# config.concurrency = 2
|
|
16
|
+
# end
|
|
17
|
+
# robin.run
|
|
18
|
+
# at_exit { robin.stop }
|
|
19
|
+
class Embedded
|
|
20
|
+
attr_reader :config, :launcher
|
|
21
|
+
|
|
22
|
+
def initialize(config)
|
|
23
|
+
@config = config
|
|
24
|
+
@stop_lock = Mutex.new
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def configure
|
|
28
|
+
yield @config if block_given?
|
|
29
|
+
self
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def run
|
|
33
|
+
Robin.server!
|
|
34
|
+
@config.redis = @config.redis_options # resize the pool for server use
|
|
35
|
+
@launcher = Launcher.new(@config, embedded: true)
|
|
36
|
+
# Stops asked for from the dashboard go through #stop too, so a later
|
|
37
|
+
# #stop (e.g. the host process exiting) waits for them to finish.
|
|
38
|
+
@launcher.on_stop_request = -> { Thread.new { stop } }
|
|
39
|
+
@launcher.run
|
|
40
|
+
self
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def quiet = @launcher&.quiet
|
|
44
|
+
|
|
45
|
+
# Stops the server gracefully and returns once it has stopped. Calling
|
|
46
|
+
# it again (from any thread) waits for the first stop and does nothing.
|
|
47
|
+
def stop
|
|
48
|
+
@stop_lock.synchronize do
|
|
49
|
+
@launcher.stop unless @launcher.nil? || @launcher.stopping?
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "openssl"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Encrypts job arguments at rest with AES-256-GCM.
|
|
7
|
+
#
|
|
8
|
+
# Robin.configure do |config|
|
|
9
|
+
# config.encryption_keys = {1 => ENV["ROBIN_KEY_V1"], 2 => ENV["ROBIN_KEY_V2"]}
|
|
10
|
+
# end
|
|
11
|
+
#
|
|
12
|
+
# class ChargeCard
|
|
13
|
+
# include Robin::Job
|
|
14
|
+
# robin_options encrypt: true
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# With +encrypt: "last"+ only the last argument is encrypted (it should
|
|
18
|
+
# hold the secrets, e.g. a Hash of them); the others stay readable, in
|
|
19
|
+
# the dashboard too.
|
|
20
|
+
#
|
|
21
|
+
# The highest key version encrypts; every listed version can decrypt, so
|
|
22
|
+
# keys can be rotated by adding a new version and removing the old one
|
|
23
|
+
# once older jobs have drained. Keys are 32 bytes, given raw, as 64 hex
|
|
24
|
+
# characters, or base64. The jid is authenticated along with the
|
|
25
|
+
# arguments, so ciphertext can't be moved to another job. Retries and the
|
|
26
|
+
# dead set only ever store ciphertext.
|
|
27
|
+
module Encryption
|
|
28
|
+
class Error < Robin::Error; end
|
|
29
|
+
|
|
30
|
+
CIPHER = "aes-256-gcm"
|
|
31
|
+
IV_BYTES = 12
|
|
32
|
+
TAG_BYTES = 16
|
|
33
|
+
|
|
34
|
+
class << self
|
|
35
|
+
def keys(config)
|
|
36
|
+
keys = config.encryption_keys
|
|
37
|
+
raise Error, "encrypted jobs need config.encryption_keys (e.g. {1 => <32-byte key>})" if keys.nil? || keys.empty?
|
|
38
|
+
|
|
39
|
+
keys.to_h { |version, key| [Integer(version), normalize_key(key)] }
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def normalize_key(key)
|
|
43
|
+
key = key.to_s
|
|
44
|
+
return key.b if key.bytesize == 32
|
|
45
|
+
return [key].pack("H*") if key.match?(/\A\h{64}\z/)
|
|
46
|
+
|
|
47
|
+
# (Base64 unpacks to a String, which Steep can't tell from the format.)
|
|
48
|
+
decoded = begin
|
|
49
|
+
key.unpack1("m0")
|
|
50
|
+
rescue
|
|
51
|
+
""
|
|
52
|
+
end #: String
|
|
53
|
+
return decoded if decoded.bytesize == 32
|
|
54
|
+
|
|
55
|
+
raise Error, "encryption keys must be 32 bytes (raw, 64 hex chars, or base64)"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Returns [version, ciphertext].
|
|
59
|
+
def encrypt(config, args, aad)
|
|
60
|
+
all = keys(config)
|
|
61
|
+
version = all.keys.max
|
|
62
|
+
cipher = cipher(:encrypt, all.fetch(version))
|
|
63
|
+
iv = cipher.random_iv
|
|
64
|
+
cipher.auth_data = aad
|
|
65
|
+
data = cipher.update(Robin.dump_json(args)) + cipher.final
|
|
66
|
+
[version, [iv + cipher.auth_tag + data].pack("m0")]
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def decrypt(config, version, blob, aad)
|
|
70
|
+
key = keys(config).fetch(version) { raise Error, "no encryption key for version #{version}" }
|
|
71
|
+
raw = blob.unpack1("m0") #: String
|
|
72
|
+
cipher = cipher(:decrypt, key)
|
|
73
|
+
# (A slice past the end of a blob cut short is nil, not a String:
|
|
74
|
+
# the TypeError that raises is rescued below.)
|
|
75
|
+
iv = raw.byteslice(0, IV_BYTES) #: String
|
|
76
|
+
tag = raw.byteslice(IV_BYTES, TAG_BYTES) #: String
|
|
77
|
+
data = raw.byteslice((IV_BYTES + TAG_BYTES)..) #: String
|
|
78
|
+
cipher.iv = iv
|
|
79
|
+
cipher.auth_tag = tag
|
|
80
|
+
cipher.auth_data = aad
|
|
81
|
+
Robin.load_json(cipher.update(data) + cipher.final)
|
|
82
|
+
rescue OpenSSL::Cipher::CipherError, ArgumentError, TypeError
|
|
83
|
+
raise Error, "could not decrypt job arguments (wrong key or tampered payload)"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Whether only the job's last argument is encrypted.
|
|
87
|
+
def last_only?(payload) = payload["encrypt"].to_s == "last"
|
|
88
|
+
|
|
89
|
+
# An encrypted job's arguments as they may be shown: the encrypted
|
|
90
|
+
# ones hidden.
|
|
91
|
+
def visible_args(payload)
|
|
92
|
+
last_only?(payload) ? [*payload.fetch("args")[...-1], "[encrypted]"] : ["[encrypted]"]
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def cipher(mode, key)
|
|
96
|
+
cipher = OpenSSL::Cipher.new(CIPHER)
|
|
97
|
+
cipher.public_send(mode)
|
|
98
|
+
cipher.key = key
|
|
99
|
+
cipher
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
class ClientMiddleware
|
|
104
|
+
include Middleware::ConfigAware
|
|
105
|
+
include Middleware::Stateless
|
|
106
|
+
|
|
107
|
+
def self.applies?(_job_class, payload, _queue, _redis) = payload.key?("encrypt")
|
|
108
|
+
|
|
109
|
+
def call(_job_class, payload, _queue, _redis)
|
|
110
|
+
if payload["encrypt"] && !payload["encrypted"]
|
|
111
|
+
*clear, secret = args = payload.fetch("args")
|
|
112
|
+
last = Encryption.last_only?(payload)
|
|
113
|
+
version, blob = Encryption.encrypt(config, last ? secret : args, payload.fetch("jid"))
|
|
114
|
+
payload["args"] = last ? [*clear, blob] : [blob]
|
|
115
|
+
payload["encrypted"] = version
|
|
116
|
+
end
|
|
117
|
+
yield
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
class ServerMiddleware
|
|
122
|
+
include Middleware::ConfigAware
|
|
123
|
+
include Middleware::Stateless
|
|
124
|
+
|
|
125
|
+
def self.applies?(_job, payload, _queue) = payload.key?("encrypted")
|
|
126
|
+
|
|
127
|
+
def call(_job, payload, _queue)
|
|
128
|
+
return yield unless payload["encrypted"]
|
|
129
|
+
|
|
130
|
+
original = payload.fetch("args")
|
|
131
|
+
version = payload.fetch("encrypted")
|
|
132
|
+
payload["args"] = if Encryption.last_only?(payload)
|
|
133
|
+
[*original[...-1], Encryption.decrypt(config, version, original.last, payload.fetch("jid"))]
|
|
134
|
+
else
|
|
135
|
+
Encryption.decrypt(config, version, original.first, payload.fetch("jid"))
|
|
136
|
+
end
|
|
137
|
+
begin
|
|
138
|
+
yield
|
|
139
|
+
ensure
|
|
140
|
+
# Jobs are written back from Robin.pristine, but code that looks
|
|
141
|
+
# at the payload afterwards shouldn't see plaintext either.
|
|
142
|
+
payload["args"] = original
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|