monkrb 0.21.1 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +94 -0
- data/README.md +4 -4
- data/lib/monk/auth/errors.rb +5 -0
- data/lib/monk/auth/sweep.rb +23 -0
- data/lib/monk/auth.rb +101 -6
- data/lib/monk/bin.rb +94 -0
- data/lib/monk/generators/auth.rb +1 -0
- data/lib/monk/generators/jobs.rb +3 -1
- data/lib/monk/jobs/adapters/pg.rb +90 -2
- data/lib/monk/jobs/cron.rb +180 -0
- data/lib/monk/jobs/errors.rb +21 -0
- data/lib/monk/jobs/firing.rb +11 -0
- data/lib/monk/jobs/recurring_task.rb +9 -0
- data/lib/monk/jobs/runtime.rb +38 -2
- data/lib/monk/jobs/scheduler.rb +91 -0
- data/lib/monk/jobs.rb +87 -8
- data/lib/monk/live/session.rb +16 -9
- data/lib/monk/templates/auth/agents.md +10 -3
- data/lib/monk/templates/auth/app/routes/auth.rb +4 -8
- data/lib/monk/templates/auth/config/auth.rb +1 -0
- data/lib/monk/templates/auth/db/migrate/00000000000004_create_login_requests.down.sql +2 -0
- data/lib/monk/templates/auth/db/migrate/00000000000004_create_login_requests.up.sql +16 -0
- data/lib/monk/templates/auth/setup.md +16 -3
- data/lib/monk/templates/auth/test/auth_test.rb +6 -0
- data/lib/monk/templates/base/Gemfile +1 -1
- data/lib/monk/templates/base/config/load.rb +5 -0
- data/lib/monk/templates/jobs/agents.md +15 -4
- data/lib/monk/templates/jobs/config/recurring.rb +21 -0
- data/lib/monk/templates/jobs/db/migrate/00000000000003_create_recurring_tables.down.sql +1 -0
- data/lib/monk/templates/jobs/db/migrate/00000000000003_create_recurring_tables.up.sql +13 -0
- data/lib/monk/templates/jobs/setup.md +20 -2
- data/lib/monk/templates/websocket/bin/websocket_server +14 -0
- data/lib/monk/templates/websocket/setup.md +2 -1
- data/lib/monk/version.rb +1 -1
- data/lib/monk/websocket/handshake.rb +32 -9
- data/lib/monk/websocket/pg_fanout.rb +16 -7
- data/lib/monk/websocket/redis_fanout.rb +16 -7
- data/lib/monk/websocket/registry.rb +40 -5
- data/lib/monk/websocket/server.rb +211 -47
- metadata +11 -1
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
require_relative "errors"
|
|
2
|
+
|
|
3
|
+
module Monk
|
|
4
|
+
module Jobs
|
|
5
|
+
# A recurring task's schedule (docs/history/plan-scheduled-jobs.md decision 3):
|
|
6
|
+
# five fields -- minute, hour, day of month, month, day of week -- each
|
|
7
|
+
# `*`, numbers, lists (`,`), ranges (`-`) and steps (`/`), plus the
|
|
8
|
+
# shortcuts in SHORTCUTS. No seconds, no names, no time zones: every
|
|
9
|
+
# time is UTC.
|
|
10
|
+
#
|
|
11
|
+
# When both day fields are restricted, a day matching either counts; a
|
|
12
|
+
# field starting with `*` (`*/2` too) doesn't restrict, as in Vixie
|
|
13
|
+
# cron. Day of week 7 is Sunday, like 0.
|
|
14
|
+
#
|
|
15
|
+
# Frozen Arrays of Integers all the way down, so a parsed expression is
|
|
16
|
+
# shareable with every Ractor.
|
|
17
|
+
Cron = Data.define(:expression, :minutes, :hours, :days, :months, :weekdays, :any_day, :any_weekday)
|
|
18
|
+
|
|
19
|
+
class Cron
|
|
20
|
+
SHORTCUTS = {
|
|
21
|
+
"@hourly" => "0 * * * *", "@daily" => "0 0 * * *", "@weekly" => "0 0 * * 0",
|
|
22
|
+
"@monthly" => "0 0 1 * *", "@yearly" => "0 0 1 1 *",
|
|
23
|
+
}.freeze
|
|
24
|
+
|
|
25
|
+
FIELDS = [["minute", 0..59], ["hour", 0..23], ["day of month", 1..31], ["month", 1..12],
|
|
26
|
+
["day of week", 0..7],].freeze
|
|
27
|
+
|
|
28
|
+
PART = %r{\A(\*|\d+(?:-\d+)?)(?:/(\d+))?\z}
|
|
29
|
+
|
|
30
|
+
# The longest month each month can have: February counts its 29th.
|
|
31
|
+
MONTH_DAYS = [nil, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31].freeze
|
|
32
|
+
|
|
33
|
+
MINUTE = 60
|
|
34
|
+
HOUR = 3600
|
|
35
|
+
DAY = 86_400
|
|
36
|
+
|
|
37
|
+
# Neither lookup can go further than this without having found an
|
|
38
|
+
# occurrence: February 29th is at most 8 years away.
|
|
39
|
+
MAX_YEARS = 9
|
|
40
|
+
|
|
41
|
+
def self.parse(expression)
|
|
42
|
+
unless expression.is_a?(String)
|
|
43
|
+
raise InvalidCronError, "a cron expression is a String, got #{expression.inspect}"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
normalized = expression.split.join(" ")
|
|
47
|
+
fields = shortcut(normalized).split
|
|
48
|
+
unless fields.size == 5
|
|
49
|
+
raise InvalidCronError,
|
|
50
|
+
"#{expression.inspect} has #{fields.size} field(s); a cron expression has 5 fields " \
|
|
51
|
+
"(minute hour day-of-month month day-of-week) or is one of #{SHORTCUTS.keys.join(", ")}"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
minutes, hours, days, months, weekdays = fields.zip(FIELDS).map do |text, (name, range)|
|
|
55
|
+
parse_field(text, name, range, expression)
|
|
56
|
+
end
|
|
57
|
+
cron = new(
|
|
58
|
+
expression: normalized.freeze, minutes:, hours:, days:, months:,
|
|
59
|
+
weekdays: weekdays.map { |day| day % 7 }.uniq.sort.freeze,
|
|
60
|
+
any_day: fields[2].start_with?("*"), any_weekday: fields[4].start_with?("*"),
|
|
61
|
+
)
|
|
62
|
+
cron.check_matches!
|
|
63
|
+
Ractor.make_shareable(cron)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def self.shortcut(normalized)
|
|
67
|
+
return normalized unless normalized.start_with?("@")
|
|
68
|
+
|
|
69
|
+
SHORTCUTS.fetch(normalized) do
|
|
70
|
+
raise InvalidCronError, "unknown cron shortcut #{normalized.inspect}; known: #{SHORTCUTS.keys.join(", ")}"
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def self.parse_field(text, name, range, expression)
|
|
75
|
+
invalid = ->(problem) { raise InvalidCronError, "#{expression.inspect}: #{name} #{problem}" }
|
|
76
|
+
|
|
77
|
+
text.split(",", -1).flat_map do |part|
|
|
78
|
+
match = PART.match(part) || invalid.call("#{part.inspect} isn't a number, range or step")
|
|
79
|
+
base, step = match.captures
|
|
80
|
+
invalid.call("#{part.inspect}: a step goes on * or a range (a-b/n)") if step && !base.match?(/[*-]/)
|
|
81
|
+
invalid.call("#{part.inspect}: the step is 0") if step&.to_i&.zero?
|
|
82
|
+
|
|
83
|
+
from, to = base == "*" ? range.minmax : base.split("-").map(&:to_i).then { |a, b| [a, b || a] }
|
|
84
|
+
[from, to].reject { |value| range.cover?(value) }.each do |value|
|
|
85
|
+
invalid.call("#{value} is out of range (#{range.min}-#{range.max})")
|
|
86
|
+
end
|
|
87
|
+
invalid.call("range #{part.inspect} runs backwards") if from > to
|
|
88
|
+
|
|
89
|
+
(from..to).step(step ? step.to_i : 1).to_a
|
|
90
|
+
end.uniq.sort.freeze
|
|
91
|
+
end
|
|
92
|
+
private_class_method :shortcut, :parse_field
|
|
93
|
+
|
|
94
|
+
def to_s = expression
|
|
95
|
+
|
|
96
|
+
# The latest occurrence at or before `time`, as a UTC Time on a whole
|
|
97
|
+
# minute. Steps field by field, so a yearly expression takes a handful
|
|
98
|
+
# of steps, not a year of minutes.
|
|
99
|
+
def previous(time)
|
|
100
|
+
time = minute_of(time)
|
|
101
|
+
limit = time.year - MAX_YEARS
|
|
102
|
+
loop do
|
|
103
|
+
raise InvalidCronError, "#{expression.inspect}: no occurrence found" if time.year < limit
|
|
104
|
+
|
|
105
|
+
time =
|
|
106
|
+
if !months.include?(time.month)
|
|
107
|
+
month = months.rfind { |m| m < time.month }
|
|
108
|
+
month ? end_of_month(time.year, month) : end_of_month(time.year - 1, months.last)
|
|
109
|
+
elsif !day?(time)
|
|
110
|
+
day_of(time) - MINUTE
|
|
111
|
+
elsif !hours.include?(time.hour)
|
|
112
|
+
hour = hours.rfind { |h| h < time.hour }
|
|
113
|
+
hour ? day_of(time) + (hour * HOUR) + (59 * MINUTE) : day_of(time) - MINUTE
|
|
114
|
+
elsif !minutes.include?(time.min)
|
|
115
|
+
minute = minutes.rfind { |m| m < time.min }
|
|
116
|
+
minute ? hour_of(time) + (minute * MINUTE) : hour_of(time) - MINUTE
|
|
117
|
+
else
|
|
118
|
+
return time
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# The first occurrence after `time`, as a UTC Time on a whole minute.
|
|
124
|
+
def next_after(time)
|
|
125
|
+
time = minute_of(time) + MINUTE
|
|
126
|
+
limit = time.year + MAX_YEARS
|
|
127
|
+
loop do
|
|
128
|
+
raise InvalidCronError, "#{expression.inspect}: no occurrence found" if time.year > limit
|
|
129
|
+
|
|
130
|
+
time =
|
|
131
|
+
if !months.include?(time.month)
|
|
132
|
+
month = months.find { |m| m > time.month }
|
|
133
|
+
month ? Time.utc(time.year, month) : Time.utc(time.year + 1, months.first)
|
|
134
|
+
elsif !day?(time)
|
|
135
|
+
day_of(time) + DAY
|
|
136
|
+
elsif !hours.include?(time.hour)
|
|
137
|
+
hour = hours.find { |h| h > time.hour }
|
|
138
|
+
hour ? day_of(time) + (hour * HOUR) : day_of(time) + DAY
|
|
139
|
+
elsif !minutes.include?(time.min)
|
|
140
|
+
minute = minutes.find { |m| m > time.min }
|
|
141
|
+
minute ? hour_of(time) + (minute * MINUTE) : hour_of(time) + HOUR
|
|
142
|
+
else
|
|
143
|
+
return time
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# A day of month on its own (day of week *) must exist in one of the
|
|
149
|
+
# months, or the expression never matches: `0 0 30 2 *`. With day of
|
|
150
|
+
# week restricted, some week always has a matching day.
|
|
151
|
+
def check_matches!
|
|
152
|
+
return unless any_weekday && !any_day
|
|
153
|
+
return if months.any? { |month| days.first <= MONTH_DAYS[month] }
|
|
154
|
+
|
|
155
|
+
raise InvalidCronError, "#{expression.inspect} never matches: none of its months has day #{days.first}"
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
private
|
|
159
|
+
|
|
160
|
+
def minute_of(time)
|
|
161
|
+
utc = time.getutc
|
|
162
|
+
Time.utc(utc.year, utc.month, utc.day, utc.hour, utc.min)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def day_of(time) = Time.utc(time.year, time.month, time.day)
|
|
166
|
+
|
|
167
|
+
def hour_of(time) = Time.utc(time.year, time.month, time.day, time.hour)
|
|
168
|
+
|
|
169
|
+
def end_of_month(year, month)
|
|
170
|
+
(month == 12 ? Time.utc(year + 1, 1, 1) : Time.utc(year, month + 1, 1)) - MINUTE
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def day?(time)
|
|
174
|
+
in_month = days.include?(time.day)
|
|
175
|
+
in_week = weekdays.include?(time.wday)
|
|
176
|
+
any_day || any_weekday ? in_month && in_week : in_month || in_week
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
data/lib/monk/jobs/errors.rb
CHANGED
|
@@ -6,6 +6,17 @@ module Monk
|
|
|
6
6
|
class NotFrozenError < StandardError
|
|
7
7
|
end
|
|
8
8
|
|
|
9
|
+
# Declaring a recurring task after Monk.freeze! has sealed the
|
|
10
|
+
# registry: it would never be seen by the scheduler.
|
|
11
|
+
class RegistryFrozenError < StandardError
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# bin/jobs starting with recurring tasks declared but no
|
|
15
|
+
# monk_recurring_tasks table: an app that has jobs from before 0.22
|
|
16
|
+
# and hasn't added the migration yet.
|
|
17
|
+
class MissingRecurringTableError < StandardError
|
|
18
|
+
end
|
|
19
|
+
|
|
9
20
|
# A job_class name that isn't a Monk::Job subclass this process knows
|
|
10
21
|
# about: a job enqueued by a newer deploy, a renamed class, or a name
|
|
11
22
|
# that was never a job at all.
|
|
@@ -21,9 +32,19 @@ module Monk
|
|
|
21
32
|
class ClearOutsideTestsError < StandardError
|
|
22
33
|
end
|
|
23
34
|
|
|
35
|
+
# Monk::Jobs.run_scheduler! outside MONK_ENV=test: it would fire a real
|
|
36
|
+
# app's schedules at a made-up time.
|
|
37
|
+
class RunSchedulerOutsideTestsError < StandardError
|
|
38
|
+
end
|
|
39
|
+
|
|
24
40
|
# An ArgumentError, like Monk::Mail::InvalidMessageError: it's always
|
|
25
41
|
# the caller's input that's wrong.
|
|
26
42
|
class InvalidArgumentsError < ArgumentError
|
|
27
43
|
end
|
|
44
|
+
|
|
45
|
+
# A recurring task's cron expression that doesn't parse, or never
|
|
46
|
+
# matches a minute. Raised when the task is declared, at boot.
|
|
47
|
+
class InvalidCronError < ArgumentError
|
|
48
|
+
end
|
|
28
49
|
end
|
|
29
50
|
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
module Monk
|
|
2
|
+
module Jobs
|
|
3
|
+
# What one fire_recurring call did for a recurring task's occurrence
|
|
4
|
+
# (docs/history/plan-scheduled-jobs.md decision 5): result is :initialized,
|
|
5
|
+
# :enqueued, :skipped_pending, :already_handled, :not_due or
|
|
6
|
+
# :reinitialized; job_id is the enqueued job's, when there is one;
|
|
7
|
+
# previous_checkpoint is the task's checkpoint before the call (nil
|
|
8
|
+
# for a new task), for the "skipped N occurrences" log line.
|
|
9
|
+
Firing = Data.define(:result, :job_id, :previous_checkpoint)
|
|
10
|
+
end
|
|
11
|
+
end
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
module Monk
|
|
2
|
+
module Jobs
|
|
3
|
+
# One declared schedule (Monk::Jobs.recurring): the key it's stored
|
|
4
|
+
# under in monk_recurring_tasks, the job class to enqueue, its parsed
|
|
5
|
+
# Cron and the args to enqueue with. Queue, priority and max_attempts
|
|
6
|
+
# come from the job class, as with any enqueue.
|
|
7
|
+
RecurringTask = Data.define(:key, :job_class, :cron, :args, :skip_if_pending)
|
|
8
|
+
end
|
|
9
|
+
end
|
data/lib/monk/jobs/runtime.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
require "socket"
|
|
2
2
|
require_relative "../jobs"
|
|
3
3
|
require_relative "worker"
|
|
4
|
+
require_relative "scheduler"
|
|
4
5
|
|
|
5
6
|
module Monk
|
|
6
7
|
module Jobs
|
|
@@ -53,6 +54,9 @@ module Monk
|
|
|
53
54
|
def run(trap_signals: true)
|
|
54
55
|
Monk.freeze!
|
|
55
56
|
@adapter = Monk::Jobs.adapter
|
|
57
|
+
tasks = Monk::Jobs.recurring_tasks
|
|
58
|
+
check_recurring_table!(tasks)
|
|
59
|
+
@scheduler = Scheduler.new(@adapter, tasks)
|
|
56
60
|
@hostname = Socket.gethostname
|
|
57
61
|
@pid = Process.pid
|
|
58
62
|
@process_id = @adapter.register_process(hostname: @hostname, pid: @pid)
|
|
@@ -65,7 +69,8 @@ module Monk
|
|
|
65
69
|
@holding_port = Ractor::Port.new
|
|
66
70
|
@worker_count.times { |index| spawn_worker(index) }
|
|
67
71
|
Monk::Log.info(
|
|
68
|
-
"Monk::Jobs process #{@process_id} started: #{@worker_count} workers on #{@queues.join(", ")}
|
|
72
|
+
"Monk::Jobs process #{@process_id} started: #{@worker_count} workers on #{@queues.join(", ")}, " \
|
|
73
|
+
"#{tasks.size} recurring task#{"s" unless tasks.size == 1} (pid #{@pid})",
|
|
69
74
|
)
|
|
70
75
|
|
|
71
76
|
ticker, timer = start_ticker
|
|
@@ -104,10 +109,19 @@ module Monk
|
|
|
104
109
|
end
|
|
105
110
|
end
|
|
106
111
|
|
|
112
|
+
# The scheduler runs after the rest and under its own rescue, so an
|
|
113
|
+
# error that only it hits can't stop the heartbeat: other processes
|
|
114
|
+
# would take this one for dead.
|
|
115
|
+
def tick(last_heartbeat)
|
|
116
|
+
last_heartbeat = housekeeping(last_heartbeat)
|
|
117
|
+
schedule
|
|
118
|
+
last_heartbeat
|
|
119
|
+
end
|
|
120
|
+
|
|
107
121
|
# A database error here (Postgres restarting, a dropped connection)
|
|
108
122
|
# mustn't end the process: it's logged, this Ractor's connection is
|
|
109
123
|
# reset, and the next tick tries again.
|
|
110
|
-
def
|
|
124
|
+
def housekeeping(last_heartbeat)
|
|
111
125
|
release_dead_workers_jobs
|
|
112
126
|
@adapter.stage_due
|
|
113
127
|
return last_heartbeat if monotonic - last_heartbeat < @heartbeat_interval
|
|
@@ -120,6 +134,28 @@ module Monk
|
|
|
120
134
|
last_heartbeat
|
|
121
135
|
end
|
|
122
136
|
|
|
137
|
+
# Recurring tasks (docs/history/plan-scheduled-jobs.md decision 1). An
|
|
138
|
+
# occurrence that errors isn't marked handled, so the next tick asks
|
|
139
|
+
# again.
|
|
140
|
+
def schedule
|
|
141
|
+
@scheduler.run
|
|
142
|
+
rescue StandardError => e
|
|
143
|
+
Monk::Log.error("Monk::Jobs scheduler: #{Monk::Jobs.describe_error(e)}; reconnecting")
|
|
144
|
+
reconnect
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Decision 9: a missing table is a deploy mistake, said once here
|
|
148
|
+
# rather than every second by the scheduler.
|
|
149
|
+
def check_recurring_table!(tasks)
|
|
150
|
+
return if tasks.empty? || @adapter.recurring_table?
|
|
151
|
+
|
|
152
|
+
raise MissingRecurringTableError,
|
|
153
|
+
"#{tasks.size} recurring task(s) declared (#{tasks.map(&:key).join(", ")}) but the database has no " \
|
|
154
|
+
"monk_recurring_tasks table. Add Monk's migration under a new timestamp in db/migrate/ " \
|
|
155
|
+
"(monk help jobs --file db/migrate/x_create_recurring_tables.up.sql, and .down.sql), then bin/migrate. " \
|
|
156
|
+
"The Monk 0.22 upgrade guide has the full steps"
|
|
157
|
+
end
|
|
158
|
+
|
|
123
159
|
# A job stays on the list until its release has gone through (or found
|
|
124
160
|
# the job no longer held), so a database error just retries it next tick.
|
|
125
161
|
def release_dead_workers_jobs
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require_relative "../log"
|
|
3
|
+
|
|
4
|
+
module Monk
|
|
5
|
+
module Jobs
|
|
6
|
+
# The supervisor's per-tick step for recurring tasks
|
|
7
|
+
# (docs/history/plan-scheduled-jobs.md decisions 5-7): asks the database about
|
|
8
|
+
# each task's most recent occurrence, which enqueues it at most once
|
|
9
|
+
# across every job process. Lives in the supervisor and isn't shared:
|
|
10
|
+
# the occurrence each task last handled is this process's own memory,
|
|
11
|
+
# so a tick reaches the database once per occurrence, not once a second.
|
|
12
|
+
class Scheduler
|
|
13
|
+
# Past this many, a catch-up log line stops counting.
|
|
14
|
+
MAX_SKIPPED_COUNT = 10_000
|
|
15
|
+
|
|
16
|
+
def initialize(adapter, tasks)
|
|
17
|
+
@adapter = adapter
|
|
18
|
+
@tasks = tasks
|
|
19
|
+
@handled = {} # task key => the occurrence last handled here
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# One pass as if the clock read `now`. clock: replaces the database
|
|
23
|
+
# clock too (Monk::Jobs.run_scheduler!); without it, the database
|
|
24
|
+
# decides whether an occurrence has arrived. Returns each task asked
|
|
25
|
+
# about this pass, key => Firing#result.
|
|
26
|
+
def run(now = Time.now, clock: nil)
|
|
27
|
+
@tasks.each_with_object({}) do |task, results|
|
|
28
|
+
occurrence = task.cron.previous(now)
|
|
29
|
+
next if @handled[task.key] == occurrence
|
|
30
|
+
|
|
31
|
+
firing = fire(task, occurrence, clock)
|
|
32
|
+
@handled[task.key] = occurrence unless firing.result == :not_due
|
|
33
|
+
report(task, occurrence, firing)
|
|
34
|
+
results[task.key] = firing.result
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def fire(task, occurrence, clock)
|
|
41
|
+
job = task.job_class
|
|
42
|
+
@adapter.fire_recurring(
|
|
43
|
+
key: task.key, cron: task.cron.expression, occurrence: occurrence, skip_if_pending: task.skip_if_pending,
|
|
44
|
+
job_class: job.name, queue: job.queue, priority: job.priority, max_attempts: job.max_attempts,
|
|
45
|
+
args: JSON.generate(task.args), clock: clock,
|
|
46
|
+
)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def report(task, occurrence, firing)
|
|
50
|
+
prefix = "Monk::Jobs recurring #{task.key}:"
|
|
51
|
+
case firing.result
|
|
52
|
+
when :initialized
|
|
53
|
+
Monk::Log.info("#{prefix} new task, first run at #{stamp(task.cron.next_after(occurrence))}")
|
|
54
|
+
when :reinitialized
|
|
55
|
+
Monk::Log.info("#{prefix} cron changed to #{task.cron.expression.inspect}, " \
|
|
56
|
+
"first run at #{stamp(task.cron.next_after(occurrence))}")
|
|
57
|
+
when :enqueued
|
|
58
|
+
Monk::Log.info("#{prefix} enqueued job #{firing.job_id} for #{stamp(occurrence)}")
|
|
59
|
+
when :skipped_pending
|
|
60
|
+
Monk::Log.info("#{prefix} skipped #{stamp(occurrence)}, the previous run is still pending")
|
|
61
|
+
end
|
|
62
|
+
report_catch_up(task, occurrence, firing, prefix)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Decision 6: after downtime only the latest occurrence fires.
|
|
66
|
+
def report_catch_up(task, occurrence, firing, prefix)
|
|
67
|
+
return unless %i[enqueued skipped_pending].include?(firing.result) && firing.previous_checkpoint
|
|
68
|
+
|
|
69
|
+
skipped = skipped_between(task.cron, firing.previous_checkpoint, occurrence)
|
|
70
|
+
return if skipped.zero?
|
|
71
|
+
|
|
72
|
+
count = skipped >= MAX_SKIPPED_COUNT ? "#{MAX_SKIPPED_COUNT}+" : skipped
|
|
73
|
+
Monk::Log.warn(
|
|
74
|
+
"#{prefix} caught up from #{stamp(firing.previous_checkpoint)}, skipped #{count} occurrences",
|
|
75
|
+
)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def skipped_between(cron, from, to)
|
|
79
|
+
count = 0
|
|
80
|
+
time = cron.next_after(from)
|
|
81
|
+
while time < to && count < MAX_SKIPPED_COUNT
|
|
82
|
+
count += 1
|
|
83
|
+
time = cron.next_after(time)
|
|
84
|
+
end
|
|
85
|
+
count
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def stamp(time) = time.utc.iso8601
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
data/lib/monk/jobs.rb
CHANGED
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
require_relative "../monk"
|
|
4
4
|
require_relative "jobs/errors"
|
|
5
5
|
require_relative "jobs/args"
|
|
6
|
+
require_relative "jobs/cron"
|
|
7
|
+
require_relative "jobs/recurring_task"
|
|
8
|
+
require_relative "jobs/scheduler"
|
|
6
9
|
require "json"
|
|
7
10
|
|
|
8
11
|
module Monk
|
|
@@ -53,7 +56,7 @@ module Monk
|
|
|
53
56
|
#
|
|
54
57
|
# SendReceipt.enqueue(order_id) is the same call.
|
|
55
58
|
def enqueue(job_class, *args, wait: nil, at: nil, conn: nil)
|
|
56
|
-
check_enqueueable!(job_class)
|
|
59
|
+
check_enqueueable!(job_class, "Monk::Jobs.enqueue")
|
|
57
60
|
Args.check!(args)
|
|
58
61
|
check_schedule!(wait, at)
|
|
59
62
|
|
|
@@ -63,6 +66,51 @@ module Monk
|
|
|
63
66
|
)
|
|
64
67
|
end
|
|
65
68
|
|
|
69
|
+
# A job enqueued on a schedule by bin/jobs, typically in
|
|
70
|
+
# config/recurring.rb (loaded after app/jobs/, so the class exists):
|
|
71
|
+
#
|
|
72
|
+
# Monk::Jobs.recurring "purge_sessions", PurgeExpiredSessions, cron: "0 3 * * *"
|
|
73
|
+
# Monk::Jobs.recurring "weekly_digest", SendDigest, cron: "0 9 * * 1",
|
|
74
|
+
# args: ["weekly"], skip_if_pending: true
|
|
75
|
+
#
|
|
76
|
+
# The key is the task's identity in monk_recurring_tasks, so renaming
|
|
77
|
+
# the job class keeps its place in the schedule. cron: is UTC (Cron).
|
|
78
|
+
# skip_if_pending: true skips an occurrence while the job enqueued for
|
|
79
|
+
# the previous one hasn't finished. Everything is checked here, so a
|
|
80
|
+
# mistake fails at boot (docs/history/plan-scheduled-jobs.md decision 2).
|
|
81
|
+
def recurring(key, job_class, cron:, args: [], skip_if_pending: false)
|
|
82
|
+
if @recurring_registry
|
|
83
|
+
raise RegistryFrozenError,
|
|
84
|
+
"Monk::Jobs.recurring(#{key.inspect}) after the job registry is frozen -- declare recurring " \
|
|
85
|
+
"tasks while the app loads (config/recurring.rb), before Monk.boot or bin/jobs starts"
|
|
86
|
+
end
|
|
87
|
+
unless key.is_a?(String) && !key.empty?
|
|
88
|
+
raise ArgumentError, "a recurring task's key must be a non-empty String, got #{key.inspect}"
|
|
89
|
+
end
|
|
90
|
+
if recurring_list.any? { |task| task.key == key }
|
|
91
|
+
raise ArgumentError, "recurring task #{key.inspect} is already declared"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
check_enqueueable!(job_class, "Monk::Jobs.recurring")
|
|
95
|
+
Args.check!(args)
|
|
96
|
+
unless [true, false].include?(skip_if_pending)
|
|
97
|
+
raise ArgumentError, "skip_if_pending: must be true or false, got #{skip_if_pending.inspect}"
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
task = RecurringTask.new(
|
|
101
|
+
key: key.dup.freeze, job_class: job_class, cron: Cron.parse(cron),
|
|
102
|
+
args: Ractor.make_shareable(Marshal.load(Marshal.dump(args))), skip_if_pending: skip_if_pending,
|
|
103
|
+
)
|
|
104
|
+
recurring_list << task
|
|
105
|
+
task
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The declared recurring tasks, in declaration order. Once the
|
|
109
|
+
# registry is frozen, a shareable Array any Ractor can read.
|
|
110
|
+
def recurring_tasks
|
|
111
|
+
@recurring_registry || recurring_list.dup.freeze
|
|
112
|
+
end
|
|
113
|
+
|
|
66
114
|
# Called by Monk::Job.inherited, in the main Ractor as the app's job
|
|
67
115
|
# files load. Only recorded here: names are resolved when the
|
|
68
116
|
# registry is frozen, so a class that gets its constant name after
|
|
@@ -94,6 +142,7 @@ module Monk
|
|
|
94
142
|
# without a name a job can never be found again once enqueued.
|
|
95
143
|
def freeze_registry!
|
|
96
144
|
@registry = Ractor.make_shareable(classes.filter_map { |job| [job.name, job] if job.name }.to_h)
|
|
145
|
+
@recurring_registry = Ractor.make_shareable(recurring_list.dup)
|
|
97
146
|
end
|
|
98
147
|
|
|
99
148
|
# A failed job back to available, with its attempts reset; true if it
|
|
@@ -150,9 +199,32 @@ module Monk
|
|
|
150
199
|
ran
|
|
151
200
|
end
|
|
152
201
|
|
|
202
|
+
# Runs one scheduler pass as if the clock read `at`, for an app's
|
|
203
|
+
# tests of its recurring tasks; returns key => result (:initialized,
|
|
204
|
+
# :enqueued, ...). `at` stands in for the database clock as well, so
|
|
205
|
+
# it only runs with MONK_ENV=test. The first pass sets a task's
|
|
206
|
+
# checkpoint without enqueueing, as a deploy would:
|
|
207
|
+
#
|
|
208
|
+
# Monk::Jobs.run_scheduler!(at: Time.utc(2026, 10, 12, 8, 59))
|
|
209
|
+
# Monk::Jobs.run_scheduler!(at: Time.utc(2026, 10, 12, 9)) # => {"weekly_digest" => :enqueued}
|
|
210
|
+
# Monk::Jobs.drain!
|
|
211
|
+
#
|
|
212
|
+
# Calls Monk.freeze! first, as bin/jobs does.
|
|
213
|
+
def run_scheduler!(at: Time.now)
|
|
214
|
+
unless Monk.env.test?
|
|
215
|
+
raise RunSchedulerOutsideTestsError,
|
|
216
|
+
"Monk::Jobs.run_scheduler! fires schedules at a made-up time, so it only runs with MONK_ENV=test " \
|
|
217
|
+
"(this is #{Monk.env})"
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
Monk.freeze!
|
|
221
|
+
Scheduler.new(adapter, recurring_tasks).run(at, clock: at)
|
|
222
|
+
end
|
|
223
|
+
|
|
153
224
|
# Empties the queue: every job, failed ones included, and every job
|
|
154
|
-
# process row. For an app's
|
|
155
|
-
# and so it refuses to run
|
|
225
|
+
# process row, and every recurring task's checkpoint. For an app's
|
|
226
|
+
# tests, between one test and the next -- and so it refuses to run
|
|
227
|
+
# unless MONK_ENV is test.
|
|
156
228
|
def clear!
|
|
157
229
|
unless Monk.env.test?
|
|
158
230
|
raise ClearOutsideTestsError,
|
|
@@ -162,12 +234,14 @@ module Monk
|
|
|
162
234
|
adapter.clear!
|
|
163
235
|
end
|
|
164
236
|
|
|
165
|
-
# Test-only: unfreezes the registry and forgets the configuration
|
|
166
|
-
# Recorded classes stay recorded, the same way a
|
|
167
|
-
# undefined.
|
|
237
|
+
# Test-only: unfreezes the registry and forgets the configuration and
|
|
238
|
+
# the recurring tasks. Recorded classes stay recorded, the same way a
|
|
239
|
+
# Ruby class can't be undefined.
|
|
168
240
|
def reset!
|
|
169
241
|
@registry = nil
|
|
170
242
|
@adapter = nil
|
|
243
|
+
@recurring_list = nil
|
|
244
|
+
@recurring_registry = nil
|
|
171
245
|
end
|
|
172
246
|
|
|
173
247
|
private
|
|
@@ -199,9 +273,9 @@ module Monk
|
|
|
199
273
|
raise e.cause, cause: nil
|
|
200
274
|
end
|
|
201
275
|
|
|
202
|
-
def check_enqueueable!(job_class)
|
|
276
|
+
def check_enqueueable!(job_class, caller_name)
|
|
203
277
|
unless job_class.is_a?(Class) && job_class < Monk::Job
|
|
204
|
-
raise ArgumentError, "
|
|
278
|
+
raise ArgumentError, "#{caller_name} needs a Monk::Job subclass, got #{job_class.inspect}"
|
|
205
279
|
end
|
|
206
280
|
return if job_class.name
|
|
207
281
|
|
|
@@ -220,6 +294,10 @@ module Monk
|
|
|
220
294
|
def classes
|
|
221
295
|
@classes ||= []
|
|
222
296
|
end
|
|
297
|
+
|
|
298
|
+
def recurring_list
|
|
299
|
+
@recurring_list ||= []
|
|
300
|
+
end
|
|
223
301
|
end
|
|
224
302
|
|
|
225
303
|
Monk.freeze_hooks << self
|
|
@@ -234,3 +312,4 @@ require_relative "jobs/job"
|
|
|
234
312
|
# themselves (docs/adr/0017).
|
|
235
313
|
require_relative "mail/deliver_later" if Monk.const_defined?(:Mail)
|
|
236
314
|
require_relative "auth/send_login_link" if Monk.const_defined?(:Auth)
|
|
315
|
+
require_relative "auth/sweep" if Monk.const_defined?(:Auth)
|
data/lib/monk/live/session.rb
CHANGED
|
@@ -58,18 +58,24 @@ module Monk
|
|
|
58
58
|
topics.all? { |topic| topic.is_a?(String) && topic.valid_encoding? }
|
|
59
59
|
end
|
|
60
60
|
|
|
61
|
+
# The new topics go to the registry in one ask, not one each: at a
|
|
62
|
+
# reconnect wave's rate, one ask per topic is most of the registry
|
|
63
|
+
# Ractor's work (docs/plan-ws-connect-ceiling.md, item 5).
|
|
61
64
|
def subscribe(topics)
|
|
62
|
-
|
|
65
|
+
added = {}
|
|
66
|
+
allowed, denied = topics.partition { |topic| try_subscribe?(topic, added) }
|
|
67
|
+
unless added.empty?
|
|
68
|
+
@registry.register_many(added.values, @port)
|
|
69
|
+
@topics.merge!(added)
|
|
70
|
+
end
|
|
63
71
|
reply(op: "subscribed", topics: allowed, denied: denied)
|
|
64
72
|
end
|
|
65
73
|
|
|
66
|
-
def try_subscribe?(topic)
|
|
67
|
-
return true if @topics.key?(topic)
|
|
68
|
-
return false unless permitted?(topic) && @topics.size < @max_topics
|
|
74
|
+
def try_subscribe?(topic, added)
|
|
75
|
+
return true if @topics.key?(topic) || added.key?(topic)
|
|
76
|
+
return false unless permitted?(topic) && @topics.size + added.size < @max_topics
|
|
69
77
|
|
|
70
|
-
|
|
71
|
-
@registry.register(key, @port)
|
|
72
|
-
@topics[topic] = key
|
|
78
|
+
added[topic] = topic.to_sym
|
|
73
79
|
true
|
|
74
80
|
end
|
|
75
81
|
|
|
@@ -79,7 +85,8 @@ module Monk
|
|
|
79
85
|
|
|
80
86
|
def unsubscribe(topics)
|
|
81
87
|
removed = topics.select { |topic| @topics.key?(topic) }
|
|
82
|
-
removed.
|
|
88
|
+
keys = removed.filter_map { |topic| @topics.delete(topic) }
|
|
89
|
+
@registry.unregister_many(keys, @port) unless keys.empty?
|
|
83
90
|
reply(op: "unsubscribed", topics: removed)
|
|
84
91
|
end
|
|
85
92
|
|
|
@@ -119,7 +126,7 @@ module Monk
|
|
|
119
126
|
# read -- so a connection never leaves a stale registry entry.
|
|
120
127
|
def cleanup
|
|
121
128
|
@relay&.kill
|
|
122
|
-
@
|
|
129
|
+
@registry.unregister_many(@topics.values, @port) unless @topics.empty?
|
|
123
130
|
@topics.clear
|
|
124
131
|
@port&.close
|
|
125
132
|
end
|
|
@@ -11,13 +11,20 @@
|
|
|
11
11
|
`Monk::Auth.redeem(token)` returns a session or nil. In routes:
|
|
12
12
|
`set_session_cookie(session)`, `current_subject` (nil when logged out),
|
|
13
13
|
`require_user!` (halts 401), `require_csrf!`, `log_out!`. With jobs:
|
|
14
|
-
`Monk::Auth::SendLoginLink.enqueue(email)`.
|
|
14
|
+
`Monk::Auth::SendLoginLink.enqueue(email)`. Rate limit, in the login
|
|
15
|
+
route before sending: `halt(429) unless Monk::Auth.allow_login_request?(email)`
|
|
16
|
+
(one per address per `login_request_interval:`). `Monk::Auth.sweep!`
|
|
17
|
+
deletes expired rows; with jobs, `Monk::Auth::Sweep` runs it nightly
|
|
18
|
+
(`config/recurring.rb`).
|
|
15
19
|
|
|
16
20
|
**Test:** `test/auth_test.rb` sends a link to `log/test.log` and redeems it.
|
|
21
|
+
A test that asks for links calls `Monk::Auth.clear_login_requests!` in
|
|
22
|
+
`teardown`, or its next request for the same address gets a 429.
|
|
17
23
|
|
|
18
24
|
**Pitfalls:**
|
|
19
|
-
- `/auth/request
|
|
20
|
-
|
|
25
|
+
- Keep `allow_login_request?` in `/auth/request`: without a rate limit,
|
|
26
|
+
anyone can make the app send mail to any address. It limits each
|
|
27
|
+
address, not a client asking for many.
|
|
21
28
|
- Never point `deliver:` at `Monk::Mail.deliver_later`: the link, token
|
|
22
29
|
included, would sit in the job queue. Use `SendLoginLink` instead.
|
|
23
30
|
- Links come from `Monk::Settings[:public_url]`, never from request headers.
|
|
@@ -3,14 +3,8 @@
|
|
|
3
3
|
# its decisions: the rate limit, where to go after logging in, JSON or HTML.
|
|
4
4
|
#
|
|
5
5
|
# monk:example auth-routes -- log in by email link, log out, a page for logged-in users.
|
|
6
|
-
# # Before going live: keep
|
|
6
|
+
# # Before going live: keep the rate limit (without one, anyone can make this
|
|
7
7
|
# # app send mail to any address), and list your redirects in config/auth.rb.
|
|
8
|
-
# module AuthRoutes
|
|
9
|
-
# # Per process and approximate: 5 links per address per 10 minutes. With
|
|
10
|
-
# # several bin/server processes, count in the database instead.
|
|
11
|
-
# LIMIT = Monk::Auth::RateLimiter.new(limit: 5, window: 600)
|
|
12
|
-
# end
|
|
13
|
-
#
|
|
14
8
|
# class App
|
|
15
9
|
# get("/login") { @title = "Log in"; render "auth/login" }
|
|
16
10
|
#
|
|
@@ -18,7 +12,9 @@
|
|
|
18
12
|
# post("/auth/request") do
|
|
19
13
|
# email = params[:email].to_s.strip.downcase
|
|
20
14
|
# halt(422) if email.empty?
|
|
21
|
-
#
|
|
15
|
+
# # One link per address per login_request_interval (config/auth.rb),
|
|
16
|
+
# # counted in the database, so across every bin/server process.
|
|
17
|
+
# halt(429) unless Monk::Auth.allow_login_request?(email)
|
|
22
18
|
#
|
|
23
19
|
# # Sent here, in the request. With jobs, replace these two lines with
|
|
24
20
|
# # Monk::Auth::SendLoginLink.enqueue(email): the token is then made and
|
|
@@ -20,6 +20,7 @@ Monk::Auth.configure(
|
|
|
20
20
|
session_ttl: 1_209_600, # seconds a session stays valid (14 days)
|
|
21
21
|
redirect_allowlist: [], # paths request_login(redirect_to:) is allowed to target
|
|
22
22
|
callback_path: "/auth/callback", # your route redeeming a token: GET "#{callback_path}/:token"
|
|
23
|
+
login_request_interval: 60, # seconds an address waits between login emails (allow_login_request?)
|
|
23
24
|
secure: !Monk.env.development?, # Secure cookie flag; off in dev so plain-http testing works (Safari drops it)
|
|
24
25
|
deliver: AuthMailer::MAGIC_LINK,
|
|
25
26
|
)
|