monkrb 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +56 -0
- data/README.md +5 -4
- data/exe/monk +45 -7
- data/lib/monk/jobs/adapters/pg.rb +272 -0
- data/lib/monk/jobs/args.rb +43 -0
- data/lib/monk/jobs/claim.rb +7 -0
- data/lib/monk/jobs/errors.rb +29 -0
- data/lib/monk/jobs/job.rb +117 -0
- data/lib/monk/jobs/runtime.rb +239 -0
- data/lib/monk/jobs/worker.rb +94 -0
- data/lib/monk/jobs.rb +229 -0
- data/lib/monk/mail/errors.rb +7 -0
- data/lib/monk/mail/later.rb +53 -0
- data/lib/monk/persistence/pg.rb +21 -1
- data/lib/monk/scaffold.rb +232 -8
- data/lib/monk/templates/jobs/bin/jobs +26 -0
- data/lib/monk/templates/jobs/config/jobs.rb +17 -0
- data/lib/monk/templates/jobs/db/migrate/00000000000002_create_jobs_tables.down.sql +3 -0
- data/lib/monk/templates/jobs/db/migrate/00000000000002_create_jobs_tables.up.sql +48 -0
- data/lib/monk/templates/jobs/jobs/hello_job.rb +9 -0
- data/lib/monk/templates/jobs/jobs/send_login_link.rb +27 -0
- data/lib/monk/templates/postgres/Dockerfile +3 -1
- data/lib/monk/version.rb +1 -1
- metadata +16 -1
data/lib/monk/scaffold.rb
CHANGED
|
@@ -56,6 +56,41 @@ module Monk
|
|
|
56
56
|
"config/mail.rb" => "mail/config/mail.rb",
|
|
57
57
|
}.freeze
|
|
58
58
|
|
|
59
|
+
# --jobs: Monk::Jobs. The migration is the canonical one under
|
|
60
|
+
# templates/jobs/db/migrate -- the same file Monk's own tests apply --
|
|
61
|
+
# numbered after auth's so the two sort in order when both are present.
|
|
62
|
+
JOBS_FILES = {
|
|
63
|
+
"config/jobs.rb" => "jobs/config/jobs.rb",
|
|
64
|
+
"jobs/hello_job.rb" => "jobs/jobs/hello_job.rb",
|
|
65
|
+
"bin/jobs" => "jobs/bin/jobs",
|
|
66
|
+
"db/migrate/00000000000002_create_jobs_tables.up.sql" => "jobs/db/migrate/00000000000002_create_jobs_tables.up.sql",
|
|
67
|
+
"db/migrate/00000000000002_create_jobs_tables.down.sql" =>
|
|
68
|
+
"jobs/db/migrate/00000000000002_create_jobs_tables.down.sql",
|
|
69
|
+
}.freeze
|
|
70
|
+
|
|
71
|
+
# --auth --jobs: the magic link is created and sent inside a job, so the
|
|
72
|
+
# raw token is never stored (docs/adr/0014).
|
|
73
|
+
JOBS_AUTH_FILES = {
|
|
74
|
+
"jobs/send_login_link.rb" => "jobs/jobs/send_login_link.rb",
|
|
75
|
+
}.freeze
|
|
76
|
+
|
|
77
|
+
# --mail --jobs (or --auth --jobs): Monk::Mail.deliver_later, loaded in
|
|
78
|
+
# config/jobs.rb right after Monk::Jobs itself.
|
|
79
|
+
JOBS_REQUIRE_ANCHOR = %(require "monk/jobs"\n).freeze
|
|
80
|
+
JOBS_MAIL_REQUIRE = %(require "monk/mail/later"\n).freeze
|
|
81
|
+
|
|
82
|
+
# --jobs's demo route, added right after this line in config.ru (the
|
|
83
|
+
# base and the --live one both end their routes with it), so the
|
|
84
|
+
# round trip -- enqueue from a request, run in bin/jobs -- works out of
|
|
85
|
+
# the box.
|
|
86
|
+
JOBS_ROUTE_ANCHOR = %( get("/api/hello") { json(message: "hello from monk") }\n).freeze
|
|
87
|
+
JOBS_ROUTE = <<~RUBY.gsub(/^(?!$)/, " ").freeze
|
|
88
|
+
|
|
89
|
+
# Enqueues the demo job (jobs/hello_job.rb) for bin/jobs to run:
|
|
90
|
+
# curl -X POST "http://localhost:9292/jobs/hello?name=Ann"
|
|
91
|
+
post("/jobs/hello") { json(enqueued: HelloJob.enqueue(params[:name] || "world")) }
|
|
92
|
+
RUBY
|
|
93
|
+
|
|
59
94
|
# --live: Monk::Live's demo (a counter whose open tabs update when
|
|
60
95
|
# another request changes it). These replace three base files outright
|
|
61
96
|
# rather than patching them -- a live config.ru and index are different
|
|
@@ -89,7 +124,7 @@ module Monk
|
|
|
89
124
|
<script type="module" src="<%= asset_path "/js/monk_live/monk_live.js" %>"></script>
|
|
90
125
|
HTML
|
|
91
126
|
|
|
92
|
-
EXECUTABLE_FILES = %w[bin/server bin/websocket_server bin/console bin/setup_db bin/migrate].freeze
|
|
127
|
+
EXECUTABLE_FILES = %w[bin/server bin/websocket_server bin/console bin/setup_db bin/migrate bin/jobs].freeze
|
|
93
128
|
|
|
94
129
|
# auth: implies postgres -- Monk::Auth is Postgres-only (lib/monk/auth.rb
|
|
95
130
|
# subclasses Monk::Persistence::Pg::Model), so there's no combination
|
|
@@ -119,10 +154,15 @@ module Monk
|
|
|
119
154
|
#
|
|
120
155
|
# mail: Monk::Mail on its own, no Postgres needed. auth: implies it, the
|
|
121
156
|
# same way it implies postgres: a magic link has to reach someone.
|
|
122
|
-
|
|
157
|
+
#
|
|
158
|
+
# jobs: implies postgres, since the queue lives there, and nothing else.
|
|
159
|
+
def initialize(dir, postgres: false, auth: false, redis: false, live: false, mail: false, jobs: false)
|
|
123
160
|
@dir = dir
|
|
161
|
+
# As passed, before anything is implied -- #summary reports the difference.
|
|
162
|
+
@requested = { postgres: postgres, auth: auth, mail: mail, jobs: jobs, redis: redis, live: live }
|
|
124
163
|
@auth = auth
|
|
125
|
-
@
|
|
164
|
+
@jobs = jobs
|
|
165
|
+
@postgres = postgres || auth || jobs
|
|
126
166
|
@mail = mail || auth
|
|
127
167
|
@live = live
|
|
128
168
|
|
|
@@ -144,6 +184,29 @@ module Monk
|
|
|
144
184
|
end
|
|
145
185
|
end
|
|
146
186
|
|
|
187
|
+
# The flags as resolved, for `monk new` to print: which flags were
|
|
188
|
+
# given, which ones they implied and why, and which combinations change
|
|
189
|
+
# what gets generated. The rule behind it, stated in `monk --help`: a
|
|
190
|
+
# flag is implied when there's only one right answer (--auth can only
|
|
191
|
+
# use Postgres), and required when there's a real choice (--live's
|
|
192
|
+
# transport).
|
|
193
|
+
def summary
|
|
194
|
+
given = FLAG_ORDER.select { |flag| @requested[flag] }
|
|
195
|
+
return ["Flags: none (the base skeleton)."] if given.empty?
|
|
196
|
+
|
|
197
|
+
implied = implied_flags
|
|
198
|
+
flags = given.map { |flag| "--#{flag}" }.join(" ")
|
|
199
|
+
lines = implied.empty? ? ["Flags: #{flags}."] : ["Flags: #{flags}, which also turned on:"]
|
|
200
|
+
implied.each do |flag, sources|
|
|
201
|
+
lines << " #{"--#{flag}".ljust(12)}(needed by #{sources.map { |source| "--#{source}" }.join(", ")})"
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
together = combinations
|
|
205
|
+
lines << "Together they also generate:" unless together.empty?
|
|
206
|
+
width = together.map { |pair, _| pair.length }.max
|
|
207
|
+
lines.concat(together.map { |pair, what| " #{pair.ljust(width)} #{what}" })
|
|
208
|
+
end
|
|
209
|
+
|
|
147
210
|
def write!
|
|
148
211
|
raise Monk::ScaffoldExistsError, "#{@dir} already exists" if File.exist?(@dir)
|
|
149
212
|
|
|
@@ -165,6 +228,13 @@ module Monk
|
|
|
165
228
|
append_gemfile_extra("mail/Gemfile.extra")
|
|
166
229
|
end
|
|
167
230
|
|
|
231
|
+
if @jobs
|
|
232
|
+
JOBS_FILES.each { |relative, template| write_file(relative, template, executable: EXECUTABLE_FILES.include?(relative)) }
|
|
233
|
+
JOBS_AUTH_FILES.each { |relative, template| write_file(relative, template) } if @auth
|
|
234
|
+
add_jobs_route!
|
|
235
|
+
add_deliver_later! if @mail
|
|
236
|
+
end
|
|
237
|
+
|
|
168
238
|
wire_config_ru! if @postgres || @mail
|
|
169
239
|
append_gemfile_extra("redis/Gemfile.extra") if @redis
|
|
170
240
|
|
|
@@ -184,8 +254,42 @@ module Monk
|
|
|
184
254
|
write_setup_md!
|
|
185
255
|
end
|
|
186
256
|
|
|
257
|
+
FLAG_ORDER = %i[postgres auth mail jobs redis live].freeze
|
|
258
|
+
|
|
259
|
+
# Only these two imply anything, and only one flag each can need.
|
|
260
|
+
IMPLIED_BY = { postgres: %i[auth jobs], mail: %i[auth] }.freeze
|
|
261
|
+
|
|
187
262
|
private
|
|
188
263
|
|
|
264
|
+
def implied_flags
|
|
265
|
+
resolved = { postgres: @postgres, mail: @mail }
|
|
266
|
+
IMPLIED_BY.filter_map do |flag, sources|
|
|
267
|
+
next if @requested[flag] || !resolved[flag]
|
|
268
|
+
|
|
269
|
+
[flag, sources.select { |source| @requested[source] }]
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def combinations
|
|
274
|
+
pairs = []
|
|
275
|
+
pairs << ["--auth + --jobs", "jobs/send_login_link.rb: login links are sent from a job"] if @auth && @jobs
|
|
276
|
+
if @mail && @jobs
|
|
277
|
+
pairs << ["--mail + --jobs",
|
|
278
|
+
"config/jobs.rb loads Monk::Mail.deliver_later; JOBS_QUEUES serves mailers first",]
|
|
279
|
+
end
|
|
280
|
+
pairs << live_combination if @live
|
|
281
|
+
pairs
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def live_combination
|
|
285
|
+
if @live_transport == :redis
|
|
286
|
+
both = @postgres ? "; --redis wins over --postgres" : ""
|
|
287
|
+
["--live + --redis", "config/live.rb fans out over Redis (Monk::WebSocket::RedisFanout#{both})"]
|
|
288
|
+
else
|
|
289
|
+
["--live + --postgres", "config/live.rb fans out over Postgres (Monk::WebSocket::PgFanout)"]
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
|
|
189
293
|
def write_live!
|
|
190
294
|
write_file("config/live.rb", LIVE_CONFIG_TEMPLATES.fetch(@live_transport))
|
|
191
295
|
LIVE_FILES.each { |relative, template| write_file(relative, template) }
|
|
@@ -251,6 +355,7 @@ module Monk
|
|
|
251
355
|
# bin/websocket_server loads config/auth.rb too and never sends mail,
|
|
252
356
|
# so it shouldn't need MAIL_URL to boot.
|
|
253
357
|
requires << "require_relative \"config/mail\"\n" if @mail
|
|
358
|
+
requires << "require_relative \"config/jobs\"\n" if @jobs
|
|
254
359
|
|
|
255
360
|
content = File.read(path)
|
|
256
361
|
raise "config.ru wiring failed: #{settings_require.inspect} not found" unless content.include?(settings_require)
|
|
@@ -258,6 +363,22 @@ module Monk
|
|
|
258
363
|
File.write(path, content.sub(settings_require, "#{settings_require}#{requires}"))
|
|
259
364
|
end
|
|
260
365
|
|
|
366
|
+
def add_jobs_route!
|
|
367
|
+
path = File.join(@dir, "config.ru")
|
|
368
|
+
content = File.read(path)
|
|
369
|
+
raise "config.ru wiring failed: #{JOBS_ROUTE_ANCHOR.inspect} not found" unless content.include?(JOBS_ROUTE_ANCHOR)
|
|
370
|
+
|
|
371
|
+
File.write(path, content.sub(JOBS_ROUTE_ANCHOR, "#{JOBS_ROUTE_ANCHOR}#{JOBS_ROUTE}"))
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
def add_deliver_later!
|
|
375
|
+
path = File.join(@dir, "config/jobs.rb")
|
|
376
|
+
content = File.read(path)
|
|
377
|
+
raise "config/jobs.rb wiring failed: #{JOBS_REQUIRE_ANCHOR.inspect} not found" unless content.include?(JOBS_REQUIRE_ANCHOR)
|
|
378
|
+
|
|
379
|
+
File.write(path, content.sub(JOBS_REQUIRE_ANCHOR, "#{JOBS_REQUIRE_ANCHOR}#{JOBS_MAIL_REQUIRE}"))
|
|
380
|
+
end
|
|
381
|
+
|
|
261
382
|
# .env/.env.test/.env.example are the other deliberate exception to
|
|
262
383
|
# "templates are static files, copied verbatim" (see the class comment
|
|
263
384
|
# above): DB_NAME needs this app's own directory name in it -- the one
|
|
@@ -289,6 +410,15 @@ module Monk
|
|
|
289
410
|
example << %(MAIL_FROM="#{app_name} <no-reply@example.com>")
|
|
290
411
|
end
|
|
291
412
|
|
|
413
|
+
# bin/jobs's settings. Not in .env.test: tests run jobs with
|
|
414
|
+
# Monk::Jobs.drain!, never a job process.
|
|
415
|
+
# With mail, the mailers queue is served first, so a backlog of other
|
|
416
|
+
# work never delays a login email (docs/adr/0014).
|
|
417
|
+
if @jobs
|
|
418
|
+
queues = @mail ? "mailers,default" : "default"
|
|
419
|
+
[dev, example].each { |lines| lines.push("JOBS_WORKERS=2", "JOBS_QUEUES=#{queues}") }
|
|
420
|
+
end
|
|
421
|
+
|
|
292
422
|
# REDIS_URL is deliberately absent from .env.test -- only a test that
|
|
293
423
|
# actually exercises RedisFanout needs it, unlike DB_NAME/AUTH_SECRET
|
|
294
424
|
# which every test touching persistence/auth needs.
|
|
@@ -589,8 +719,8 @@ module Monk
|
|
|
589
719
|
```bash
|
|
590
720
|
bin/server # HTTP app on :9292
|
|
591
721
|
bin/websocket_server # WS chat process on :9293, in another terminal
|
|
592
|
-
```
|
|
593
|
-
#{auth_or_redis_confirmation_note}#{mail_setup_note}
|
|
722
|
+
#{"bin/jobs # background jobs (jobs/), in a third terminal\n" if @jobs}```
|
|
723
|
+
#{auth_or_redis_confirmation_note}#{mail_setup_note}#{jobs_setup_note}
|
|
594
724
|
## Test environment
|
|
595
725
|
|
|
596
726
|
### 1. Create the test database
|
|
@@ -637,7 +767,7 @@ module Monk
|
|
|
637
767
|
|
|
638
768
|
require "minitest/autorun"
|
|
639
769
|
require_relative "../config/settings"
|
|
640
|
-
require_relative "../config/#{@auth ? "auth" : "persistence"}"#{%(\nrequire_relative "../config/mail") if @mail}
|
|
770
|
+
require_relative "../config/#{@auth ? "auth" : "persistence"}"#{%(\nrequire_relative "../config/mail") if @mail}#{%(\nrequire_relative "../config/jobs") if @jobs}
|
|
641
771
|
```
|
|
642
772
|
|
|
643
773
|
**Rakefile**:
|
|
@@ -663,7 +793,7 @@ module Monk
|
|
|
663
793
|
#{sample_test_body}
|
|
664
794
|
end
|
|
665
795
|
```
|
|
666
|
-
|
|
796
|
+
#{jobs_test_sample}
|
|
667
797
|
Then:
|
|
668
798
|
|
|
669
799
|
```bash
|
|
@@ -714,6 +844,100 @@ module Monk
|
|
|
714
844
|
MARKDOWN
|
|
715
845
|
end
|
|
716
846
|
|
|
847
|
+
# Interpolated into a squiggly heredoc, so no indentation of its own.
|
|
848
|
+
def jobs_setup_note
|
|
849
|
+
return "" unless @jobs
|
|
850
|
+
|
|
851
|
+
<<~MARKDOWN
|
|
852
|
+
|
|
853
|
+
### Background jobs
|
|
854
|
+
|
|
855
|
+
`config/jobs.rb` configures `Monk::Jobs` on the app's own database --
|
|
856
|
+
its tables come from `db/migrate/00000000000002_create_jobs_tables`,
|
|
857
|
+
which `bin/setup_db` already applied -- and loads the job classes in
|
|
858
|
+
`jobs/`. With `bin/server` and `bin/jobs` both running, enqueue the
|
|
859
|
+
demo job and watch it run:
|
|
860
|
+
|
|
861
|
+
```bash
|
|
862
|
+
curl -X POST "http://localhost:9292/jobs/hello?name=Ann"
|
|
863
|
+
tail -f log/development.log # ... INFO Hello, Ann, from a background job
|
|
864
|
+
```
|
|
865
|
+
|
|
866
|
+
`JOBS_WORKERS` and `JOBS_QUEUES` (in `.env`) set how many worker Ractors
|
|
867
|
+
`bin/jobs` runs and which queues they take jobs from, in order. A job
|
|
868
|
+
may run more than once (its process was killed mid-job, say), so make
|
|
869
|
+
each one safe to repeat. `TERM` or Ctrl-C lets the jobs in hand finish
|
|
870
|
+
before `bin/jobs` exits.#{jobs_mail_note}#{jobs_auth_note}
|
|
871
|
+
MARKDOWN
|
|
872
|
+
end
|
|
873
|
+
|
|
874
|
+
# Interpolated into a squiggly heredoc, so no indentation of its own.
|
|
875
|
+
def jobs_mail_note
|
|
876
|
+
return "" unless @mail
|
|
877
|
+
|
|
878
|
+
<<~MARKDOWN.chomp
|
|
879
|
+
|
|
880
|
+
To send an email from a job instead of the request, use
|
|
881
|
+
`Monk::Mail.deliver_later` -- the same arguments as `deliver` (render
|
|
882
|
+
HTML first with `Monk::Mail.render`), plus `wait:`/`at:`, or `conn:`
|
|
883
|
+
to enqueue inside your own transaction. It goes on the `mailers`
|
|
884
|
+
queue, which `JOBS_QUEUES` serves first. Temporary failures are
|
|
885
|
+
retried; a refusal the mail server made final is not.
|
|
886
|
+
MARKDOWN
|
|
887
|
+
end
|
|
888
|
+
|
|
889
|
+
# Interpolated into a squiggly heredoc, so no indentation of its own.
|
|
890
|
+
def jobs_auth_note
|
|
891
|
+
return "" unless @auth
|
|
892
|
+
|
|
893
|
+
<<~MARKDOWN.chomp
|
|
894
|
+
|
|
895
|
+
Magic links go through a job too, `jobs/send_login_link.rb`: it
|
|
896
|
+
creates the token and sends the link inside the job, so the raw
|
|
897
|
+
token is never stored, not even in the queue. Your login route,
|
|
898
|
+
after its own per-email rate limit, just enqueues it:
|
|
899
|
+
|
|
900
|
+
```ruby
|
|
901
|
+
post("/auth/request") do
|
|
902
|
+
SendLoginLink.enqueue(params[:email])
|
|
903
|
+
json(sent: true)
|
|
904
|
+
end
|
|
905
|
+
```
|
|
906
|
+
|
|
907
|
+
Don't point `config/auth.rb`'s `deliver:` at `deliver_later`: that
|
|
908
|
+
would store the link, token included, in the queue until it's sent.
|
|
909
|
+
MARKDOWN
|
|
910
|
+
end
|
|
911
|
+
|
|
912
|
+
# Interpolated into a squiggly heredoc, so no indentation of its own.
|
|
913
|
+
def jobs_test_sample
|
|
914
|
+
return "" unless @jobs
|
|
915
|
+
|
|
916
|
+
<<~MARKDOWN
|
|
917
|
+
|
|
918
|
+
**test/jobs_test.rb** -- `Monk::Jobs.drain!` runs every enqueued job
|
|
919
|
+
right in the test (each in its own Ractor, as `bin/jobs` would, so a
|
|
920
|
+
job that only works outside one fails here too), and
|
|
921
|
+
`Monk::Jobs.clear!` empties the queue between tests:
|
|
922
|
+
|
|
923
|
+
```ruby
|
|
924
|
+
require_relative "test_helper"
|
|
925
|
+
|
|
926
|
+
class JobsTest < Minitest::Test
|
|
927
|
+
def teardown
|
|
928
|
+
Monk::Jobs.clear!
|
|
929
|
+
end
|
|
930
|
+
|
|
931
|
+
def test_the_demo_job_runs
|
|
932
|
+
HelloJob.enqueue("test")
|
|
933
|
+
|
|
934
|
+
assert_equal 1, Monk::Jobs.drain!
|
|
935
|
+
end
|
|
936
|
+
end
|
|
937
|
+
```
|
|
938
|
+
MARKDOWN
|
|
939
|
+
end
|
|
940
|
+
|
|
717
941
|
# The no-Postgres test helper normally has nothing to load from
|
|
718
942
|
# .env.test -- but with --mail it has MAIL_URL=log://, which tests need:
|
|
719
943
|
# they boot outside development, where an unset MAIL_URL raises.
|
|
@@ -745,7 +969,7 @@ module Monk
|
|
|
745
969
|
<<~RUBY.chomp.gsub(/^/, " ")
|
|
746
970
|
def test_connects_to_the_test_database
|
|
747
971
|
Monk::Persistence::Pg.checkout(:primary) do |conn|
|
|
748
|
-
assert_equal
|
|
972
|
+
assert_equal 1, conn.exec("SELECT 1").getvalue(0, 0)
|
|
749
973
|
end
|
|
750
974
|
end
|
|
751
975
|
RUBY
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
require "bundler/setup"
|
|
3
|
+
require_relative "../config/settings"
|
|
4
|
+
|
|
5
|
+
# Loaded when this app has them, so jobs can send mail or use Monk::Auth --
|
|
6
|
+
# a missing file is a harmless no-op, the same way bin/websocket_server
|
|
7
|
+
# treats config/auth.rb.
|
|
8
|
+
%w[mail auth].each do |name|
|
|
9
|
+
require_relative "../config/#{name}"
|
|
10
|
+
rescue LoadError
|
|
11
|
+
nil
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
require_relative "../config/jobs"
|
|
15
|
+
require "monk/jobs/runtime"
|
|
16
|
+
|
|
17
|
+
# The app's templates, so a job can render one (a mail's HTML part, say):
|
|
18
|
+
# there's no App class here to point Monk::Views at them.
|
|
19
|
+
Monk::Views.root = File.expand_path("../views", __dir__)
|
|
20
|
+
|
|
21
|
+
workers = Integer(ENV.fetch("JOBS_WORKERS", "5"))
|
|
22
|
+
queues = ENV.fetch("JOBS_QUEUES", "default").split(",").map(&:strip)
|
|
23
|
+
puts "Monk::Jobs: #{workers} worker(s) on #{queues.join(", ")} -- TERM or Ctrl-C finishes the jobs in hand and stops"
|
|
24
|
+
$stdout.flush
|
|
25
|
+
|
|
26
|
+
Monk::Jobs::Runtime.new(workers: workers, queues: queues).run
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
require "monk"
|
|
2
|
+
require "monk/jobs"
|
|
3
|
+
require_relative "persistence"
|
|
4
|
+
|
|
5
|
+
# Background jobs (Monk::Jobs): enqueue from any route, e.g.
|
|
6
|
+
# HelloJob.enqueue("world"); bin/jobs runs them. Every job is a class under
|
|
7
|
+
# jobs/, loaded below so the web process can enqueue it and bin/jobs can
|
|
8
|
+
# run it.
|
|
9
|
+
#
|
|
10
|
+
# The queue lives in the app's own database, so a job can be enqueued
|
|
11
|
+
# inside the app's own transaction: HelloJob.enqueue("world", conn: conn).
|
|
12
|
+
# Once the app has long-running transactions or heavy load, give the queue
|
|
13
|
+
# a database of its own -- register it in config/persistence.rb and pass
|
|
14
|
+
# its name here (Monk's docs/guides/jobs.md, "Keeping the queue healthy").
|
|
15
|
+
Monk::Jobs.configure(db_name: :primary)
|
|
16
|
+
|
|
17
|
+
Dir[File.expand_path("../jobs/*.rb", __dir__)].sort.each { |file| require file }
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
-- Monk::Jobs: a narrow state table, a payload table, one row per job
|
|
2
|
+
-- process. Why this shape: docs/adr/0013-jobs-narrow-state-table-plus-payloads.md.
|
|
3
|
+
|
|
4
|
+
-- Scheduling and state only, about 70 bytes a row: claiming a job
|
|
5
|
+
-- rewrites this row, never the payload.
|
|
6
|
+
CREATE TABLE monk_jobs (
|
|
7
|
+
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
8
|
+
queue TEXT NOT NULL DEFAULT 'default',
|
|
9
|
+
priority SMALLINT NOT NULL DEFAULT 0, -- lower runs sooner
|
|
10
|
+
state TEXT NOT NULL CHECK (state IN ('available', 'scheduled', 'running', 'failed')),
|
|
11
|
+
run_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
12
|
+
attempts SMALLINT NOT NULL DEFAULT 0,
|
|
13
|
+
max_attempts SMALLINT NOT NULL DEFAULT 5,
|
|
14
|
+
locked_by BIGINT, -- monk_processes.id while running
|
|
15
|
+
locked_at TIMESTAMPTZ
|
|
16
|
+
);
|
|
17
|
+
|
|
18
|
+
-- One partial index per lookup, each holding only the rows it wants:
|
|
19
|
+
-- workers claim from the first, the stager promotes due jobs from the
|
|
20
|
+
-- second, pruning a dead process finds its jobs through the third.
|
|
21
|
+
CREATE INDEX monk_jobs_available ON monk_jobs (queue, priority, run_at, id) WHERE state = 'available';
|
|
22
|
+
CREATE INDEX monk_jobs_scheduled ON monk_jobs (run_at) WHERE state = 'scheduled';
|
|
23
|
+
CREATE INDEX monk_jobs_running ON monk_jobs (locked_by) WHERE state = 'running';
|
|
24
|
+
|
|
25
|
+
-- Every job's row is updated and deleted within seconds, so vacuum it
|
|
26
|
+
-- sooner than Postgres's defaults would (1% of rows dead rather than 20%,
|
|
27
|
+
-- and without cost-based throttling).
|
|
28
|
+
ALTER TABLE monk_jobs SET (autovacuum_vacuum_scale_factor = 0.01, autovacuum_vacuum_cost_delay = 0);
|
|
29
|
+
|
|
30
|
+
-- Written once at enqueue, deleted with its job, updated only to record
|
|
31
|
+
-- an error.
|
|
32
|
+
CREATE TABLE monk_job_payloads (
|
|
33
|
+
job_id BIGINT PRIMARY KEY REFERENCES monk_jobs (id) ON DELETE CASCADE,
|
|
34
|
+
job_class TEXT NOT NULL,
|
|
35
|
+
args JSONB NOT NULL,
|
|
36
|
+
last_error TEXT,
|
|
37
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
-- One row per bin/jobs process: a heartbeat that stops means its
|
|
41
|
+
-- running jobs go back to available.
|
|
42
|
+
CREATE TABLE monk_processes (
|
|
43
|
+
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
44
|
+
hostname TEXT NOT NULL,
|
|
45
|
+
pid INTEGER NOT NULL,
|
|
46
|
+
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
47
|
+
last_heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
48
|
+
);
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# The demo job: POST /jobs/hello enqueues it, bin/jobs runs it, and its
|
|
2
|
+
# line lands in log/<env>.log. A job's args must be plain JSON values (an
|
|
3
|
+
# id, not a record), and a job may run more than once, so make it safe to
|
|
4
|
+
# repeat. Replace this with the app's own jobs.
|
|
5
|
+
class HelloJob < Monk::Job
|
|
6
|
+
def self.perform(name)
|
|
7
|
+
Monk::Log.info("Hello, #{name}, from a background job")
|
|
8
|
+
end
|
|
9
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Sends a magic link from bin/jobs instead of the login request. The token
|
|
2
|
+
# is created here, right before it's sent, so the raw token is never
|
|
3
|
+
# stored anywhere -- not even in the job queue, which a plain
|
|
4
|
+
# Monk::Mail.deliver_later of the link would do (Monk's
|
|
5
|
+
# docs/adr/0014-mail-from-jobs-and-login-links-created-in-the-job.md).
|
|
6
|
+
# Enqueue it from your login route, after your own per-email rate limit
|
|
7
|
+
# and with a redirect_to already checked against redirect_allowlist:
|
|
8
|
+
#
|
|
9
|
+
# post("/auth/request") do
|
|
10
|
+
# SendLoginLink.enqueue(params[:email])
|
|
11
|
+
# json(sent: true)
|
|
12
|
+
# end
|
|
13
|
+
class SendLoginLink < Monk::Job
|
|
14
|
+
queue "mailers"
|
|
15
|
+
# About 50 seconds of retries: a login email minutes late, after the
|
|
16
|
+
# user has likely asked for another, is worse than none.
|
|
17
|
+
max_attempts 3
|
|
18
|
+
never_retry Monk::InvalidRedirectError
|
|
19
|
+
|
|
20
|
+
def self.perform(email, redirect_to = nil)
|
|
21
|
+
token = Monk::Auth.request_login(email, redirect_to: redirect_to)
|
|
22
|
+
# Your callback route; config/settings.rb's public_url is the origin.
|
|
23
|
+
link = "#{Monk::Settings[:public_url]}/auth/callback/#{token}"
|
|
24
|
+
# Calls config/auth.rb's deliver: -- a synchronous send, here in the job.
|
|
25
|
+
Monk::Auth.deliver_link(email: email, link: link, token: token)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -25,6 +25,8 @@ COPY . .
|
|
|
25
25
|
# platform that assigns its own). bin/websocket_server (WS_PORT default
|
|
26
26
|
# 9293) is a separate process/deploy unit -- run this same image with
|
|
27
27
|
# `bin/websocket_server` as the command instead of publishing another
|
|
28
|
-
# port from this Dockerfile -- see docs/guides/deploying.md.
|
|
28
|
+
# port from this Dockerfile -- see docs/guides/deploying.md. With --jobs,
|
|
29
|
+
# bin/jobs is another one, the same way: this image, `bin/jobs` as the
|
|
30
|
+
# command, no port at all.
|
|
29
31
|
EXPOSE 9292
|
|
30
32
|
CMD ["bin/server", "--bind", "0.0.0.0"]
|
data/lib/monk/version.rb
CHANGED
|
@@ -5,5 +5,5 @@ module Monk
|
|
|
5
5
|
# class of bug .freeze! guards against for routes/error handlers/models,
|
|
6
6
|
# just on a top-level constant that every worker Ractor reads on boot
|
|
7
7
|
# (found live under kino: GET /hello 500'd until this was frozen).
|
|
8
|
-
VERSION = "0.
|
|
8
|
+
VERSION = "0.18.0".freeze
|
|
9
9
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: monkrb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.18.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Matteo Folin
|
|
@@ -176,6 +176,14 @@ files:
|
|
|
176
176
|
- lib/monk/environment.rb
|
|
177
177
|
- lib/monk/errors.rb
|
|
178
178
|
- lib/monk/freeze_hooks.rb
|
|
179
|
+
- lib/monk/jobs.rb
|
|
180
|
+
- lib/monk/jobs/adapters/pg.rb
|
|
181
|
+
- lib/monk/jobs/args.rb
|
|
182
|
+
- lib/monk/jobs/claim.rb
|
|
183
|
+
- lib/monk/jobs/errors.rb
|
|
184
|
+
- lib/monk/jobs/job.rb
|
|
185
|
+
- lib/monk/jobs/runtime.rb
|
|
186
|
+
- lib/monk/jobs/worker.rb
|
|
179
187
|
- lib/monk/live.rb
|
|
180
188
|
- lib/monk/live/client/idiomorph.LICENSE
|
|
181
189
|
- lib/monk/live/client/idiomorph.js
|
|
@@ -192,6 +200,7 @@ files:
|
|
|
192
200
|
- lib/monk/mail.rb
|
|
193
201
|
- lib/monk/mail/address.rb
|
|
194
202
|
- lib/monk/mail/errors.rb
|
|
203
|
+
- lib/monk/mail/later.rb
|
|
195
204
|
- lib/monk/mail/message.rb
|
|
196
205
|
- lib/monk/mail/mime.rb
|
|
197
206
|
- lib/monk/mail/transports.rb
|
|
@@ -223,6 +232,12 @@ files:
|
|
|
223
232
|
- lib/monk/templates/base/public/js/app.js
|
|
224
233
|
- lib/monk/templates/base/views/index.erb
|
|
225
234
|
- lib/monk/templates/base/views/layouts/app.erb
|
|
235
|
+
- lib/monk/templates/jobs/bin/jobs
|
|
236
|
+
- lib/monk/templates/jobs/config/jobs.rb
|
|
237
|
+
- lib/monk/templates/jobs/db/migrate/00000000000002_create_jobs_tables.down.sql
|
|
238
|
+
- lib/monk/templates/jobs/db/migrate/00000000000002_create_jobs_tables.up.sql
|
|
239
|
+
- lib/monk/templates/jobs/jobs/hello_job.rb
|
|
240
|
+
- lib/monk/templates/jobs/jobs/send_login_link.rb
|
|
226
241
|
- lib/monk/templates/live/bin/websocket_server
|
|
227
242
|
- lib/monk/templates/live/config.ru
|
|
228
243
|
- lib/monk/templates/live/config/live.rb
|