monkrb 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +133 -0
  3. data/README.md +6 -5
  4. data/exe/monk +45 -7
  5. data/lib/monk/jobs/adapters/pg.rb +272 -0
  6. data/lib/monk/jobs/args.rb +43 -0
  7. data/lib/monk/jobs/claim.rb +7 -0
  8. data/lib/monk/jobs/errors.rb +29 -0
  9. data/lib/monk/jobs/job.rb +117 -0
  10. data/lib/monk/jobs/runtime.rb +239 -0
  11. data/lib/monk/jobs/worker.rb +94 -0
  12. data/lib/monk/jobs.rb +229 -0
  13. data/lib/monk/live/client/monk_live.js +2 -2
  14. data/lib/monk/live/client/protocol.js +5 -0
  15. data/lib/monk/live.rb +10 -0
  16. data/lib/monk/mail/errors.rb +7 -0
  17. data/lib/monk/mail/later.rb +53 -0
  18. data/lib/monk/persistence/pg.rb +21 -1
  19. data/lib/monk/scaffold.rb +369 -70
  20. data/lib/monk/templates/auth/app/mailers/app_mailer.rb +22 -0
  21. data/lib/monk/templates/auth/config/auth.rb +7 -20
  22. data/lib/monk/templates/base/app/app.rb +9 -0
  23. data/lib/monk/templates/base/app/broadcasts/.keep +0 -0
  24. data/lib/monk/templates/base/app/helpers/.keep +0 -0
  25. data/lib/monk/templates/base/app/jobs/.keep +0 -0
  26. data/lib/monk/templates/base/app/mailers/.keep +0 -0
  27. data/lib/monk/templates/base/app/models/.keep +0 -0
  28. data/lib/monk/templates/base/app/presenters/.keep +0 -0
  29. data/lib/monk/templates/base/bin/websocket_server +6 -0
  30. data/lib/monk/templates/base/config/load.rb +18 -0
  31. data/lib/monk/templates/base/config.ru +3 -12
  32. data/lib/monk/templates/base/public/js/app.js +1 -1
  33. data/lib/monk/templates/jobs/app/jobs/hello_job.rb +9 -0
  34. data/lib/monk/templates/jobs/app/jobs/send_login_link.rb +28 -0
  35. data/lib/monk/templates/jobs/bin/jobs +18 -0
  36. data/lib/monk/templates/jobs/config/jobs.rb +15 -0
  37. data/lib/monk/templates/jobs/db/migrate/00000000000002_create_jobs_tables.down.sql +3 -0
  38. data/lib/monk/templates/jobs/db/migrate/00000000000002_create_jobs_tables.up.sql +48 -0
  39. data/lib/monk/templates/live/{config.ru → app/app.rb} +1 -6
  40. data/lib/monk/templates/live/{views → app/views}/index.erb +2 -2
  41. data/lib/monk/templates/live/bin/websocket_server +14 -2
  42. data/lib/monk/templates/live/config/live.rb +1 -1
  43. data/lib/monk/templates/live/config/live_pg.rb +1 -1
  44. data/lib/monk/templates/mail/config/mail.rb +2 -2
  45. data/lib/monk/templates/postgres/Dockerfile +3 -1
  46. data/lib/monk/templates/postgres/bin/console +4 -2
  47. data/lib/monk/version.rb +1 -1
  48. data/lib/monk/websocket/errors.rb +11 -0
  49. data/lib/monk/websocket/listeners.rb +23 -0
  50. data/lib/monk/websocket/pg_fanout.rb +55 -20
  51. data/lib/monk/websocket/redis_fanout.rb +57 -14
  52. data/lib/monk/websocket/registry.rb +6 -0
  53. data/lib/monk/websocket/server.rb +77 -17
  54. metadata +32 -7
  55. /data/lib/monk/templates/auth/{views → app/views}/mail/magic_link.erb +0 -0
  56. /data/lib/monk/templates/base/{views → app/views}/index.erb +0 -0
  57. /data/lib/monk/templates/base/{views → app/views}/layouts/app.erb +0 -0
  58. /data/lib/monk/templates/live/{views → app/views}/live/_hits.erb +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b983d19c7b6de8509f55f50c796d00d90c2d5d345cbe2a9e0f30e8f1349fe7be
4
- data.tar.gz: 0fc9a16f56f69aafd1f3937268d337054399c7de34dd1a9a4986d3a6ae3e1bd5
3
+ metadata.gz: 5745b05b44d3f2953a037262d97c7a211ebea46b72f9872e6d626c58886338bf
4
+ data.tar.gz: f6a970fb8f3e724e992238c7c89f230e215cba4b9d36aadc6122bab452b36ea5
5
5
  SHA512:
6
- metadata.gz: 582398dd16f93f3d352ed51edfb79904a0e494c98daba3230abce233901b468a5e482d5fa97dfe29832293d79e85e340a491d5899cda327a298b120108008f1a
7
- data.tar.gz: 1bde7799803407e6b41c52040e6fa3893a4fd8c2ebb9cca5324cfeb6a1ac291b8a5eaabd7f591204c835179985d14887b009610ebab82f650a6b3d528355f901
6
+ metadata.gz: 9f16caa7f5d287d95cf1adaf65f72a1da511f9a02671ab06ac51634c4f2a44af2ae76c232b7931ec6118f461116f9fc0ce89d08a56bd1e23eeb8f076f4ff834e
7
+ data.tar.gz: 8083b153954fee1475e08b778772bdb8592ff597feae9367bf6d589bf22c5859792ea3bf9bc74e5c1baef0567568ee5302b87d1e82addc65a5703081988b7ec7
data/CHANGELOG.md CHANGED
@@ -4,6 +4,139 @@ All notable changes to this project are documented here. Format is loosely
4
4
  [Keep a Changelog](https://keepachangelog.com/); versions are as released
5
5
  in `lib/monk/version.rb`.
6
6
 
7
+ ## 0.19.0 - 2026-10-01
8
+
9
+ ### Fixed
10
+
11
+ - `Monk::WebSocket::Server#run` stops on `TERM` as it does on Ctrl-C: it
12
+ stops accepting and returns, so `bin/websocket_server` exits 0 on
13
+ `docker stop` and deploys instead of being ended by the signal (exit
14
+ 143). Open sockets are still dropped without a close frame; see
15
+ `docs/guides/websocket.md`, "Stopping and restarting", which also warns
16
+ what other clients have to handle.
17
+ - `monk_live.js` jitters its reconnect delay (50%–150% of the backoff), so
18
+ the pages that lose their socket together on a WebSocket restart don't
19
+ all reconnect and refetch at once. An existing app has a copy of the
20
+ client in `public/js/monk_live/`: copy `lib/monk/live/client/` over it
21
+ again to get this.
22
+
23
+ ### Added
24
+
25
+ - `Monk::WebSocket::Server.new(authenticate: :optional)`: a connection
26
+ with a valid session gets its subject, and one without (none, revoked or
27
+ expired) comes in anonymous instead of getting a `401`. The handler
28
+ decides what an anonymous connection may do; under `Monk::Live`, the
29
+ topic rules (`anonymous: true`). The `Origin` check covers cookies, as
30
+ under `true`, and anonymous connections that send an `Origin`.
31
+ `reverify_interval:` works with it, for connections that have a session.
32
+ `monk new --auth --live`'s `bin/websocket_server` uses it, so the demo
33
+ updates for visitors too, and SETUP.md explains who gets which topics.
34
+ The plain chat `bin/websocket_server` keeps `true`. See
35
+ `docs/design/websocket.md`, "Anonymous connections".
36
+
37
+ ### Changed
38
+
39
+ - `monk new` lays the app out under `app/`, by role (ADR 0015):
40
+ `app/app.rb` holds `class App` and its routes, templates are in
41
+ `app/views/`, and every role directory is created with a `.keep`:
42
+ `app/models`, `presenters`, `helpers`, `mailers`, `broadcasts`, `jobs`.
43
+ `config.ru` is the same three lines for every flag set, and the new
44
+ `config/load.rb` requires the settings and each enabled module's config,
45
+ then loads `app/` by role. It never boots the app. `bin/jobs`,
46
+ `bin/console` and the generated test helper require it too. `--auth`'s
47
+ magic-link sender moves from `config/auth.rb` to
48
+ `app/mailers/app_mailer.rb` and is renamed `AppMailer::MAGIC_LINK`;
49
+ `config/auth.rb` requires that file itself. `config/jobs.rb` no longer
50
+ loads the job classes. The generated SETUP.md's test helper loads and
51
+ boots the app, with a sample request test. `docs/guides/scaffolding.md`,
52
+ "Where code goes", says what goes in each directory.
53
+ **An 0.18 app keeps working as it is.** To move it to the new layout:
54
+ 1. `git mv views app/views` and `git mv jobs app/jobs`; create the other
55
+ role directories you want.
56
+ 2. Move `class App` and its routes from `config.ru` into `app/app.rb`,
57
+ and change `views "views"` to `views "app/views"`.
58
+ 3. Create `config/load.rb`: the `require_relative` lines `config.ru` had
59
+ for `config/*` (without the `config/` prefix), then the loop over
60
+ `app/` roles from a freshly generated one. Make `config.ru`
61
+ `require_relative "config/load"`, `require_relative "app/app"`,
62
+ `run Monk.boot(App)`.
63
+ 4. Remove the `Dir[...]` loop from `config/jobs.rb`. In `bin/jobs`,
64
+ replace its config requires with `require_relative "../config/load"`
65
+ and point `Monk::Views.root` at `../app/views`. Do the same in
66
+ `bin/console`.
67
+ 5. Optional: move `AppMailer` from `config/auth.rb` into
68
+ `app/mailers/app_mailer.rb` and `require_relative` it from
69
+ `config/auth.rb`.
70
+ - **Breaking:** `Monk::WebSocket::RedisFanout` and `PgFanout` no longer
71
+ subscribe when built. The process that holds sockets calls `#listen!`
72
+ once at boot: `Monk::Live.listen!`, or `REGISTRY.listen!` for plain
73
+ `Monk::WebSocket`. Publish-only processes (`bin/server`, `bin/jobs`,
74
+ `bin/console`) then hold no idle subscriber connection. `#listen!`
75
+ returns once the subscription is in effect and raises
76
+ `Monk::WebSocket::ListenError` if Redis or Postgres can't be reached.
77
+ `#register` before `#listen!` raises
78
+ `Monk::WebSocket::NotListeningError`. `Registry#listen!` is a no-op, so
79
+ the same line works with either. The scaffolded `bin/websocket_server`
80
+ calls it. **To upgrade**, add `Monk::Live.listen!` (or
81
+ `REGISTRY.listen!`) to your `bin/websocket_server` before
82
+ `server.run`. See `docs/design/websocket.md`, "Who listens".
83
+
84
+ ## 0.18.0 - 2026-09-29
85
+
86
+ ### Added
87
+
88
+ - `Monk::Jobs` (`require "monk/jobs"`): background jobs on Postgres, with
89
+ no other dependency. A job is a `Monk::Job` subclass with
90
+ `self.perform` and optional `queue`, `priority`, `max_attempts` and
91
+ `timeout` settings, enqueued with `SendReceipt.enqueue(*args, wait:,
92
+ at:, conn:)`. Args must be plain JSON values, checked at enqueue.
93
+ `conn:` enqueues inside the app's own transaction. The queue is a narrow
94
+ state table plus a payload table, claimed with `FOR UPDATE SKIP
95
+ LOCKED`: ADR 0013 records the design and the benchmarks that chose it
96
+ over one wide table and over Solid Queue's split.
97
+ `Monk::Jobs::Runtime` (`require "monk/jobs/runtime"`) is the job
98
+ process. It runs a supervisor and a pool of worker Ractors, retries
99
+ failures with backoff, keeps jobs that run out of attempts as failed
100
+ (`Monk::Jobs.retry_failed`/`discard_failed`), and releases the jobs of
101
+ dead workers and dead processes. On `TERM` it finishes the jobs in hand
102
+ first. Delivery is at-least-once. For tests, `Monk::Jobs.drain!` runs
103
+ every due job in its own Ractor, and `Monk::Jobs.clear!` empties the
104
+ queue (test environment only). See `docs/guides/jobs.md`.
105
+ - `monk new --jobs` (implies `--postgres`): `config/jobs.rb`, a demo
106
+ `HelloJob` with a `POST /jobs/hello` route, `bin/jobs`, the queue's
107
+ migration, `JOBS_WORKERS`/`JOBS_QUEUES` in `.env`, and `SETUP.md`
108
+ steps for running and testing jobs.
109
+ - `Monk::Mail.deliver_later` (`require "monk/mail/later"`): `deliver`'s
110
+ arguments plus `wait:`/`at:`/`conn:`, sent from a job on the `mailers`
111
+ queue. The message is checked when called, so bad input raises there.
112
+ Temporary failures are retried, while a refusal the server made final
113
+ (SMTP 5xx, refused credentials) fails the job at once as the new
114
+ `Monk::Mail::PermanentDeliveryError`, a `DeliveryError`. `deliver` is
115
+ unchanged. See ADR 0014.
116
+ - `never_retry *error_classes` on `Monk::Job`: errors that fail a job at
117
+ once instead of spending its remaining attempts.
118
+ - `monk new` prints the flags it resolved: which ones another flag turned
119
+ on, and why (`--postgres (needed by --auth, --jobs)`), and which pairs
120
+ change what gets generated. `monk --help` and the scaffolding guide
121
+ state the rule behind it: a flag is implied when there's only one right
122
+ answer, and required when there's a real choice (`--live`'s transport).
123
+ - `monk new --auth --jobs` adds `jobs/send_login_link.rb`, which creates
124
+ the login token and sends the link inside the job, so the raw token is
125
+ never stored in the queue. With mail, `--jobs` also loads
126
+ `deliver_later` and serves the `mailers` queue first.
127
+
128
+ ### Fixed
129
+
130
+ - `json` and `jsonb` columns read through `Monk::Persistence::Pg` (raw
131
+ queries and `Model` alike) raised `ArgumentError: unknown keyword:
132
+ quirks_mode`. pg 1.6.3's JSON decoder passes that keyword to
133
+ `JSON.parse`, and json 3 removed it. Monk's connections now decode
134
+ `json`/`jsonb` with their own decoder, keeping pg's other default
135
+ types.
136
+ - The sample test `SETUP.md` gives a `monk new --postgres` app (without
137
+ `--auth`) failed as written: it expected `SELECT 1` to return the
138
+ String `"1"`, but Monk's connections decode it to the Integer `1`.
139
+
7
140
  ## 0.17.0 - 2026-09-26
8
141
 
9
142
  ### Added
data/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  A light Ruby web framework designed to be fully `Ractor`-safe: every app it produces is a valid Rack 3 app that is also `Ractor.shareable?`, so it can be served in parallel across Ractor worker pools without silently losing that safety property. Named after Thelonious Sphere Monk, great and unique Jazz piano player and composer.
4
4
 
5
- **Built in** (loaded by `require "monk"`): routing, context and error handling, boot-time Ractor-shareability checks, `Monk::StateRactor` for shared state, settings, ERB views, static assets and logging. **Opt-in** (each needs its own `require`): Postgres persistence and migrations, passwordless auth and sessions, a WebSocket server with Redis or Postgres fan-out, and `Monk::Live` server-pushed HTML updates. Details for each are in the [Features](#features) table below.
5
+ **Built in** (loaded by `require "monk"`): routing, context and error handling, boot-time Ractor-shareability checks, `Monk::StateRactor` for shared state, settings, ERB views, static assets and logging. **Opt-in** (each needs its own `require`): Postgres persistence and migrations, passwordless auth and sessions, email, background jobs on Postgres, a WebSocket server with Redis or Postgres fan-out, and `Monk::Live` server-pushed HTML updates. Details for each are in the [Features](#features) table below.
6
6
 
7
7
  Monk is Kino-agnostic — it's built on stdlib `Ractor` primitives only, with no runtime dependency on any particular server. [Kino](https://github.com/yaroslav/kino) is the reference/development server (see `bin/server`), but any Ractor-aware Rack server, or a conventional one, can run a Monk app.
8
8
 
@@ -22,7 +22,7 @@ bin/server # -> http://localhost:9292/hello
22
22
 
23
23
  `monk new` scaffolds the app's own `Gemfile` with `gem "monkrb", require: "monk"`.
24
24
 
25
- `monk new` writes a working skeleton (an HTML home page, a `/hello` route, a `/api/hello` JSON route, `views/`, `public/`, a `SETUP.md`). Flags add Postgres, auth, Redis and live updates: see [`docs/guides/scaffolding.md`](docs/guides/scaffolding.md).
25
+ `monk new` writes a working skeleton (an HTML home page, a `/hello` route, a `/api/hello` JSON route, `public/`, a `SETUP.md`), with the app's own code under `app/`: `app/app.rb` for routes, `app/views/`, and one directory per role (`models/`, `presenters/`, `helpers/`, `mailers/`, `broadcasts/`, `jobs/`). Flags add Postgres, auth, email, background jobs, Redis and live updates: see [`docs/guides/scaffolding.md`](docs/guides/scaffolding.md), which also says where each kind of code goes.
26
26
 
27
27
  All `monk` commands and flags are listed by:
28
28
 
@@ -60,10 +60,11 @@ Everything beyond the core is opt-in (`require "monk"` alone loads none of it).
60
60
  | Persistence | `Monk::Persistence::Pg`: raw `pg`, per-Ractor connections, hash-based `Model` | `require "monk/persistence/pg"` (+ `.../pg/model`); needs the `pg` gem | [`persistence.md`](docs/guides/persistence.md) |
61
61
  | Migrations | plain `.sql` up/down pairs, `Migrator` | `require "monk/persistence/pg/migrator"`; needs the `pg` gem | [`migrations.md`](docs/guides/migrations.md) |
62
62
  | Auth and sessions | `Monk::Auth`: passwordless tokens, Bearer or cookie + CSRF | `require "monk/auth"`; needs the `pg` gem and a registered Postgres connection | [`auth.md`](docs/guides/auth.md) |
63
- | Email | `Monk::Mail`: text/HTML email over SMTP or a local relay, one `MAIL_URL`, sent from any worker Ractor | `require "monk/mail"`; SMTP needs the `net-smtp` gem | [`mail.md`](docs/guides/mail.md) |
63
+ | Email | `Monk::Mail`: text/HTML email over SMTP or a local relay, one `MAIL_URL`, sent from any worker Ractor, or from a background job with `deliver_later` | `require "monk/mail"` (+ `require "monk/mail/later"` for `deliver_later`, which needs background jobs); SMTP needs the `net-smtp` gem | [`mail.md`](docs/guides/mail.md) |
64
+ | Background jobs | `Monk::Jobs`: a queue in Postgres, run by `bin/jobs` on worker Ractors; retries with backoff, scheduled jobs, enqueue inside the app's own transaction, `drain!` for tests | `require "monk/jobs"` (+ `require "monk/jobs/runtime"` in the job process); needs the `pg` gem and a registered Postgres connection | [`jobs.md`](docs/guides/jobs.md) |
64
65
  | WebSocket | `Monk::WebSocket`: RFC 6455 server as its own process, Redis or Postgres fan-out | `require "monk/websocket"`; fan-out: `require "monk/websocket/redis_fanout"` (+ the `redis` gem) or `require "monk/websocket/pg_fanout"` (+ the `pg` gem) | [`websocket.md`](docs/guides/websocket.md) |
65
66
  | Live updates | `Monk::Live`: server-rendered HTML patches pushed to open tabs | `require "monk/live"`; needs the WebSocket server and, across processes, Redis or Postgres | [`live.md`](docs/guides/live.md) |
66
- | Scaffolding | `monk new` and its flags; retrofitting Postgres, Auth or Redis | — (the `monk` command; flags `--postgres`, `--auth`, `--redis`, `--live`) | [`scaffolding.md`](docs/guides/scaffolding.md) |
67
+ | Scaffolding | `monk new` and its flags; retrofitting Postgres, Auth, Redis or Jobs | — (the `monk` command; flags `--postgres`, `--auth`, `--mail`, `--jobs`, `--redis`, `--live`) | [`scaffolding.md`](docs/guides/scaffolding.md) |
67
68
 
68
69
  ## More documentation
69
70
 
@@ -86,6 +87,6 @@ More (server modes, Docker) in [`docs/development.md`](docs/development.md).
86
87
 
87
88
  ## Status
88
89
 
89
- Monk is **pre-1.0** (see `lib/monk/version.rb` and the [changelog](CHANGELOG.md)): the API may still change between minor versions. The core is described in [`docs/history/core-plan.md`](docs/history/core-plan.md). There's no open roadmap issue at the moment; new work is proposed and tracked as it comes up. Done so far, each with its own design doc and plan: persistence, migrations, HTML templating and static assets, auth and sessions, WebSocket with Redis fan-out, log levels, live updates (`Monk::Live`, as of 2026-09-19), deployment support (Dockerfile scaffolding, `docs/guides/deploying.md`), the RubyGems release as `monkrb` (both 2026-09-22), and `Monk::WebSocket::PgFanout`/`monk new --live --postgres` — the same cross-process fan-out over Postgres `LISTEN`/`NOTIFY` instead of Redis, for an app that doesn't want a second piece of infrastructure (`docs/design/live-pg-fanout.md`, 2026-09-24).
90
+ Monk is **pre-1.0** (see `lib/monk/version.rb` and the [changelog](CHANGELOG.md)): the API may still change between minor versions. The core is described in [`docs/history/core-plan.md`](docs/history/core-plan.md). There's no open roadmap issue at the moment; new work is proposed and tracked as it comes up. Done so far, each with its own design doc and plan: persistence, migrations, HTML templating and static assets, auth and sessions, WebSocket with Redis fan-out, log levels, live updates (`Monk::Live`, as of 2026-09-19), deployment support (Dockerfile scaffolding, `docs/guides/deploying.md`), the RubyGems release as `monkrb` (both 2026-09-22), and `Monk::WebSocket::PgFanout`/`monk new --live --postgres` — the same cross-process fan-out over Postgres `LISTEN`/`NOTIFY` instead of Redis, for an app that doesn't want a second piece of infrastructure (`docs/design/live-pg-fanout.md`, 2026-09-24), the built-in mailer `Monk::Mail` (0.17, 2026-09-26, ADR 0012), and background jobs on Postgres, `Monk::Jobs`, with mail sent from jobs (0.18, 2026-09-29, ADRs 0013 and 0014). In 0.19 (2026-10-01): `monk new` lays the app out under `app/`, one directory per role (ADR 0015); a WebSocket fanout listens only in the process that holds the sockets (`#listen!`); `authenticate: :optional` lets visitors' sockets in anonymously, for Live's rules to decide; and the WebSocket server stops cleanly on `TERM` (`docs/design/websocket.md`).
90
91
 
91
92
  [Made with Love ❤️, Ruby 💎 and AI 🤖](docs/ai_usage_disclaimer.md)
data/exe/monk CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
  require_relative "../lib/monk/scaffold"
3
3
 
4
- USAGE = "Usage: monk new APP_NAME [--postgres] [--auth] [--mail] [--redis] [--live]".freeze
4
+ USAGE = "Usage: monk new APP_NAME [--postgres] [--auth] [--mail] [--redis] [--live] [--jobs]".freeze
5
5
 
6
6
  HELP = <<~TEXT.freeze
7
7
  #{USAGE}
@@ -79,6 +79,37 @@ HELP = <<~TEXT.freeze
79
79
  views/index.erb and bin/websocket_server with live
80
80
  versions, and adds the runtime's <meta>/<script> tags
81
81
  to the layout.
82
+ --jobs Also scaffold Monk::Jobs (background jobs on Postgres).
83
+ Implies --postgres, where the queue lives. Adds
84
+ config/jobs.rb (required by config.ru, after
85
+ persistence/auth/mail), jobs/hello_job.rb (a demo
86
+ job, enqueued by a POST /jobs/hello route added to
87
+ config.ru), bin/jobs (the job process, run beside
88
+ bin/server), and the migration creating the queue's
89
+ tables. .env/.env.example get JOBS_WORKERS and
90
+ JOBS_QUEUES (not .env.test: tests run jobs with
91
+ Monk::Jobs.drain!).
92
+
93
+ How flags combine -- a flag is implied when there's only one right
94
+ answer, and required when there's a real choice:
95
+
96
+ Flag Implies Requires
97
+ --auth --postgres, --mail -
98
+ --jobs --postgres -
99
+ --live - --redis or --postgres (Live's
100
+ transport: a choice, so it's yours)
101
+ --postgres, --mail, --redis stand alone.
102
+
103
+ Some pairs also change what gets generated:
104
+
105
+ --auth + --jobs jobs/send_login_link.rb: login links sent from a job
106
+ --mail + --jobs config/jobs.rb loads Monk::Mail.deliver_later;
107
+ JOBS_QUEUES serves mailers first
108
+ --live + --redis config/live.rb fans out over Redis
109
+ --live + --postgres config/live.rb fans out over Postgres (without --redis)
110
+
111
+ `monk new` prints which flags it turned on, and why, after creating the
112
+ project.
82
113
 
83
114
  Commands:
84
115
  new APP_NAME Create a new project (see Options above)
@@ -119,20 +150,27 @@ when "new"
119
150
  app_name = rest.find { |arg| !arg.start_with?("-") }
120
151
  usage_error! if app_name.nil?
121
152
 
153
+ # Flags as typed: Monk::Scaffold resolves what they imply, and reports it.
122
154
  auth = rest.include?("--auth")
123
- mail = rest.include?("--mail") || auth
155
+ mail = rest.include?("--mail") || auth # for the next-steps lines below
124
156
  redis = rest.include?("--redis")
125
157
  live = rest.include?("--live")
126
- Monk::Scaffold.new(
127
- app_name, postgres: rest.include?("--postgres"), auth: auth, mail: mail, redis: redis, live: live,
128
- ).write!
129
-
130
- puts "Created #{app_name}. Next steps:"
158
+ jobs = rest.include?("--jobs")
159
+ scaffold = Monk::Scaffold.new(
160
+ app_name, postgres: rest.include?("--postgres"), auth: auth, mail: rest.include?("--mail"), redis: redis,
161
+ live: live, jobs: jobs,
162
+ )
163
+ scaffold.write!
164
+
165
+ puts "Created #{app_name}."
166
+ puts scaffold.summary
167
+ puts "Next steps:"
131
168
  puts " cd #{app_name}"
132
169
  puts " bundle install"
133
170
  puts " see SETUP.md for the full dev-then-test walkthrough"
134
171
  puts " change the placeholder AUTH_SECRET in .env/.env.test before relying on it" if auth
135
172
  puts " set MAIL_URL and MAIL_FROM outside development (see config/mail.rb)" if mail
173
+ puts " run bin/jobs beside bin/server to process background jobs (see SETUP.md)" if jobs
136
174
  if live
137
175
  if redis
138
176
  puts " Monk::Live needs Redis running, bin/server and bin/websocket_server (see SETUP.md)"
@@ -0,0 +1,272 @@
1
+ require "json"
2
+ require "time"
3
+ require_relative "../../persistence/pg"
4
+ require_relative "../claim"
5
+
6
+ module Monk
7
+ module Jobs
8
+ module Adapters
9
+ # The queue on Postgres: monk_jobs + monk_job_payloads + monk_processes
10
+ # (docs/adr/0013-jobs-narrow-state-table-plus-payloads.md). Holds only
11
+ # the registered database's name and freezes itself, so the one
12
+ # instance is shareable with every Ractor; each Ractor reaches the
13
+ # database through its own connection, via Monk::Persistence::Pg.
14
+ class Pg
15
+ # One statement, so a job and its payload are written together even
16
+ # outside a transaction. The database clock decides run_at, and so
17
+ # whether the job starts scheduled or available -- an app server's
18
+ # clock running a little ahead can't make a job look due early.
19
+ ENQUEUE_SQL = <<~SQL.freeze
20
+ WITH due AS (
21
+ SELECT COALESCE($4::timestamptz, now() + COALESCE($5::float8, 0) * interval '1 second') AS run_at
22
+ ), job AS (
23
+ INSERT INTO monk_jobs (queue, priority, max_attempts, run_at, state)
24
+ SELECT $1::text, $2::smallint, $3::smallint, run_at,
25
+ CASE WHEN run_at > now() THEN 'scheduled' ELSE 'available' END
26
+ FROM due
27
+ RETURNING id
28
+ )
29
+ INSERT INTO monk_job_payloads (job_id, job_class, args)
30
+ SELECT id, $6::text, $7::jsonb FROM job
31
+ RETURNING job_id
32
+ SQL
33
+
34
+ # Decision 2: the claim rewrites only the narrow monk_jobs row. args
35
+ # comes back as text and is parsed here rather than by the
36
+ # connection's result type map, so it decodes the same whichever
37
+ # connection runs it.
38
+ CLAIM_SQL = <<~SQL.freeze
39
+ WITH claimed AS (
40
+ UPDATE monk_jobs
41
+ SET state = 'running', locked_by = $2, locked_at = now(), attempts = attempts + 1
42
+ WHERE id = (
43
+ SELECT id FROM monk_jobs
44
+ WHERE queue = $1 AND state = 'available'
45
+ ORDER BY priority, run_at, id
46
+ FOR UPDATE SKIP LOCKED
47
+ LIMIT 1
48
+ )
49
+ RETURNING id, attempts
50
+ )
51
+ SELECT claimed.id, claimed.attempts, payload.job_class, payload.args::text AS args
52
+ FROM claimed JOIN monk_job_payloads payload ON payload.job_id = claimed.id
53
+ SQL
54
+
55
+ # Only while this process still holds the job: one pruned from a dead
56
+ # process and claimed again elsewhere isn't deleted by the first
57
+ # worker finishing late. The payload goes by ON DELETE CASCADE.
58
+ FINISH_SQL = <<~SQL.freeze
59
+ DELETE FROM monk_jobs WHERE id = $1 AND state = 'running' AND locked_by = $2
60
+ SQL
61
+
62
+ # Only while this process still holds the job, like FINISH_SQL. The
63
+ # database decides where it goes: back to scheduled, due retry_in
64
+ # seconds from now, or to failed once it has used its last attempt
65
+ # (attempts was already counted when it was claimed) or when
66
+ # retry_in is NULL. The payload keeps the error either way; the
67
+ # payload update runs whether or not the final SELECT reads it.
68
+ FAIL_SQL = <<~SQL.freeze
69
+ WITH job AS (
70
+ UPDATE monk_jobs SET
71
+ state = CASE WHEN $3::float8 IS NULL OR attempts >= max_attempts THEN 'failed' ELSE 'scheduled' END,
72
+ run_at = CASE WHEN $3::float8 IS NULL OR attempts >= max_attempts THEN run_at
73
+ ELSE now() + $3::float8 * interval '1 second' END,
74
+ locked_by = NULL, locked_at = NULL
75
+ WHERE id = $1 AND state = 'running' AND locked_by = $2
76
+ RETURNING id, state
77
+ ), payload AS (
78
+ UPDATE monk_job_payloads SET last_error = $4 FROM job WHERE monk_job_payloads.job_id = job.id
79
+ )
80
+ SELECT state FROM job
81
+ SQL
82
+
83
+ # The stager: due scheduled jobs become available, oldest first, at
84
+ # most $1 per call. SKIP LOCKED lets every job process stage at
85
+ # once without two of them moving the same job.
86
+ STAGE_SQL = <<~SQL.freeze
87
+ UPDATE monk_jobs SET state = 'available'
88
+ WHERE id IN (
89
+ SELECT id FROM monk_jobs
90
+ WHERE state = 'scheduled' AND run_at <= now()
91
+ ORDER BY run_at
92
+ LIMIT $1
93
+ FOR UPDATE SKIP LOCKED
94
+ )
95
+ SQL
96
+
97
+ RETRY_FAILED_SQL = <<~SQL.freeze
98
+ UPDATE monk_jobs SET state = 'available', attempts = 0, run_at = now() WHERE id = $1 AND state = 'failed'
99
+ SQL
100
+
101
+ DISCARD_FAILED_SQL = <<~SQL.freeze
102
+ DELETE FROM monk_jobs WHERE id = $1 AND state = 'failed'
103
+ SQL
104
+
105
+ REGISTER_PROCESS_SQL = <<~SQL.freeze
106
+ INSERT INTO monk_processes (hostname, pid) VALUES ($1, $2) RETURNING id
107
+ SQL
108
+
109
+ # An upsert, not an UPDATE: a process that stalled long enough for
110
+ # another to prune it gets its row back under the same id. The jobs
111
+ # pruning released stay with whoever claimed them since; this
112
+ # process's late finish/fail calls on them just find nothing held.
113
+ HEARTBEAT_SQL = <<~SQL.freeze
114
+ INSERT INTO monk_processes (id, hostname, pid) OVERRIDING SYSTEM VALUE VALUES ($1, $2, $3)
115
+ ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()
116
+ SQL
117
+
118
+ # Deletes processes whose heartbeat is older than $1 seconds, and
119
+ # releases their running jobs back to available with their attempt
120
+ # still counted (Decision 4). Also releases running jobs with no
121
+ # process row at all once they've been running that long -- a
122
+ # drain! that crashed mid-job, or rows left by a bug. The NOT EXISTS
123
+ # sees the rows as they were before the DELETE, so a pruned
124
+ # process's jobs are caught by the first condition.
125
+ PRUNE_SQL = <<~SQL.freeze
126
+ WITH dead AS (
127
+ DELETE FROM monk_processes
128
+ WHERE last_heartbeat_at < now() - $1::float8 * interval '1 second'
129
+ RETURNING id
130
+ )
131
+ UPDATE monk_jobs SET state = 'available', locked_by = NULL, locked_at = NULL
132
+ WHERE state = 'running' AND (
133
+ locked_by IN (SELECT id FROM dead)
134
+ OR (locked_at < now() - $1::float8 * interval '1 second'
135
+ AND NOT EXISTS (SELECT 1 FROM monk_processes p WHERE p.id = monk_jobs.locked_by))
136
+ )
137
+ SQL
138
+
139
+ # A graceful stop: whatever this process still has running goes back
140
+ # to available, and its row goes.
141
+ DEREGISTER_SQL = <<~SQL.freeze
142
+ WITH gone AS (DELETE FROM monk_processes WHERE id = $1)
143
+ UPDATE monk_jobs SET state = 'available', locked_by = NULL, locked_at = NULL
144
+ WHERE state = 'running' AND locked_by = $1
145
+ SQL
146
+
147
+ # One job back to available, if process_id still holds it: the job a
148
+ # worker was running when that worker died.
149
+ RELEASE_SQL = <<~SQL.freeze
150
+ UPDATE monk_jobs SET state = 'available', locked_by = NULL, locked_at = NULL
151
+ WHERE id = $1 AND state = 'running' AND locked_by = $2
152
+ SQL
153
+
154
+ CLEAR_SQL = <<~SQL.freeze
155
+ DELETE FROM monk_jobs;
156
+ DELETE FROM monk_processes;
157
+ SQL
158
+
159
+ attr_reader :db_name
160
+
161
+ def initialize(db_name:)
162
+ @db_name = db_name
163
+ freeze
164
+ end
165
+
166
+ # Returns the new job's id. conn: runs it on a connection the caller
167
+ # already holds, inside the caller's transaction -- required rather
168
+ # than optional there, since checking out the same Ractor's
169
+ # connection again would wait on itself.
170
+ def enqueue(job_class:, queue:, priority:, max_attempts:, args:, wait: nil, at: nil, conn: nil)
171
+ params = [queue, priority, max_attempts, at&.utc&.iso8601(6), wait&.to_f, job_class, args]
172
+ with_connection(conn) { |c| Integer(c.exec_params(ENQUEUE_SQL, params).getvalue(0, 0)) }
173
+ end
174
+
175
+ # The next available job on `queue`, now running for process_id, or
176
+ # nil when there's none.
177
+ def claim(queue, process_id)
178
+ row = with_connection { |c| c.exec_params(CLAIM_SQL, [queue, process_id]).first }
179
+ return nil unless row
180
+
181
+ Claim.new(
182
+ id: Integer(row["id"]), job_class: row["job_class"],
183
+ args: JSON.parse(row["args"]), attempts: Integer(row["attempts"]),
184
+ )
185
+ end
186
+
187
+ # True if the job was deleted, false if process_id no longer held it.
188
+ def finish(id, process_id)
189
+ with_connection { |c| c.exec_params(FINISH_SQL, [id, process_id]).cmd_tuples == 1 }
190
+ end
191
+
192
+ # Records a failure of a job process_id holds: :scheduled when it will
193
+ # be retried in retry_in seconds, :failed when it won't (no attempts
194
+ # left, or retry_in nil), nil when process_id no longer held it.
195
+ def fail(id, process_id, error:, retry_in:)
196
+ row = with_connection { |c| c.exec_params(FAIL_SQL, [id, process_id, retry_in&.to_f, error]).first }
197
+ row && row["state"].to_sym
198
+ end
199
+
200
+ # Moves up to `limit` due scheduled jobs to available; returns how many.
201
+ def stage_due(limit = 500)
202
+ with_connection { |c| c.exec_params(STAGE_SQL, [limit]).cmd_tuples }
203
+ end
204
+
205
+ # A failed job back to available with fresh attempts. False if it
206
+ # isn't failed (or doesn't exist).
207
+ def retry_failed(id)
208
+ with_connection { |c| c.exec_params(RETRY_FAILED_SQL, [id]).cmd_tuples == 1 }
209
+ end
210
+
211
+ # Deletes a failed job and its payload. False if it isn't failed.
212
+ def discard_failed(id)
213
+ with_connection { |c| c.exec_params(DISCARD_FAILED_SQL, [id]).cmd_tuples == 1 }
214
+ end
215
+
216
+ # A job process's own row; returns its id, the process_id it claims
217
+ # jobs as.
218
+ def register_process(hostname:, pid:)
219
+ with_connection { |c| Integer(c.exec_params(REGISTER_PROCESS_SQL, [hostname, pid]).getvalue(0, 0)) }
220
+ end
221
+
222
+ def heartbeat(process_id, hostname:, pid:)
223
+ with_connection { |c| c.exec_params(HEARTBEAT_SQL, [process_id, hostname, pid]) }
224
+ nil
225
+ end
226
+
227
+ # Releases the jobs of processes silent for more than `timeout`
228
+ # seconds, and deletes those processes; returns how many jobs were
229
+ # released.
230
+ def prune(timeout)
231
+ with_connection { |c| c.exec_params(PRUNE_SQL, [timeout.to_f]).cmd_tuples }
232
+ end
233
+
234
+ # The job a dead worker was holding, back to available with its
235
+ # attempt counted; true if process_id still held it.
236
+ def release(id, process_id)
237
+ with_connection { |c| c.exec_params(RELEASE_SQL, [id, process_id]).cmd_tuples == 1 }
238
+ end
239
+
240
+ # Releases this process's running jobs and deletes its row.
241
+ def deregister(process_id)
242
+ with_connection { |c| c.exec_params(DEREGISTER_SQL, [process_id]) }
243
+ nil
244
+ end
245
+
246
+ # After a job timed out mid-query: cancels whatever the calling
247
+ # Ractor's connection is still running on the server, then
248
+ # reconnects, so the worker's next query doesn't wait for the
249
+ # abandoned one (Phase 0.3) or land inside its open transaction.
250
+ def reset_connection
251
+ conn = Monk::Persistence::Pg[@db_name]
252
+ conn.cancel
253
+ conn.reset
254
+ nil
255
+ end
256
+
257
+ # Every job (payloads by cascade) and every process row. Behind
258
+ # Monk::Jobs.clear!, which only runs in tests.
259
+ def clear!
260
+ with_connection { |c| c.exec(CLEAR_SQL) }
261
+ nil
262
+ end
263
+
264
+ private
265
+
266
+ def with_connection(conn = nil, &)
267
+ conn ? yield(conn) : Monk::Persistence::Pg.checkout(@db_name, &)
268
+ end
269
+ end
270
+ end
271
+ end
272
+ end
@@ -0,0 +1,43 @@
1
+ module Monk
2
+ module Jobs
3
+ # A job's args are stored as JSON (monk_job_payloads.args) and handed
4
+ # back to #perform in another process, so only values that come back
5
+ # from that round trip unchanged are accepted: Strings, Integers,
6
+ # finite Floats, true/false/nil, and Arrays and String-keyed Hashes of
7
+ # those. Checked when a job is enqueued, so a bad argument fails in
8
+ # the code that passed it rather than later, in a worker.
9
+ module Args
10
+ def self.check!(args)
11
+ raise InvalidArgumentsError, "a job's args must be an Array, got #{args.class}" unless args.is_a?(Array)
12
+
13
+ args.each_with_index { |value, i| check_value!(value, "args[#{i}]") }
14
+ nil
15
+ end
16
+
17
+ def self.check_value!(value, path)
18
+ case value
19
+ when String, Integer, true, false, nil
20
+ nil
21
+ when Float
22
+ raise InvalidArgumentsError, "#{path} is #{value}, which JSON can't represent" unless value.finite?
23
+ when Array
24
+ value.each_with_index { |item, i| check_value!(item, "#{path}[#{i}]") }
25
+ when Hash
26
+ value.each do |key, item|
27
+ unless key.is_a?(String)
28
+ raise InvalidArgumentsError,
29
+ "#{path} has the key #{key.inspect} (a #{key.class}); JSON only has String keys, so " \
30
+ "#perform would get back a different Hash -- use #{key.to_s.inspect}"
31
+ end
32
+ check_value!(item, "#{path}[#{key.inspect}]")
33
+ end
34
+ else
35
+ raise InvalidArgumentsError,
36
+ "#{path} is a #{value.class}, which doesn't survive a JSON round trip -- pass plain values " \
37
+ "(e.g. an id rather than a record, an ISO 8601 String rather than a Time)"
38
+ end
39
+ end
40
+ private_class_method :check_value!
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,7 @@
1
+ module Monk
2
+ module Jobs
3
+ # A job a worker has claimed: what it needs to run it (job_class, args)
4
+ # and to decide what happens if it raises (attempts, counting this one).
5
+ Claim = Data.define(:id, :job_class, :args, :attempts)
6
+ end
7
+ end
@@ -0,0 +1,29 @@
1
+ module Monk
2
+ module Jobs
3
+ # Looking up a job class before Monk.freeze! (Monk.boot calls it) has
4
+ # sealed the registry -- from a worker Ractor the unfrozen registry
5
+ # couldn't even be read.
6
+ class NotFrozenError < StandardError
7
+ end
8
+
9
+ # A job_class name that isn't a Monk::Job subclass this process knows
10
+ # about: a job enqueued by a newer deploy, a renamed class, or a name
11
+ # that was never a job at all.
12
+ class UnknownJobError < StandardError
13
+ end
14
+
15
+ # Enqueueing (or claiming) before Monk::Jobs.configure has picked the
16
+ # database the queue lives in.
17
+ class NotConfiguredError < StandardError
18
+ end
19
+
20
+ # Monk::Jobs.clear! outside MONK_ENV=test: it would empty a real queue.
21
+ class ClearOutsideTestsError < StandardError
22
+ end
23
+
24
+ # An ArgumentError, like Monk::Mail::InvalidMessageError: it's always
25
+ # the caller's input that's wrong.
26
+ class InvalidArgumentsError < ArgumentError
27
+ end
28
+ end
29
+ end