monkrb 0.21.1 → 0.22.1

data/lib/monk/templates/auth/db/migrate/00000000000004_create_login_requests.down.sql

@@ -0,0 +1,2 @@
+DROP INDEX sessions_subject;
+DROP TABLE login_requests;

data/lib/monk/templates/auth/db/migrate/00000000000004_create_login_requests.up.sql

@@ -0,0 +1,16 @@
+-- Monk::Auth housekeeping (Monk 0.22): the login-email rate limit's table
+-- and an index for finding a person's sessions.
+
+-- One row per address: when a login link was last requested for it, for
+-- Monk::Auth.allow_login_request?. Checked in the login route, before the
+-- link is sent or enqueued: with jobs the login token only exists once the
+-- job runs, so login_tokens can't tell a request is already queued.
+-- Monk::Auth.sweep! deletes rows older than login_request_interval.
+CREATE TABLE login_requests (
+  email TEXT PRIMARY KEY, -- stripped and downcased
+  requested_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- A person's sessions by subject: Monk::Auth.revoke_all, or who is online.
+-- An app that already has an index on sessions (subject) can drop this line.
+CREATE INDEX sessions_subject ON sessions (subject);

data/lib/monk/templates/auth/setup.md

@@ -1,8 +1,9 @@
 ## auth
 
 Logging in by email link. `config/auth.rb` configures Monk::Auth,
-`app/mailers/auth_mailer.rb` sends the link through mail, and a migration
-creates the `login_tokens` and `sessions` tables: run `bin/setup_db` and
+`app/mailers/auth_mailer.rb` sends the link through mail, and two
+migrations create the `login_tokens`, `sessions` and `login_requests`
+tables: run `bin/setup_db` and
 `DB_NAME={{app}}_test bin/setup_db` (postgres, above).
 
 ### Before production
@@ -12,6 +13,17 @@ one in production's environment, e.g. from
 `ruby -rsecurerandom -e 'puts SecureRandom.hex(32)'`. `config/auth.rb`
 reads it with `ENV.fetch`: unset, the app doesn't boot.
 
+### Housekeeping
+
+- **Rate limit.** The login route allows one email per address per
+  minute (`login_request_interval:` in `config/auth.rb`), counted in
+  `login_requests`, so across every `bin/server` process. A limit across
+  addresses (per IP, say) is up to the app.
+- **Cleanup.** With jobs (`monk add jobs`), `bin/jobs` deletes expired
+  sessions and login tokens every night (`monk_auth_sweep` in
+  `config/recurring.rb`). Without jobs, run `Monk::Auth.sweep!` on a
+  schedule of your own.
+
 ### Check it works
 
 1. `bundle exec rake test` runs `test/auth_test.rb`: a link is sent (to
@@ -33,5 +45,6 @@ reads it with `ENV.fetch`: unset, the app doesn't boot.
      ```
 
      and `/me` answers `401` again.
-3. Before going live: keep the rate limit, and list in `config/auth.rb`
+3. Before going live: keep the rate limit (the `allow_login_request?`
+   line in `app/routes/auth.rb`), and list in `config/auth.rb`
    (`redirect_allowlist`) the paths a login may send the user back to.

data/lib/monk/templates/auth/test/auth_test.rb

@@ -4,6 +4,12 @@ require_relative "test_helper"
 # sent by config/auth.rb's deliver: (AuthMailer, to log/test.log here), and
 # redeemed into a session. SETUP.md, auth.
 class AuthTest < Minitest::Test
+  # Lifts the login-email rate limit, so the next test can ask for a link
+  # for the same address at once.
+  def teardown
+    Monk::Auth.clear_login_requests!
+  end
+
   def test_a_login_link_is_sent_and_redeems_into_a_session
     email = "login-check-#{rand(1_000_000)}@example.test"
     token = Monk::Auth.request_login(email)

data/lib/monk/templates/base/Gemfile

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
-gem "monkrb", "~> 0.21", require: "monk"
+gem "monkrb", "~> 0.22", require: "monk"
 gem "rackup"
 gem "webrick"
 gem "kino"

data/lib/monk/templates/base/config/load.rb

@@ -25,6 +25,11 @@ end
   Dir[File.expand_path("../app/#{role}/**/*.rb", __dir__)].sort.each { |file| require file }
 end
 
+# With jobs (monk add jobs), the schedules of recurring jobs. Last, because
+# each one names a job class from app/jobs/.
+recurring = File.expand_path("recurring.rb", __dir__)
+require recurring if File.exist?(recurring)
+
 # With websocket (monk add websocket), every process that loads this file
 # can broadcast (bin/server, bin/jobs, bin/console, the tests), so its
 # transport is started here, once everything above has declared its

data/lib/monk/templates/jobs/agents.md

@@ -1,18 +1,25 @@
 ## jobs
 
 **Where:** `config/jobs.rb` (the queue lives in the `:primary` database),
-job classes in `app/jobs/`, `bin/jobs` runs them.
+job classes in `app/jobs/`, schedules in `config/recurring.rb`, `bin/jobs`
+runs them.
 
 **Main calls:** `class SendInvoice < Monk::Job` with
 `def self.perform(id)`; `SendInvoice.enqueue(id)`, `wait: 300`, `at: time`,
 or `conn: conn` to enqueue inside your own transaction. Class settings:
 `queue "mailers"`, `max_attempts 5`, `never_retry SomeError`. With mail:
 `Monk::Mail.deliver_later(...)`; with auth:
-`Monk::Auth::SendLoginLink.enqueue(email)`.
+`Monk::Auth::SendLoginLink.enqueue(email)`. Recurring, in
+`config/recurring.rb` only:
+`Monk::Jobs.recurring "key", JobClass, cron: "0 3 * * *", args: [...]`
+(`skip_if_pending: true` to skip while the last run is pending). With
+auth, `config/recurring.rb` already schedules `monk_auth_sweep`.
 
 **Test:** `test/jobs_test.rb`. In tests, `Monk::Jobs.drain!` runs every
 enqueued job right away, each in a Ractor; `Monk::Jobs.clear!` in
-`teardown` empties the queue.
+`teardown` empties the queue. `Monk::Jobs.run_scheduler!(at: time)` runs
+the schedules as if the clock read `time` (call it once to set the
+checkpoint, then at the next occurrence).
 
 **Pitfalls:**
 - A job's arguments are stored as JSON: pass ids and plain values, never
@@ -20,6 +27,10 @@ enqueued job right away, each in a Ractor; `Monk::Jobs.clear!` in
 - A job may run more than once (its process was killed mid-job): make it
   safe to repeat.
 - Jobs run in Ractors: the same shareability rules as routes.
+- Cron is UTC. A new recurring task first fires at its next occurrence,
+  not on deploy, and a missed run fires once, not once per occurrence
+  missed. Keep a task's key when renaming its class.
 
-**Examples:** `jobs-enqueue` in `app/routes/jobs.rb`; `app/jobs/hello_job.rb`
+**Examples:** `jobs-enqueue` in `app/routes/jobs.rb`, `jobs-recurring` in
+`config/recurring.rb`; `app/jobs/hello_job.rb`
 is a real, deletable demo job.

data/lib/monk/templates/jobs/config/recurring.rb

@@ -0,0 +1,21 @@
+# Recurring jobs: bin/jobs enqueues each one on its schedule, once per
+# occurrence however many job processes are running. Loaded after app/,
+# since each names a job class.
+#
+#   Monk::Jobs.recurring "<key>", JobClass, cron: "<expression>", args: [...]
+#
+# The key is the task's identity in the database: keep it when renaming
+# the class. cron: is a five-field cron expression (minute hour
+# day-of-month month day-of-week, or @hourly, @daily, ...), always in UTC.
+# A new task first runs at its next occurrence, not on deploy; after
+# downtime only the latest missed occurrence runs.
+# skip_if_pending: true skips an occurrence while the previous run hasn't
+# finished. Monk's docs/guides/jobs.md, "Recurring jobs".
+
+# With auth (monk add auth): deletes expired sessions and login tokens, and
+# old login requests, every night at 03:17 UTC.
+Monk::Jobs.recurring "monk_auth_sweep", Monk::Auth::Sweep, cron: "17 3 * * *" if defined?(Monk::Auth::Sweep)
+
+# monk:example jobs-recurring -- the demo job (app/jobs/hello_job.rb) at the top of every hour.
+# Monk::Jobs.recurring "hello_hourly", HelloJob, cron: "@hourly", args: ["cron"]
+# monk:end

data/lib/monk/templates/jobs/db/migrate/00000000000003_create_recurring_tables.down.sql

@@ -0,0 +1 @@
+DROP TABLE monk_recurring_tasks;

data/lib/monk/templates/jobs/db/migrate/00000000000003_create_recurring_tables.up.sql

@@ -0,0 +1,13 @@
+-- Monk::Jobs.recurring: one checkpoint row per declared task, the latest
+-- occurrence it has been handled for. Why this shape:
+-- docs/adr/0016-recurring-jobs-on-the-job-process-tick-with-a-checkpoint-row.md.
+CREATE TABLE monk_recurring_tasks (
+  key TEXT PRIMARY KEY, -- the key given to Monk::Jobs.recurring
+  cron TEXT NOT NULL, -- the expression the checkpoint was set under
+  last_occurrence_at TIMESTAMPTZ NOT NULL, -- the checkpoint
+  -- The job enqueued for the checkpoint, for skip_if_pending. No foreign
+  -- key: ON DELETE SET NULL would add a write to every job's finish, and
+  -- a dangling id just means that job is done.
+  last_job_id BIGINT,
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);

data/lib/monk/templates/jobs/setup.md

@@ -1,7 +1,7 @@
 ## jobs
 
-Background jobs on the app's own Postgres. A migration creates the queue's
-tables: run `bin/setup_db` and `DB_NAME={{app}}_test bin/setup_db`
+Background jobs on the app's own Postgres. Two migrations create the
+queue's tables and the recurring jobs' one: run `bin/setup_db` and `DB_NAME={{app}}_test bin/setup_db`
 (postgres, above).
 
 ### Run them
@@ -18,6 +18,19 @@ first. `TERM` or Ctrl-C lets the jobs in hand finish before it exits. In
 production it's one more process from the same image, with `bin/jobs` as
 its command.
 
+### Recurring jobs
+
+Declare schedules in `config/recurring.rb`:
+
+```ruby
+Monk::Jobs.recurring "purge_sessions", PurgeSessions, cron: "0 3 * * *"
+```
+
+Cron times are UTC. A running `bin/jobs` enqueues each occurrence once,
+however many are running; with none running, nothing fires, and on
+restart only the latest missed occurrence runs. A new task first runs at
+its next occurrence, not on deploy.
+
 ### Check it works
 
 1. `bundle exec rake test` runs `test/jobs_test.rb`, which enqueues the
@@ -28,3 +41,8 @@ its command.
 3. The example: uncomment the `jobs-enqueue` block in `app/routes/jobs.rb`,
    restart `bin/server`, and
    `curl -X POST "http://localhost:9292/jobs/hello?name=Bob"`.
+4. A recurring job, in a test: `Monk::Jobs.run_scheduler!(at: time)` runs
+   the schedule as if the clock read `time`. The first call only records
+   where the task is; a second call at the next occurrence enqueues it,
+   and `Monk::Jobs.drain!` runs it. The `jobs-recurring` example in
+   `config/recurring.rb` schedules the demo job every hour.

data/lib/monk/templates/websocket/bin/websocket_server

@@ -4,6 +4,19 @@ require "bundler/setup"
 # with what to fix (SETUP.md's commands), not a backtrace.
 require "monk/bin"
 
+# Fewer GC pauses while thousands of sockets are open, set the only way
+# Ruby takes them: at boot. Any already in the environment win.
+Monk::Bin.boot_with_env!(__FILE__, Monk::Bin::WEBSOCKET_GC_ENV)
+
+# WS_PROCESSES (1): this many server processes on WS_PORT, each with its own
+# heap and its own GC pauses over a share of the sockets
+# (docs/guides/websocket.md, "Several processes").
+processes = Integer(ENV.fetch("WS_PROCESSES", "1"), exception: false)
+unless processes&.positive?
+  abort "bin/websocket_server: WS_PROCESSES must be a whole number, 1 or more, got #{ENV["WS_PROCESSES"].inspect}."
+end
+Monk::Bin.spawn_processes!(__FILE__, processes)
+
 Monk::Bin.run(__FILE__) do
   require_relative "../config/settings"
   require_relative "../config/websocket"
@@ -55,6 +68,7 @@ Monk::Bin.run(__FILE__) do
 
   server = Monk::WebSocket::Server.new(
     port: port, authenticate: authenticate, allowed_origins: allowed_origins, db_pool: db_pool,
+    reuse_port: processes > 1,
   )
   puts "Monk::WebSocket::Server listening on ws://0.0.0.0:#{port} " \
     "(#{live ? "live" : "app/sockets"}, transport: #{AppWebSocket::TRANSPORT}, authenticate: #{authenticate})"

data/lib/monk/templates/websocket/setup.md

@@ -18,7 +18,8 @@ exits. In production it's one more process from the same image, with
 
 `WS_ALLOWED_ORIGINS` (default: `PUBLIC_URL`) must list every origin your
 pages are served from, or browsers' sockets are refused. Behind TLS, pages
-connect with `wss://`.
+connect with `wss://`. Past ~5K open sockets, `WS_PROCESSES` (default 1)
+runs several server processes on the port, on Linux.
 
 ### Switching transports
 

data/lib/monk/version.rb

@@ -5,5 +5,5 @@ module Monk
   # class of bug .freeze! guards against for routes/error handlers/models,
   # just on a top-level constant that every worker Ractor reads on boot
   # (found live under kino: GET /hello 500'd until this was frozen).
-  VERSION = "0.21.1".freeze
+  VERSION = "0.22.1".freeze
 end

data/lib/monk/websocket/handshake.rb

@@ -9,16 +9,35 @@ module Monk
       # read from a connection Ractor. .freeze is not optional.
       MAGIC = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11".freeze
 
+      # The most a handshake request's head may be (Server.serve answers
+      # 431 past it): room for a large cookie, nowhere near a memory risk.
+      MAX_REQUEST_SIZE = 16 * 1024
+
       def self.accept_key(client_key)
         Base64.strict_encode64(Digest::SHA1.digest(client_key + MAGIC))
       end
 
+      # Two Strings a header line (its name and its value), plus the line:
+      # this runs once a connect, and every object it makes is one more
+      # towards the next GC (docs/plan-ws-connect-ceiling.md, item 4).
       def self.parse_request(raw_request)
-        headers = raw_request.split("\r\n").drop(1).each_with_object({}) do |line, h|
-          next if line.empty?
+        headers = {}
+        request_line = true
+        raw_request.each_line("\r\n", chomp: true) do |line|
+          if request_line
+            request_line = false
+            next
+          end
+
+          colon = line.index(":")
+          next unless colon
 
-          name, value = line.split(":", 2)
-          h[name.strip.downcase] = value.strip if name && value
+          name = line.byteslice(0, colon)
+          name.strip!
+          name.downcase!
+          value = line.byteslice(colon + 1, line.bytesize)
+          value.strip!
+          headers[name] = value
         end
 
         unless headers["upgrade"]&.downcase == "websocket"
@@ -55,17 +74,21 @@ module Monk
       end
       private_class_method :bearer_token
 
+      # The last session_token in the header wins, as when it was a Hash.
       def self.session_cookie_token(headers)
-        headers["cookie"].to_s.split(";").each_with_object({}) do |pair, cookies|
+        token = nil
+        headers["cookie"]&.split(";")&.each do |pair|
           name, value = pair.strip.split("=", 2)
-          cookies[name] = value if name
-        end["session_token"]
+          token = value if name == "session_token"
+        end
+        token
       end
       private_class_method :session_cookie_token
 
-      def self.response_for(raw_request)
-        headers = parse_request(raw_request)
+      def self.response_for(raw_request) = response(parse_request(raw_request))
 
+      # The 101 for headers parse_request already returned.
+      def self.response(headers)
         "HTTP/1.1 101 Switching Protocols\r\n" \
         "Upgrade: websocket\r\n" \
         "Connection: Upgrade\r\n" \

data/lib/monk/websocket/pg_fanout.rb

@@ -225,17 +225,17 @@ module Monk
       end
 
       def register(key, port)
-        unless Listeners.listening?(@origin)
-          raise NotListeningError,
-            "Monk::WebSocket::PgFanout#register before #listen!: this socket would never get " \
-            "other processes' broadcasts -- call listen! once at boot in the process that holds " \
-            "sockets (bin/websocket_server: Monk::Live.listen!)"
-        end
-
+        ensure_listening!(:register)
         @registry.register(key, port)
       end
 
+      def register_many(keys, port)
+        ensure_listening!(:register_many)
+        @registry.register_many(keys, port)
+      end
+
       def unregister(key, port) = @registry.unregister(key, port)
+      def unregister_many(keys, port) = @registry.unregister_many(keys, port)
       def count(key) = @registry.count(key)
 
       def broadcast(key, payload)
@@ -266,6 +266,15 @@ module Monk
 
       private
 
+      def ensure_listening!(method)
+        return if Listeners.listening?(@origin)
+
+        raise NotListeningError,
+          "Monk::WebSocket::PgFanout##{method} before #listen!: this socket would never get " \
+          "other processes' broadcasts -- call listen! once at boot in the process that holds " \
+          "sockets (bin/websocket_server: Monk::Live.listen!)"
+      end
+
       # The pool's database's connection options, for #listen!.
       def pooled_pg_opts(db_pool)
         config = Monk::Persistence::Pg.pool_config(db_pool)

data/lib/monk/websocket/redis_fanout.rb

@@ -161,17 +161,17 @@ module Monk
       end
 
       def register(key, port)
-        unless Listeners.listening?(@origin)
-          raise NotListeningError,
-            "Monk::WebSocket::RedisFanout#register before #listen!: this socket would never get " \
-            "other processes' broadcasts -- call listen! once at boot in the process that holds " \
-            "sockets (bin/websocket_server: Monk::Live.listen!)"
-        end
-
+        ensure_listening!(:register)
         @registry.register(key, port)
       end
 
+      def register_many(keys, port)
+        ensure_listening!(:register_many)
+        @registry.register_many(keys, port)
+      end
+
       def unregister(key, port) = @registry.unregister(key, port)
+      def unregister_many(keys, port) = @registry.unregister_many(keys, port)
       def count(key) = @registry.count(key)
 
       def broadcast(key, payload)
@@ -181,6 +181,15 @@ module Monk
 
       private
 
+      def ensure_listening!(method)
+        return if Listeners.listening?(@origin)
+
+        raise NotListeningError,
+          "Monk::WebSocket::RedisFanout##{method} before #listen!: this socket would never get " \
+          "other processes' broadcasts -- call listen! once at boot in the process that holds " \
+          "sockets (bin/websocket_server: Monk::Live.listen!)"
+      end
+
       # A single shared Redis client called concurrently from every
       # connection Ractor's #broadcast wouldn't be safe -- redis-rb clients
       # aren't documented safe for concurrent commands from multiple

data/lib/monk/websocket/registry.rb

@@ -8,8 +8,9 @@ module Monk
     # A dedicated Ractor holding the live set of connection handles keyed by
     # an app-assigned channel/subject -- mirrors StateRactor's shape
     # (docs/design/ractor.md): the mutable Hash stays hidden inside this one
-    # Ractor, and every call is a synchronous "ask" via a fresh
-    # Ractor::Port, so registration/broadcast can never race each other.
+    # Ractor, and every call is a synchronous "ask" answered on the
+    # caller's reply port, so registration/broadcast can never race each
+    # other.
     class Registry
       def initialize
         @ractor = Ractor.new do
@@ -25,6 +26,14 @@ module Monk
               when :unregister
                 keys[key].delete(arg)
                 true
+              when :register_many
+                # key is the Array of keys: a Live subscribe's topics in
+                # one ask, not one each.
+                key.each { |k| keys[k] << arg unless keys[k].include?(arg) }
+                true
+              when :unregister_many
+                key.each { |k| keys[k].delete(arg) }
+                true
               when :broadcast
                 # A port can be closed out from under this loop without
                 # ever going through #unregister -- e.g. the owning
@@ -58,7 +67,13 @@ module Monk
                 end
                 ports.size
               end
-            reply_port.send(result)
+            begin
+              reply_port.send(result)
+            rescue Ractor::ClosedError
+              # The caller gave up on this ask (its thread was killed while
+              # it waited) and closed its port: nobody to answer, and no
+              # reason to end the registry for every other caller.
+            end
           end
         end
         freeze
@@ -72,6 +87,16 @@ module Monk
         ask(:unregister, key, port)
       end
 
+      # register for each of keys, in one ask: one message and one reply
+      # instead of one each (docs/plan-ws-connect-ceiling.md, item 5).
+      def register_many(keys, port)
+        ask(:register_many, keys, port)
+      end
+
+      def unregister_many(keys, port)
+        ask(:unregister_many, keys, port)
+      end
+
       def broadcast(key, payload)
         ask(:broadcast, key, payload)
       end
@@ -99,10 +124,20 @@ module Monk
 
       private
 
+      # An ask is synchronous, so a fiber has at most one outstanding: its
+      # reply port is made once and reused (a Port is an allocation and a
+      # message each), taken out of fiber storage while the ask is in
+      # flight. An ask cut short (the thread killed while it waits) closes
+      # the port instead of putting it back, so its late reply can't be
+      # read as the next ask's.
       def ask(op, key, arg)
-        reply_port = Ractor::Port.new
+        reply_port = Thread.current[:monk_registry_reply_port] || Ractor::Port.new
+        Thread.current[:monk_registry_reply_port] = nil
         @ractor.send([op, key, arg, reply_port])
-        reply_port.receive
+        result = reply_port.receive
+        Thread.current[:monk_registry_reply_port] = reply_port
+        reply_port = nil
+        result
       rescue Ractor::ClosedError
         # The registry Ractor itself can be gone by the time this runs --
         # e.g. process-wide shutdown (Ctrl+C) tearing down Ractors in