monkrb 0.19.0 → 0.21.0

data/lib/monk/templates/postgres/bin/migrate

@@ -1,22 +1,28 @@
 #!/usr/bin/env ruby
 require "bundler/setup"
-require_relative "../config/settings"
-require_relative "../config/persistence"
-require "monk/persistence/pg/migrator"
+# A service that's down, a missing database or setting ends this script
+# with what to fix (SETUP.md's commands), not a backtrace.
+require "monk/bin"
 
-migrator = Monk::Persistence::Pg::Migrator.new(db_name: :primary, dir: File.expand_path("../db/migrate", __dir__))
+Monk::Bin.run(__FILE__) do
+  require_relative "../config/settings"
+  require_relative "../config/persistence"
+  require "monk/persistence/pg/migrator"
 
-case ARGV[0]
-when nil, "migrate"
-  applied = migrator.migrate!
-  puts applied.empty? ? "Nothing to migrate." : "Applied: #{applied.join(", ")}"
-when "rollback"
-  reverted = migrator.rollback!(steps: (ARGV[1] || 1).to_i)
-  puts reverted.empty? ? "Nothing to roll back." : "Reverted: #{reverted.join(", ")}"
-when "status"
-  puts "Applied: #{migrator.applied.join(", ")}"
-  puts "Pending: #{migrator.pending.join(", ")}"
-else
-  warn "Usage: bin/migrate [migrate|rollback [N]|status]"
-  exit 1
+  migrator = Monk::Persistence::Pg::Migrator.new(db_name: :primary, dir: File.expand_path("../db/migrate", __dir__))
+
+  case ARGV[0]
+  when nil, "migrate"
+    applied = migrator.migrate!
+    puts applied.empty? ? "Nothing to migrate." : "Applied: #{applied.join(", ")}"
+  when "rollback"
+    reverted = migrator.rollback!(steps: (ARGV[1] || 1).to_i)
+    puts reverted.empty? ? "Nothing to roll back." : "Reverted: #{reverted.join(", ")}"
+  when "status"
+    puts "Applied: #{migrator.applied.join(", ")}"
+    puts "Pending: #{migrator.pending.join(", ")}"
+  else
+    warn "Usage: bin/migrate [migrate|rollback [N]|status]"
+    exit 1
+  end
 end

data/lib/monk/templates/postgres/bin/setup_db

@@ -1,9 +1,15 @@
 #!/usr/bin/env ruby
 require "bundler/setup"
-require_relative "../config/settings"
-require_relative "../config/persistence"
-require "monk/persistence/pg/migrator"
+# A service that's down, a missing database or setting ends this script
+# with what to fix (SETUP.md's commands), not a backtrace.
+require "monk/bin"
 
-Monk::Persistence::Pg::Migrator.new(db_name: :primary, dir: File.expand_path("../db/migrate", __dir__)).migrate!
+Monk::Bin.run(__FILE__) do
+  require_relative "../config/settings"
+  require_relative "../config/persistence"
+  require "monk/persistence/pg/migrator"
 
-puts "Migrated #{ENV.fetch("DB_NAME", "app_development")}."
+  Monk::Persistence::Pg::Migrator.new(db_name: :primary, dir: File.expand_path("../db/migrate", __dir__)).migrate!
+
+  puts "Migrated #{ENV.fetch("DB_NAME", "app_development")}."
+end

data/lib/monk/templates/postgres/config/persistence.rb

@@ -1,5 +1,6 @@
 require "monk"
 require "monk/persistence/pg"
+require "monk/persistence/pg/model" # Monk::Persistence::Pg::Model, for app/models/
 
 Monk::Persistence::Pg.register(:primary,
   host: ENV.fetch("DB_HOST", "127.0.0.1"),

data/lib/monk/templates/postgres/setup.md

@@ -0,0 +1,38 @@
+## postgres
+
+`.env` and `.env.test` point at a local Postgres (`DB_*`), with database
+names taken from this app's directory: `{{app}}_development` and
+`{{app}}_test`. `MONK_ENV=test` doesn't change the database: only `DB_NAME`
+does, and `test/test_helper.rb` takes it from `.env.test`.
+
+### Start Postgres
+
+Check first whether one is already running (`docker ps`): a second
+container on the same port fails with "port is already allocated". Either
+point `DB_HOST`/`DB_PORT`/`DB_USER`/`DB_PASSWORD` in `.env` and `.env.test`
+at the running one, or start one:
+
+```bash
+docker run --rm -d -p 5432:5432 -e POSTGRES_PASSWORD=postgres --name {{app}}_pg postgres:16
+```
+
+### Create the databases and migrate them
+
+```bash
+PGPASSWORD=postgres createdb -h 127.0.0.1 -p 5432 -U postgres {{app}}_development
+PGPASSWORD=postgres createdb -h 127.0.0.1 -p 5432 -U postgres {{app}}_test
+bin/setup_db
+DB_NAME={{app}}_test bin/setup_db
+```
+
+`createdb` without `-h` uses a Unix socket, which a container doesn't
+provide. Run both `bin/setup_db` lines again after adding a migration.
+
+### Check it works
+
+1. `bundle exec rake test` runs `test/persistence_test.rb`, which connects
+   to the test database.
+2. By hand: `bin/console` opens IRB with the app's configs and models
+   loaded; `Monk::Persistence::Pg.checkout(:primary) { |conn| conn.exec("SELECT 1").values }`
+   answers `[[1]]`. `bin/migrate status` lists applied and pending
+   migrations.

data/lib/monk/templates/postgres/test/persistence_test.rb

@@ -0,0 +1,11 @@
+require_relative "test_helper"
+
+# The test database is reachable with .env.test's settings, and migrated
+# (SETUP.md, postgres).
+class PersistenceTest < Minitest::Test
+  def test_connects_to_the_test_database
+    Monk::Persistence::Pg.checkout(:primary) do |conn|
+      assert_equal 1, conn.exec("SELECT 1").getvalue(0, 0)
+    end
+  end
+end

data/lib/monk/templates/redis/agents.md

@@ -0,0 +1,18 @@
+## redis
+
+**Where:** `config/redis.rb` declares the `redis_url` setting (from
+`REDIS_URL`) and loads the `redis` gem.
+
+**Main calls:** `Redis.new(url: settings[:redis_url])` in a route (or
+`Monk::Settings[:redis_url]` elsewhere), then the redis gem's commands:
+`get`, `set(key, value, ex: seconds)`, `incr`, `del`. Close it when done.
+
+**Test:** `test/redis_test.rb` pings Redis at `.env.test`'s `REDIS_URL`.
+
+**Pitfalls:**
+- A Redis client isn't Ractor-shareable: never keep one in a constant or a
+  module variable. Open it in the route (or job) that uses it.
+- Monk has no Redis pool: each `Redis.new` is a connection. Close it.
+- `REDIS_URL` is required outside development and test.
+
+**Examples:** `redis-cache` in `app/routes/redis.rb`.

data/lib/monk/templates/redis/app/routes/redis.rb

@@ -0,0 +1,15 @@
+# Routes that use the app's Redis (monk add redis).
+#
+# monk:example redis-cache -- a value computed once a minute, cached in Redis.
+# # A Redis client can't be shared between Ractors, and routes run in them,
+# # so each request opens its own; Monk has no Redis pool yet.
+# class App
+#   get("/cached/now") do
+#     redis = Redis.new(url: settings[:redis_url])
+#     now = redis.get("cached:now") || Time.now.to_i.to_s.tap { |value| redis.set("cached:now", value, ex: 60) }
+#     json(now: now)
+#   ensure
+#     redis&.close
+#   end
+# end
+# monk:end

data/lib/monk/templates/redis/config/redis.rb

@@ -0,0 +1,13 @@
+require "monk"
+require "redis"
+
+# Where this app's Redis is (monk add redis): for a cache, rate limits,
+# locks, or as WebSocket's transport (config/websocket.rb). A local one by
+# default in development and test; anywhere else REDIS_URL must be set.
+Monk::Settings.configure do
+  if Monk.env.development? || Monk.env.test?
+    optional :redis_url, default: "redis://localhost:6379/0"
+  else
+    required :redis_url
+  end
+end

data/lib/monk/templates/redis/setup.md

@@ -0,0 +1,20 @@
+## redis
+
+`REDIS_URL` in `.env` and `.env.test` points at a local Redis
+(`redis://localhost:6379/0`). Outside development and test it has no
+default: set it, or the app fails at boot.
+
+### Start Redis
+
+Check first whether one is already running (`docker ps`). Either point
+`REDIS_URL` at it, or start one:
+
+```bash
+docker run --rm -d -p 6379:6379 --name {{app}}_redis redis:7
+```
+
+### Check it works
+
+1. `bundle exec rake test` runs `test/redis_test.rb`, which pings it.
+2. By hand, in `bin/console`: `Redis.new(url: Monk::Settings[:redis_url]).ping`
+   answers `"PONG"`.

data/lib/monk/templates/redis/test/redis_test.rb

@@ -0,0 +1,12 @@
+require_relative "test_helper"
+
+# Redis is reachable at REDIS_URL (.env.test): SETUP.md, redis.
+class RedisTest < Minitest::Test
+  def test_redis_answers
+    redis = Redis.new(url: Monk::Settings[:redis_url])
+
+    assert_equal "PONG", redis.ping
+  ensure
+    redis&.close
+  end
+end

data/lib/monk/templates/websocket/agents.md

@@ -0,0 +1,25 @@
+## websocket
+
+**Where:** `config/websocket.rb` (`AppWebSocket::REGISTRY`, over the app's
+transport: `AppWebSocket::TRANSPORT`), `bin/websocket_server` (the socket
+process), `app/sockets/` (handlers, when the app doesn't use live).
+
+**Main calls:** `AppWebSocket::REGISTRY.broadcast(topic, payload)` from any
+process. In a handler, `proc { |connection| ... }`: `connection.read`,
+`connection.subscribe(AppWebSocket::REGISTRY, topic)`,
+`connection.subject` (the user, with auth). The server runs
+`AppSockets::HANDLER`, or live's.
+
+**Test:** `test/websocket_test.rb` sends a broadcast across two registries
+over the transport.
+
+**Pitfalls:**
+- Handlers run in each connection's own Ractor: build them in a module
+  body, never at a file's top level.
+- Only `bin/websocket_server` calls `listen!`; every other process only
+  publishes.
+- With Postgres, a broadcast is capped at just under 8000 bytes (NOTIFY).
+- `WS_ALLOWED_ORIGINS` must list your pages' origins.
+- Switching transports is three steps in SETUP.md, websocket.
+
+**Examples:** `websocket-chat` in `app/sockets/chat.rb`.

data/lib/monk/templates/websocket/app/sockets/chat.rb

@@ -0,0 +1,18 @@
+# Socket handlers for bin/websocket_server, in an app without Monk::Live
+# (with live, the server runs live's handler instead).
+#
+# monk:example websocket-chat -- one chat room: what a socket sends goes to every socket.
+# # The handler runs in each connection's own Ractor, so it's built in a
+# # module body, where self is shareable.
+# module AppSockets
+#   HANDLER = proc do |connection|
+#     connection.subscribe(AppWebSocket::REGISTRY, :chat)
+#     loop do
+#       message = connection.read
+#       break unless message
+#
+#       AppWebSocket::REGISTRY.broadcast(:chat, "#{connection.subject || "guest"}: #{message}")
+#     end
+#   end
+# end
+# monk:end

data/lib/monk/templates/websocket/bin/websocket_server

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+require "bundler/setup"
+# A service that's down, a missing database or setting ends this script
+# with what to fix (SETUP.md's commands), not a backtrace.
+require "monk/bin"
+
+Monk::Bin.run(__FILE__) do
+  require_relative "../config/settings"
+  require_relative "../config/websocket"
+
+  # Each only if this app has it: auth says whose a socket is, live what it
+  # may subscribe to.
+  %w[auth live].each do |name|
+    path = File.expand_path("../config/#{name}.rb", __dir__)
+    require path if File.exist?(path)
+  end
+  live = defined?(Monk::Live) && Monk::Live.registry
+
+  # Without live, the app's own handler: AppSockets::HANDLER, in app/sockets/.
+  Dir[File.expand_path("../app/sockets/*.rb", __dir__)].sort.each { |file| require file }
+  handler = live ? Monk::Live::HANDLER : (AppSockets::HANDLER if defined?(AppSockets::HANDLER))
+  unless handler
+    $stderr.puts "bin/websocket_server: nothing to serve. Add live (monk add live), or define " \
+      "AppSockets::HANDLER in app/sockets/ -- the websocket-chat example in app/sockets/chat.rb is one."
+    exit 1
+  end
+
+  # This process holds the sockets, so it's the only one that listens for
+  # what the others publish: the transport's subscriber connection opens
+  # here, and an unreachable Postgres or Redis fails right here.
+  AppWebSocket::REGISTRY.listen!
+
+  # With auth, a socket carries its user's session. With live, a visitor's
+  # socket comes in anonymous instead of refused, and live's rules
+  # (config/live.rb) decide what it may subscribe to.
+  authenticate = if defined?(Monk::Auth) && Monk::Auth.config
+    live ? :optional : true
+  else
+    false
+  end
+  # Sessions are checked through config/auth.rb's :auth pool, not in each
+  # socket's own Ractor, which would hold a Postgres connection per open
+  # page. This is the only process that uses the pool, so it starts it.
+  db_pool = (:auth if authenticate)
+  Monk::Persistence::Pg.start_pools!(db_pool) if db_pool
+
+  port = ENV.fetch("WS_PORT", "9293").to_i
+  # Defaults to public_url (config/settings.rb): one PUBLIC_URL keeps both
+  # in sync. More than one origin: comma-separated.
+  allowed_origins = ENV.fetch("WS_ALLOWED_ORIGINS", Monk::Settings[:public_url]).split(",")
+
+  server = Monk::WebSocket::Server.new(
+    port: port, authenticate: authenticate, allowed_origins: allowed_origins, db_pool: db_pool,
+  )
+  puts "Monk::WebSocket::Server listening on ws://0.0.0.0:#{port} " \
+    "(#{live ? "live" : "app/sockets"}, transport: #{AppWebSocket::TRANSPORT}, authenticate: #{authenticate})"
+  server.run(&handler)
+end

data/lib/monk/templates/websocket/config/websocket_postgres.rb

@@ -0,0 +1,24 @@
+require "monk"
+require "monk/websocket"
+require "monk/websocket/pg_fanout"
+require_relative "persistence"
+
+# The WebSocket registry (monk add websocket): which sockets are subscribed
+# to which topics, and how a broadcast reaches them. bin/server, bin/jobs
+# and bin/websocket_server are separate processes, so Postgres
+# LISTEN/NOTIFY carries a broadcast from one to the others, over the
+# :primary connection's settings (config/persistence.rb). A broadcast is
+# capped at just under 8000 bytes, Postgres's NOTIFY limit. To switch to
+# Redis: SETUP.md, websocket.
+module AppWebSocket
+  TRANSPORT = "postgres".freeze
+
+  # A registry over this app's transport. bin/websocket_server's (REGISTRY)
+  # also listens; every other process only publishes.
+  def self.build_registry
+    Monk::WebSocket::PgFanout.new(Monk::WebSocket::Registry.new,
+      pg_opts: Monk::Persistence::Pg.connection_options(:primary),)
+  end
+
+  REGISTRY = build_registry
+end

data/lib/monk/templates/websocket/config/websocket_redis.rb

@@ -0,0 +1,21 @@
+require "monk"
+require "monk/websocket"
+require "monk/websocket/redis_fanout"
+require_relative "redis"
+
+# The WebSocket registry (monk add websocket): which sockets are subscribed
+# to which topics, and how a broadcast reaches them. bin/server, bin/jobs
+# and bin/websocket_server are separate processes, so Redis carries a
+# broadcast from one to the others, at REDIS_URL (config/redis.rb). To
+# switch to Postgres: SETUP.md, websocket.
+module AppWebSocket
+  TRANSPORT = "redis".freeze
+
+  # A registry over this app's transport. bin/websocket_server's (REGISTRY)
+  # also listens; every other process only publishes.
+  def self.build_registry
+    Monk::WebSocket::RedisFanout.new(Monk::WebSocket::Registry.new, redis_url: Monk::Settings[:redis_url])
+  end
+
+  REGISTRY = build_registry
+end

data/lib/monk/templates/websocket/setup.md

@@ -0,0 +1,51 @@
+## websocket
+
+`bin/websocket_server` holds the browsers' WebSocket connections, on :9293
+(`WS_PORT`), next to `bin/server`. They're separate processes, so a
+broadcast travels between them over this app's transport, chosen when the
+module was added: `config/websocket.rb` says which.
+
+### Run it
+
+```bash
+bin/websocket_server
+```
+
+It serves Monk::Live (`monk add live`) or, without it, the handler in
+`app/sockets/` (the `websocket-chat` example). With neither, it says so and
+exits. In production it's one more process from the same image, with
+`bin/websocket_server` as its command.
+
+`WS_ALLOWED_ORIGINS` (default: `PUBLIC_URL`) must list every origin your
+pages are served from, or browsers' sockets are refused. Behind TLS, pages
+connect with `wss://`.
+
+### Switching transports
+
+1. If the other one isn't installed, add it: `monk add redis` or
+   `monk add postgres`.
+2. In `config/websocket.rb`, swap the fanout: `Monk::WebSocket::RedisFanout`
+   with `redis_url: Monk::Settings[:redis_url]` (and `require_relative
+   "redis"`), or `Monk::WebSocket::PgFanout` with
+   `pg_opts: Monk::Persistence::Pg.connection_options(:primary)` (and
+   `require_relative "persistence"`).
+3. Restart `bin/server`, `bin/websocket_server` and `bin/jobs`.
+
+### Check it works
+
+1. `bundle exec rake test` runs `test/websocket_test.rb`: a broadcast
+   crosses from one registry to another over the transport.
+2. By hand, without live: uncomment the `websocket-chat` block in
+   `app/sockets/chat.rb` and start `bin/websocket_server` next to
+   `bin/server`. Open http://localhost:9292 in two tabs, and in each
+   browser console:
+
+   ```js
+   ws = new WebSocket("ws://localhost:9293")
+   ws.onmessage = (event) => console.log(event.data)
+   ```
+
+   `ws.send("hi")` in one tab logs `guest: hi` in both. With auth, the
+   server lets only logged-in sockets in: log in first (auth, above), and
+   the message carries your email. With live, `bin/websocket_server` runs
+   live's handler instead: live, below.

data/lib/monk/templates/websocket/test/websocket_test.rb

@@ -0,0 +1,21 @@
+require_relative "test_helper"
+require "timeout"
+
+# A broadcast crosses processes over this app's transport
+# (config/websocket.rb): a second registry, built the same way, publishes
+# as bin/server would, and AppWebSocket::REGISTRY, listening as
+# bin/websocket_server does, delivers it. SETUP.md, websocket.
+class WebSocketTest < Minitest::Test
+  def test_a_broadcast_reaches_a_socket_through_the_transport
+    AppWebSocket::REGISTRY.listen!
+    port = Ractor::Port.new
+    AppWebSocket::REGISTRY.register(:websocket_check, port)
+
+    AppWebSocket.build_registry.broadcast(:websocket_check, "hello")
+
+    assert_equal "hello", Timeout.timeout(5) { port.receive }
+  ensure
+    AppWebSocket::REGISTRY.unregister(:websocket_check, port) if port
+    port&.close
+  end
+end

data/lib/monk/version.rb

@@ -5,5 +5,5 @@ module Monk
   # class of bug .freeze! guards against for routes/error handlers/models,
   # just on a top-level constant that every worker Ractor reads on boot
   # (found live under kino: GET /hello 500'd until this was frozen).
-  VERSION = "0.19.0".freeze
+  VERSION = "0.21.0".freeze
 end

data/lib/monk/websocket/connection.rb

@@ -85,11 +85,22 @@ module Monk
 
       # The server-initiated close path (step 14): send a close frame
       # carrying the 2-byte status code + reason RFC 6455 expects, then
-      # close the socket outright -- no wait for the client's own close
-      # frame in reply.
+      # end the connection -- no wait for the client's own close frame in
+      # reply.
+      #
+      # Shut down, not closed: this runs from other threads too (a relay
+      # on Registry#close_all, the reverify thread) while the handler's
+      # thread is blocked reading. IO#close would wait for that reader,
+      # and the reader's cleanup kills the closing thread mid-close,
+      # leaving the descriptor open: the client never sees the close.
+      # shutdown tells the client at once and ends the blocked read (it
+      # returns nil, as on any disconnect); Server.serve's ensure closes
+      # the socket.
       def close(code: 1000, reason: "")
         write_frame([code].pack("n") + reason.to_s, 0x8)
-        @socket.close
+        @socket.shutdown(:RDWR)
+      rescue Errno::ENOTCONN
+        # The client went away first: nothing left to shut down.
       end
 
       # Registers this connection's own Ractor::Port -- the "connection
@@ -154,9 +165,17 @@ module Monk
       private
 
       def relay_broadcasts
-        loop { write(@port.receive) }
-      rescue Ractor::ClosedError
-        # #unsubscribe! closed the port out from under a pending #receive.
+        loop do
+          message = @port.receive
+          next write(message) unless message.is_a?(CloseRequest)
+
+          # Registry#close_all: the read loop then sees the socket closed
+          # and Server.serve's cleanup runs as on any other close.
+          return close(code: message.code, reason: message.reason)
+        end
+      rescue Ractor::ClosedError, IOError, SystemCallError
+        # #unsubscribe! closed the port out from under a pending #receive,
+        # or the socket went away first.
       end
 
       def write_frame(payload, opcode)

data/lib/monk/websocket/listeners.rb

@@ -5,12 +5,36 @@ module Monk
     # module constant), so it can't record this on itself; a frozen list
     # in a module ivar is written from the main Ractor, where #listen! runs
     # at boot, and read from any Ractor, where #register runs.
+    #
+    # Also what both fanouts' subscribers share for surviving a dropped
+    # connection: the reconnect backoff and a log that can't end them.
     module Listeners
+      # 500 ms doubling to 30 s, the same curve as monk_live.js's reconnect.
+      RETRY_INITIAL = 0.5
+      RETRY_MAX = 30
+
+      # What a subscriber closes every open socket with after reconnecting:
+      # what was published while it was down is lost, and no client can
+      # tell (its seq counts what this process sent it), so each must
+      # reconnect and resync (ADR 0011).
+      MISSED_CODE = 1011
+      MISSED_REASON = "missed broadcasts".freeze
+
       @origins = [].freeze
 
       class << self
         def listening?(origin) = @origins.include?(origin)
 
+        def next_delay(delay) = [delay * 2, RETRY_MAX].min
+
+        # Monk::Log writes to log/<env>.log once Monk has booted. Logging
+        # must never end a subscriber, whatever state Monk::Log is in.
+        def log(source, level, message)
+          Monk::Log.public_send(level, "#{source}: #{message}")
+        rescue StandardError
+          nil
+        end
+
         # Main Ractor only -- a module ivar write from another Ractor raises
         # Ractor::IsolationError, which is the right answer for a boot-time
         # call made from the wrong place.