monkrb 0.19.0 → 0.21.0

data/lib/monk/generators/websocket.rb

@@ -0,0 +1,24 @@
+# The WebSocket server process, over a transport between processes:
+# Postgres or Redis.
+Monk::Generator.define(:websocket) do
+  summary "WebSocket server process"
+  installed_if "config/websocket.rb"
+  option :transport, values: %w[postgres redis], dependency: true,
+    describe: { postgres: "LISTEN/NOTIFY, nothing else to run; 8 KB per broadcast",
+                redis: "no size limit; Redis must be running", }
+
+  copy "config/websocket.rb", role: :wiring,
+    from: ->(options) { "websocket/config/websocket_#{options[:transport]}.rb" }
+  copy "bin/websocket_server", role: :wiring, executable: true
+  copy "app/sockets/chat.rb", role: :example
+  copy "test/websocket_test.rb", role: :test
+
+  env :example, { "WS_PORT" => "9293" }
+  env :example, { "WS_ALLOWED_ORIGINS" => "https://example.com" }, commented: true
+
+  setup_section "websocket/setup.md"
+  agents_section "websocket/agents.md"
+  set_before_production "WS_ALLOWED_ORIGINS", "defaults to PUBLIC_URL; list every origin your pages use"
+  next_step "bin/websocket_server", note: "beside bin/server"
+  example "websocket-chat", todo: "uncomment for one chat room, or write your own handler in app/sockets/"
+end

data/lib/monk/generators.rb

@@ -0,0 +1,7 @@
+require_relative "generator"
+
+# Monk's own modules, one generator each (docs/adr/0017), in dependency
+# order only for readability: the registry resolves dependencies itself.
+%w[base postgres redis mail auth jobs websocket live].each do |name|
+  require_relative "generators/#{name}"
+end

data/lib/monk/jobs/adapters/pg.rb

@@ -247,10 +247,12 @@ module Monk
         # Ractor's connection is still running on the server, then
         # reconnects, so the worker's next query doesn't wait for the
         # abandoned one (Phase 0.3) or land inside its open transaction.
+        # The cancel stays here: that connection is alive, just busy, so
+        # checkout's own probe wouldn't reset it.
         def reset_connection
           conn = Monk::Persistence::Pg[@db_name]
           conn.cancel
-          conn.reset
+          Monk::Persistence::Pg.revive(conn)
           nil
         end
 

data/lib/monk/jobs.rb

@@ -227,3 +227,10 @@ module Monk
 end
 
 require_relative "jobs/job"
+
+# What jobs provide together with another module, when that module is
+# loaded too -- either order works (lib/monk/mail.rb and lib/monk/auth.rb
+# end the same way), so an app's config files never require these
+# themselves (docs/adr/0017).
+require_relative "mail/deliver_later" if Monk.const_defined?(:Mail)
+require_relative "auth/send_login_link" if Monk.const_defined?(:Auth)

data/lib/monk/live/helpers.rb

@@ -17,6 +17,19 @@ module Monk
 
         raw(%(data-live-topic="#{names.join(" ")}"))
       end
+
+      # For the layout's <%= monk_head %>: where the client opens its
+      # WebSocket (live_ws_url, which config/live.rb declares) and the
+      # client runtime itself. Nothing until Live is configured, i.e. until
+      # config/live.rb has run.
+      def head_tags
+        return super unless Monk::Live.registry
+
+        super + [
+          %(<meta name="monk-live-url" content="#{h(settings[:live_ws_url])}">),
+          %(<script type="module" src="#{h(asset_path("#{Live::CLIENT_PATH}/monk_live.js"))}"></script>),
+        ]
+      end
     end
   end
 end

data/lib/monk/live/policy.rb

@@ -7,8 +7,16 @@ module Monk
     # the matching rule says `anonymous: true`: a block like
     # `topic == "contacts:#{subject}"` would otherwise let a nil subject
     # in through "contacts:". A block that raises fails closed.
+    #
+    # A rule with db_pool: runs its block in that Monk::Persistence::Pg
+    # pool (docs/history/plan-pg-pool.md), not in the socket's own Ractor,
+    # so a rule that queries the database doesn't open a connection per
+    # open page. The block crosses to the pool as it is: rules are
+    # shareable procs already. If the pool itself fails (not started in
+    # this process, timed out, full), the rule denies and the reason is
+    # logged.
     module Policy
-      Rule = Data.define(:pattern, :anonymous, :check) do
+      Rule = Data.define(:pattern, :anonymous, :check, :db_pool) do
         def matches?(topic)
           if pattern.end_with?("*")
             topic.start_with?(pattern.chomp("*"))
@@ -18,14 +26,17 @@ module Monk
         end
       end
 
-      def self.build_rule(pattern, anonymous, block)
+      def self.build_rule(pattern, anonymous, block, db_pool = nil)
         unless pattern.is_a?(String) && !pattern.strip.empty? && valid_wildcard?(pattern)
           raise ArgumentError,
             "pattern must be a topic or a topic prefix ending in a single *, got #{pattern.inspect}"
         end
         raise ArgumentError, "authorize needs a block: { |subject, topic| ... }" unless block
+        unless db_pool.nil? || db_pool.is_a?(Symbol)
+          raise ArgumentError, "db_pool must be a pool name (a Symbol), got #{db_pool.inspect}"
+        end
 
-        Rule.new(pattern: pattern.dup.freeze, anonymous: anonymous, check: shareable(block))
+        Rule.new(pattern: pattern.dup.freeze, anonymous: anonymous, check: shareable(block), db_pool: db_pool)
       end
 
       def self.allowed?(rules, subject, topic)
@@ -33,11 +44,35 @@ module Monk
         return false unless rule
         return false if subject.nil? && !rule.anonymous
 
-        rule.check.call(subject, topic) ? true : false
+        run(rule, subject, topic) ? true : false
+      rescue Monk::PersistenceTimeoutError, Monk::PoolFullError, Monk::PoolNotStartedError, Monk::UnknownPoolError,
+             Monk::PoolWorkerDiedError, Monk::PoolStoppedError => e
+        log_pool_failure(rule, topic, e)
+        false
       rescue StandardError
         false
       end
 
+      # What a pooled rule runs in the pool's worker.
+      def self.check(block, subject, topic) = block.call(subject, topic)
+
+      def self.run(rule, subject, topic)
+        return rule.check.call(subject, topic) unless rule.db_pool
+
+        Monk::Persistence::Pg.pool(rule.db_pool).call(Policy, :check, rule.check, subject, topic)
+      end
+      private_class_method :run
+
+      def self.log_pool_failure(rule, topic, error)
+        Monk::Log.error(
+          "Monk::Live: denied #{topic} (rule #{rule.pattern}): its pool #{rule.db_pool.inspect} failed: " \
+          "#{error.class}: #{error.message}",
+        )
+      rescue StandardError
+        nil
+      end
+      private_class_method :log_pool_failure
+
       def self.valid_wildcard?(pattern)
         pattern.count("*").zero? || (pattern.count("*") == 1 && pattern.end_with?("*"))
       end

data/lib/monk/live/session.rb

@@ -95,6 +95,13 @@ module Monk
         seq = 0
         loop do
           payload = @port.receive
+          # Registry#close_all: broadcasts were missed, so the client must
+          # reconnect and resync (ADR 0011). run's read loop ends, and
+          # cleanup unregisters every topic.
+          if payload.is_a?(Monk::WebSocket::CloseRequest)
+            return @connection.close(code: payload.code, reason: payload.reason)
+          end
+
           seq += 1
           @connection.write(payload.start_with?("{") ? "{\"seq\":#{seq},#{payload.byteslice(1..)}" : payload)
         end

data/lib/monk/live.rb

@@ -35,14 +35,25 @@ module Monk
       ).run
     end
 
+    # A rule block that lets everyone in, for Monk::Live.authorize:
+    #   Monk::Live.authorize("demo:hits", anonymous: true, &Monk::Live::ALLOW_ALL)
+    # A block written at the top of config/live.rb isn't Ractor-shareable
+    # (self there is the main object), so rules need one built here.
+    ALLOW_ALL = Ractor.make_shareable(proc { |_subject, _topic| true })
+
     CLIENT_DIR = File.expand_path("live/client", __dir__).freeze
+    # Where Monk serves CLIENT_DIR from: the layout's <%= monk_head %> loads
+    # CLIENT_PATH/monk_live.js (Live::Helpers#head_tags), so an app never
+    # copies the client and never serves an older copy than its gem.
+    CLIENT_PATH = "/_monk/live".freeze
+    Monk::Assets.mount(CLIENT_PATH, CLIENT_DIR)
 
     class << self
       attr_reader :publisher, :registry, :rules, :max_topics
 
       # Where the browser runtime ships inside the gem: monk_live.js,
       # protocol.js and idiomorph.js (they import each other by relative
-      # path, so an app serves or copies the directory as a unit).
+      # path, so they're served as a unit, from CLIENT_PATH).
       def client_dir = CLIENT_DIR
 
       # Boot-time, main Ractor. `registry` is a Monk::WebSocket::Registry,
@@ -74,9 +85,11 @@ module Monk
 
       # Boot-time, main Ractor. See Monk::Live::Policy for the semantics:
       # deny by default, first matching rule wins, anonymous subjects need
-      # `anonymous: true`. The block must be Ractor-shareable.
-      def authorize(pattern, anonymous: false, &block)
-        @rules = Ractor.make_shareable([*@rules, Policy.build_rule(pattern, anonymous, block)])
+      # `anonymous: true`. The block must be Ractor-shareable. db_pool:
+      # names a Monk::Persistence::Pg pool to run the block in, for a rule
+      # that queries the database; the WebSocket process must start it.
+      def authorize(pattern, anonymous: false, db_pool: nil, &block)
+        @rules = Ractor.make_shareable([*@rules, Policy.build_rule(pattern, anonymous, block, db_pool)])
       end
 
       def authorized?(subject, topic) = Policy.allowed?(rules, subject, topic)

data/lib/monk/log.rb

@@ -118,7 +118,16 @@ module Monk
       end
 
       def handle
-        Ractor.current[:monk_log_handle] ||= File.open(@path, "a")
+        Ractor.current[:monk_log_handle] ||= File.open(@path || path_before_boot, "a")
+      end
+
+      # Before Boot -- bin/console, a script that loads config/load.rb and
+      # never boots -- nothing has frozen the path yet. Worked out here
+      # instead, the way #freeze_registry! does: only the main Ractor runs
+      # before Boot, so reading Monk.env and writing log/ is safe.
+      def path_before_boot
+        FileUtils.mkdir_p(root)
+        File.join(root, "#{Monk.env}.log")
       end
     end
 

data/lib/monk/mail/deliver_later.rb

@@ -0,0 +1,54 @@
+# Monk::Mail.deliver_later and its job. Loaded at the end of whichever of
+# monk/mail and monk/jobs is required second, once both are defined
+# (docs/adr/0017), so it requires neither itself.
+
+module Monk
+  module Mail
+    # Sends one message from a job process (docs/adr/0014-mail-from-jobs-and-login-links-created-in-the-job.md).
+    # Enqueued by Monk::Mail.deliver_later, never directly: its argument
+    # is the message's fields, already checked and with from: resolved.
+    class DeliveryJob < Monk::Job
+      queue "mailers"
+      never_retry PermanentDeliveryError
+
+      # The SMTP failures that are final, by class name so this file never
+      # loads net/smtp itself (the app's Gemfile decides that, ADR 0012):
+      # a 5xx reply, a malformed command, refused credentials. Everything
+      # else -- a 4xx "try later", a refused connection, TLS, a timeout --
+      # is worth retrying.
+      PERMANENT_SMTP_ERRORS = %w[Net::SMTPFatalError Net::SMTPSyntaxError Net::SMTPAuthenticationError].freeze
+
+      def self.perform(fields)
+        Monk::Mail.deliver(**fields.transform_keys(&:to_sym))
+      rescue DeliveryError => e
+        raise unless PERMANENT_SMTP_ERRORS.include?(e.cause&.class&.name)
+
+        raise PermanentDeliveryError, e.message
+      end
+    end
+
+    class << self
+      # deliver, later: the same arguments, plus enqueue's wait:, at: and
+      # conn: (the last one enqueues inside the caller's transaction, so the
+      # email commits or rolls back with the app's own writes). Returns the
+      # job's id. The message is built -- and so checked -- here, before
+      # anything is enqueued: bad input raises InvalidMessageError in the
+      # caller, exactly as deliver would, rather than failing later in the
+      # job process. Render HTML before calling (html: Monk::Mail.render(...)):
+      # the job stores the finished message and only sends it.
+      #
+      #   Monk::Mail.deliver_later(to: order[:email], subject: "Your receipt", text: receipt_text)
+      def deliver_later(to:, subject:, text: nil, html: nil, reply_to: nil, from: nil, wait: nil, at: nil, conn: nil)
+        raise NotConfiguredError, "call Monk::Mail.configure(url:, from:) before Monk::Mail.deliver_later" unless config
+
+        from ||= config[:from]
+        if from.nil?
+          raise InvalidMessageError, "no from: given, and Monk::Mail.configure has no default from: (MAIL_FROM)"
+        end
+
+        message = Message.new(from: from, to: to, subject: subject, text: text, html: html, reply_to: reply_to)
+        DeliveryJob.enqueue(message.to_h.transform_keys(&:to_s), wait: wait, at: at, conn: conn)
+      end
+    end
+  end
+end

data/lib/monk/mail/later.rb

@@ -1,53 +1,5 @@
+# Monk::Mail.deliver_later (docs/adr/0014). Loading monk/mail and
+# monk/jobs is enough -- the second one loads it (docs/adr/0017) -- so
+# this file only exists for apps whose config/jobs.rb still requires it.
 require_relative "../mail"
 require_relative "../jobs"
-
-module Monk
-  module Mail
-    # Sends one message from a job process (docs/adr/0014-mail-from-jobs-and-login-links-created-in-the-job.md).
-    # Enqueued by Monk::Mail.deliver_later, never directly: its argument
-    # is the message's fields, already checked and with from: resolved.
-    class DeliveryJob < Monk::Job
-      queue "mailers"
-      never_retry PermanentDeliveryError
-
-      # The SMTP failures that are final, by class name so this file never
-      # loads net/smtp itself (the app's Gemfile decides that, ADR 0012):
-      # a 5xx reply, a malformed command, refused credentials. Everything
-      # else -- a 4xx "try later", a refused connection, TLS, a timeout --
-      # is worth retrying.
-      PERMANENT_SMTP_ERRORS = %w[Net::SMTPFatalError Net::SMTPSyntaxError Net::SMTPAuthenticationError].freeze
-
-      def self.perform(fields)
-        Monk::Mail.deliver(**fields.transform_keys(&:to_sym))
-      rescue DeliveryError => e
-        raise unless PERMANENT_SMTP_ERRORS.include?(e.cause&.class&.name)
-
-        raise PermanentDeliveryError, e.message
-      end
-    end
-
-    class << self
-      # deliver, later: the same arguments, plus enqueue's wait:, at: and
-      # conn: (the last one enqueues inside the caller's transaction, so the
-      # email commits or rolls back with the app's own writes). Returns the
-      # job's id. The message is built -- and so checked -- here, before
-      # anything is enqueued: bad input raises InvalidMessageError in the
-      # caller, exactly as deliver would, rather than failing later in the
-      # job process. Render HTML before calling (html: Monk::Mail.render(...)):
-      # the job stores the finished message and only sends it.
-      #
-      #   Monk::Mail.deliver_later(to: order[:email], subject: "Your receipt", text: receipt_text)
-      def deliver_later(to:, subject:, text: nil, html: nil, reply_to: nil, from: nil, wait: nil, at: nil, conn: nil)
-        raise NotConfiguredError, "call Monk::Mail.configure(url:, from:) before Monk::Mail.deliver_later" unless config
-
-        from ||= config[:from]
-        if from.nil?
-          raise InvalidMessageError, "no from: given, and Monk::Mail.configure has no default from: (MAIL_FROM)"
-        end
-
-        message = Message.new(from: from, to: to, subject: subject, text: text, html: html, reply_to: reply_to)
-        DeliveryJob.enqueue(message.to_h.transform_keys(&:to_s), wait: wait, at: at, conn: conn)
-      end
-    end
-  end
-end

data/lib/monk/mail.rb

@@ -119,3 +119,7 @@ module Monk
     Monk.freeze_hooks << self
   end
 end
+
+# Monk::Mail.deliver_later, when jobs are loaded too -- either order works
+# (lib/monk/jobs.rb ends the same way, docs/adr/0017).
+require_relative "mail/deliver_later" if Monk.const_defined?(:Jobs)

data/lib/monk/persistence/errors.rb

@@ -4,4 +4,36 @@ module Monk
 
   class UnknownPersistenceError < StandardError
   end
+
+  # Monk::Persistence::Pg.pool(name) for a pool never declared.
+  class UnknownPoolError < StandardError
+  end
+
+  # Monk::Persistence::Pg.pool(name) for a pool declared but not started
+  # in this process (start_pools!).
+  class PoolNotStartedError < StandardError
+  end
+
+  # A pool call's method returned a value that can't be copied to the
+  # caller's Ractor (a PG::Result, say).
+  class PoolReturnError < StandardError
+  end
+
+  # A pool's queue is full: as many calls as its queue: option allows are
+  # already waiting for a worker.
+  class PoolFullError < StandardError
+  end
+
+  # The pool worker running a call died (not an exception from the called
+  # method, which the caller gets): the call may or may not have finished.
+  class PoolWorkerDiedError < StandardError
+  end
+
+  # The pool has stopped: calls waiting when it stopped, and calls after.
+  class PoolStoppedError < StandardError
+  end
+
+  # start_pools! couldn't connect a pool's workers.
+  class PoolStartError < StandardError
+  end
 end

data/lib/monk/persistence/pg.rb

@@ -36,11 +36,102 @@ module Monk
           .tap { |types| %w[json jsonb].each { |name| types.register_type(0, name, nil, JsonDecoder) } },
       )
 
+      # Without a connect_timeout, libpq waits indefinitely for a host that
+      # doesn't answer -- and #checkout's revive does so holding the
+      # Ractor's slot. The same 5 s as Registry's checkout timeout. An
+      # app's own connect_timeout: wins.
+      DEFAULT_CONNECT_TIMEOUT = 5
+
+      # Every diagnostic field libpq reports for an error (SQLSTATE,
+      # constraint, table, detail, hint, ...).
+      DIAG_FIELDS = PG.constants.grep(/\APG_DIAG_/).map { |name| PG.const_get(name) }.freeze
+
+      # What a PG::Error's #result becomes when the error leaves a pool's
+      # worker (#make_portable): a PG::Result is tied to its connection and
+      # can't be copied to another Ractor. Keeps what reading an error
+      # needs; anything else raises NoMethodError.
+      ErrorResult = Data.define(:fields, :error_message, :result_status, :res_status) do
+        def self.from(result)
+          fields = DIAG_FIELDS.to_h { |code| [code, result.error_field(code)] }.compact
+          new(
+            fields: fields, error_message: result.error_message,
+            result_status: result.result_status, res_status: result.res_status,
+          )
+        end
+
+        def error_field(code) = fields[code]
+        alias_method :result_error_field, :error_field
+        alias_method :result_error_message, :error_message
+      end
+
       class << self
+        # A copy of an exception about to leave a pool's worker
+        # (Pool::Portable): a PG::Error holds its connection and result,
+        # which can't be copied to another Ractor. The connection goes (no
+        # use in another Ractor anyway), and the result becomes an
+        # ErrorResult, so error_field still answers. The error keeps its
+        # class, so `rescue PG::UniqueViolation` matches as it would here.
+        def make_portable(error)
+          return unless error.is_a?(PG::Error)
+
+          result = error.result
+          error.instance_variable_set(:@connection, nil)
+          error.instance_variable_set(:@result, result && ErrorResult.from(result))
+        end
+
+        # Detects a connection the server has closed (a restart, a
+        # failover, pg_terminate_backend). libpq still reports
+        # CONNECTION_OK, but the socket is readable: a healthy idle
+        # connection's isn't. So the common case costs one zero-timeout
+        # select, about 1 µs, and nothing sent.
+        #
+        # A readable socket is either the server's FATAL before it closes,
+        # or a NOTIFY on a connection that LISTENs. Reading on to tell
+        # them apart races the close: the FATAL can arrive a moment before
+        # the end of file, and a second select in between sees nothing.
+        # So a readable socket gets an empty query, one round trip, which
+        # fails on a closed connection and leaves pending notifies queued.
+        #
+        # A network path that dies without closing the socket (a NAT
+        # timing out an idle connection) leaves it unreadable, and this
+        # can't see it: the next query waits for TCP to give up.
+        #
+        # Public: PgFanout probes its own connections with it.
+        def alive?(conn)
+          return false unless conn.status == PG::CONNECTION_OK
+          return true unless conn.socket_io.wait_readable(0)
+
+          conn.exec("")
+          true
+        rescue PG::Error, IOError
+          false
+        end
+
+        # Same connection object, new backend: the result type map set by
+        # #connect survives. Raises PG::ConnectionBad if Postgres is still
+        # unreachable; the next checkout tries again.
+        def revive(conn)
+          conn.reset
+        end
+
         private
 
+        # A block that ran a raw BEGIN and raised leaves the connection in
+        # a transaction, and the next checkout in this Ractor would run
+        # inside it. (conn.transaction rolls back by itself.) Skipped for
+        # PQTRANS_ACTIVE, a query still running: a ROLLBACK would first
+        # wait for it. Monk::Jobs cancels those itself.
+        def release(conn)
+          return unless conn.status == PG::CONNECTION_OK
+          return unless [PG::PQTRANS_INTRANS, PG::PQTRANS_INERROR].include?(conn.transaction_status)
+
+          conn.exec("ROLLBACK")
+        rescue PG::Error
+          nil
+        end
+
         def connect(**opts)
-          conn = PG.connect(**opts)
+          conn = PG.connect(connect_timeout: DEFAULT_CONNECT_TIMEOUT, **opts)
           conn.type_map_for_results = PG::BasicTypeMapForResults.new(conn, registry: RESULT_TYPES)
           conn
         end