monkrb 0.21.1 → 0.22.1

data/lib/monk/websocket/server.rb

@@ -55,14 +55,49 @@ module Monk
       # hostile client on a public endpoint, not an opt-in behavior
       # change, so a server that doesn't mention it still gets the
       # default cap rather than no cap at all.
+      #
+      # max_handshakes: caps the connections still in their handshake (the
+      # HTTP upgrade and, with authenticate:, verifying the session). At the
+      # cap the accept loop stops accepting until one finishes, and new
+      # connects wait in the kernel's listen backlog. Without it, a burst
+      # past what the process can verify (a restart's reconnect wave) starts
+      # every handshake at once, fills db_pool:'s queue, and ends with most
+      # of them timing out instead of the rest queueing
+      # (docs/plan-ws-connect-ceiling.md, item 1). Kept well under the
+      # pool's queue (1,000 by default), so a full cap can't overfill it.
+      # Always on, like max_payload_size:.
+      #
+      # handshake_timeout: (seconds) bounds the whole handshake, from
+      # accept to the 101, and Handshake::MAX_REQUEST_SIZE its request; a
+      # client past the deadline is dropped, one past the size gets a 431.
+      # Without them, a socket that connects and sends nothing holds a
+      # max_handshakes: slot forever, so a few hundred of them would shut
+      # the server, and a request that never ends grows without limit
+      # (item 2). Off once the 101 is written: an idle socket is the
+      # handler's business, not the handshake's.
+      #
+      # reuse_port: true lets several processes listen on the one port, and
+      # on Linux the kernel spreads connects across them. Each has its own
+      # heap, so its own GC pauses, over a share of the sockets: on Ruby 4.0
+      # K processes cut what GC costs a connect by about K squared
+      # (item 8). A broadcast reaches all of them through the fan-out.
       AUTHENTICATE_MODES = [false, true, :optional].freeze
 
+      DEFAULT_MAX_HANDSHAKES = 128
+      DEFAULT_HANDSHAKE_TIMEOUT = 10
+
+      # What the handshake's verify returns when it raised instead of
+      # answering (the pool full or timed out, the database down): neither
+      # a subject nor "no session", so the connection gets a 503.
+      VERIFY_FAILED = Object.new.freeze
+
       # The signals #run treats as "stop": Ctrl+C and TERM.
       STOP_SIGNALS = [Signal.list.fetch("INT"), Signal.list.fetch("TERM")].freeze
 
       def initialize(
         port:, bind: "0.0.0.0", allowed_origins: nil, authenticate: false, ping_interval: nil,
-        reverify_interval: nil, max_payload_size: Connection::DEFAULT_MAX_PAYLOAD_SIZE, db_pool: nil
+        reverify_interval: nil, max_payload_size: Connection::DEFAULT_MAX_PAYLOAD_SIZE, db_pool: nil,
+        max_handshakes: DEFAULT_MAX_HANDSHAKES, handshake_timeout: DEFAULT_HANDSHAKE_TIMEOUT, reuse_port: false
       )
         if ping_interval && !ping_interval.positive?
           raise ArgumentError, "ping_interval must be positive, got #{ping_interval.inspect}"
@@ -85,6 +120,14 @@ module Monk
           raise ArgumentError, "max_payload_size must be positive, got #{max_payload_size.inspect}"
         end
 
+        unless max_handshakes.is_a?(Integer) && max_handshakes.positive?
+          raise ArgumentError, "max_handshakes must be a positive Integer, got #{max_handshakes.inspect}"
+        end
+
+        unless handshake_timeout.is_a?(Numeric) && handshake_timeout.positive?
+          raise ArgumentError, "handshake_timeout must be positive, got #{handshake_timeout.inspect}"
+        end
+
         if authenticate
           unless defined?(Monk::Auth) && Monk::Auth.config
             raise Monk::AuthNotConfiguredError,
@@ -106,19 +149,46 @@ module Monk
         # declared or isn't started in this process.
         Monk::Persistence::Pg.pool(db_pool) if db_pool
 
-        @tcp_server = TCPServer.new(bind, port)
+        @tcp_server = reuse_port ? Server.listen_reusing_port(bind, port) : TCPServer.new(bind, port)
         @allowed_origins = allowed_origins ? Ractor.make_shareable(allowed_origins.dup) : nil
         @authenticate = authenticate
         @ping_interval = ping_interval
         @reverify_interval = reverify_interval
         @max_payload_size = max_payload_size
         @db_pool = db_pool
+        @max_handshakes = max_handshakes
+        @handshake_timeout = handshake_timeout
       end
 
       def port
         @tcp_server.addr[1]
       end
 
+      # reuse_port:, for several processes on one port (bin/websocket_server's
+      # WS_PROCESSES): SO_REUSEPORT has to be set before the bind, which
+      # TCPServer.new does at once. On Linux the kernel spreads connects
+      # across the processes listening.
+      def self.listen_reusing_port(bind, port)
+        unless defined?(Socket::SO_REUSEPORT)
+          raise ArgumentError, "reuse_port: true needs SO_REUSEPORT, which this platform doesn't have"
+        end
+
+        address = Addrinfo.tcp(bind, port)
+        socket = Socket.new(address.afamily, :STREAM)
+        socket.setsockopt(:SOCKET, :REUSEADDR, true)
+        socket.setsockopt(:SOCKET, :REUSEPORT, true)
+        socket.bind(address)
+        socket.listen(Socket::SOMAXCONN)
+        # A TCPServer on the same descriptor, so accept returns a TCPSocket
+        # as without reuse_port:.
+        socket.autoclose = false
+        TCPServer.for_fd(socket.fileno)
+      rescue StandardError
+        socket&.autoclose = true
+        socket&.close
+        raise
+      end
+
       def run(&block)
         shareable_block =
           begin
@@ -130,16 +200,30 @@ module Monk
               "top-level block -- mirrors StateRactor#update's own constraint)"
           end
 
+        # One slot a handshake in flight (max_handshakes:). Each connection
+        # Ractor sends to handshake_done once its handshake is over, however
+        # it ended, and the releaser thread frees a slot for it.
+        slots = Thread::SizedQueue.new(@max_handshakes)
+        handshake_done = Ractor::Port.new
+        releaser = Thread.new do
+          loop do
+            handshake_done.receive
+            slots.pop
+          end
+        end
+
         loop do
+          slots.push(true)
           socket = @tcp_server.accept
           ractor = Ractor.new(
             shareable_block, @authenticate, @allowed_origins, @ping_interval, @reverify_interval, @max_payload_size,
-            @db_pool
-          ) do |blk, authenticate, origins, ping_interval, reverify_interval, max_payload_size, db_pool|
+            @db_pool, handshake_done, @handshake_timeout
+          ) do |blk, authenticate, origins, ping_interval, reverify_interval, max_payload_size, db_pool, done, timeout|
             Monk::WebSocket::Server.serve(
               Ractor.receive, blk,
               authenticate: authenticate, allowed_origins: origins, ping_interval: ping_interval,
-              reverify_interval: reverify_interval, max_payload_size: max_payload_size, db_pool: db_pool
+              reverify_interval: reverify_interval, max_payload_size: max_payload_size, db_pool: db_pool,
+              handshake_done: done, handshake_timeout: timeout
             )
           end
           ractor.send(socket, move: true)
@@ -161,45 +245,63 @@ module Monk
         # different failure point in this same shutdown -- a live
         # connection's cleanup racing the registry Ractor's teardown.
         @tcp_server.close
+      ensure
+        releaser&.kill
+        handshake_done&.close
       end
 
       # Runs entirely inside the connection's own dedicated Ractor
       # (docs/history/plan-websocket.md Decision 3) -- a class method, not an instance
       # method, since the Server instance itself (holding a live
       # TCPServer) is never Ractor-shareable and can't cross into here.
+      #
+      # handshake_done: is #run's port for max_handshakes:, told once the
+      # handshake is over: the 101 written, or the connection refused or
+      # dropped before it.
       def self.serve(
         socket, block, authenticate: false, allowed_origins: nil, ping_interval: nil, reverify_interval: nil,
-        max_payload_size: Connection::DEFAULT_MAX_PAYLOAD_SIZE, db_pool: nil
+        max_payload_size: Connection::DEFAULT_MAX_PAYLOAD_SIZE, db_pool: nil, handshake_done: nil,
+        handshake_timeout: DEFAULT_HANDSHAKE_TIMEOUT
       )
-        request = read_handshake_request(socket)
-        return socket.close unless request
-
-        headers =
-          begin
-            Handshake.parse_request(request)
-          rescue Monk::WebSocket::HandshakeError
-            return socket.write("HTTP/1.1 400 Bad Request\r\n\r\n")
+        # A `return` in here (a refused handshake) ends serve, and its
+        # ensure frees the slot.
+        credential, subject = with_deadline(socket, handshake_timeout) do
+          request = read_handshake_request(socket)
+          return unless request
+          return socket.write("HTTP/1.1 431 Request Header Fields Too Large\r\n\r\n") if request == :too_large
+
+          headers =
+            begin
+              Handshake.parse_request(request)
+            rescue Monk::WebSocket::HandshakeError
+              return socket.write("HTTP/1.1 400 Bad Request\r\n\r\n")
+            end
+
+          subject = nil
+          credential = Handshake.credential_from(headers) if authenticate
+          case authenticate
+          when true
+            subject = authenticate!(socket, headers, credential, allowed_origins, db_pool)
+            return unless subject
+          when :optional
+            return unless origin_allowed_for_optional?(socket, headers, credential, allowed_origins)
+
+            subject = identify(credential, db_pool)
+            # Not anonymous: that would quietly sign the user out of their
+            # topics until the next reconnect.
+            return unavailable!(socket, headers) if subject.equal?(VERIFY_FAILED)
           end
 
-        subject = nil
-        case authenticate
-        when true
-          subject = authenticate!(socket, headers, allowed_origins, db_pool)
-          return unless subject
-        when :optional
-          return unless origin_allowed_for_optional?(socket, headers, allowed_origins)
-
-          subject = identify(headers, db_pool)
+          socket.write(Handshake.response(headers))
+          [credential, subject]
         end
-
-        socket.write(Handshake.response_for(request))
+        handshake_done = handshake_over(handshake_done)
 
         connection = Connection.new(socket, subject: subject, max_payload_size: max_payload_size)
         connection.start_heartbeat(ping_interval) if ping_interval
-        reverify_thread =
-          if reverify_interval && subject
-            start_reverify(connection, Handshake.credential_from(headers)[:token], reverify_interval, db_pool)
-          end
+        if reverify_interval && subject
+          reverify_thread = start_reverify(connection, credential[:token], reverify_interval, db_pool)
+        end
         begin
           block.call(connection)
         rescue StandardError
@@ -209,11 +311,16 @@ module Monk
           # not re-raised -- there's no caller left to hand it to once
           # we're inside this connection's own dedicated Ractor.
         end
+      rescue IOError, SystemCallError
+        # The client went away, or was shut out by the deadline, before
+        # the handshake was over: nothing to answer.
       ensure
         # Runs on every exit path -- normal completion, the close
         # handshake, and a crashed handler alike -- so a subscribed
         # connection never leaks a stale registry entry (step 17), a
         # heartbeat thread, or a reverify thread outliving its socket.
+        # A handshake that never reached its 101 still frees its slot.
+        handshake_over(handshake_done)
         reverify_thread&.kill
         connection&.unsubscribe!
         connection&.stop_heartbeat!
@@ -221,6 +328,35 @@ module Monk
         disconnect_all
       end
 
+      # Runs the block with a watchdog that shuts the socket down once
+      # `seconds` have passed, which ends a read or write blocked on it (a
+      # read returns nil, a write raises). Not IO#timeout=: on Ruby 4.0,
+      # once an IO#timeout has fired in one Ractor, reads in Ractors made
+      # after it can raise IO::TimeoutError on sockets with no timeout set.
+      def self.with_deadline(socket, seconds)
+        watchdog = Thread.new do
+          sleep seconds
+          socket.shutdown(:RDWR)
+        rescue IOError, SystemCallError
+          # Already closed or gone.
+        end
+        yield
+      ensure
+        watchdog&.kill
+      end
+      private_class_method :with_deadline
+
+      # Frees this connection's max_handshakes: slot in #run; returns nil,
+      # so serve can mark it freed. The port is gone once #run has
+      # returned, and then there's no slot to free.
+      def self.handshake_over(handshake_done)
+        handshake_done&.send(true)
+        nil
+      rescue Ractor::ClosedError
+        nil
+      end
+      private_class_method :handshake_over
+
       # Closes the connections this socket's Ractor opened: its database
       # connection (verifying the session without db_pool:, or a block
       # that queries) and a fan-out's publisher (a block that broadcasts).
@@ -239,10 +375,9 @@ module Monk
       # credential -- a Bearer connection has no Origin header to check
       # by construction (mirrors require_csrf!'s own Bearer exemption in
       # docs/history/plan-auth.md). Returns the verified subject, or nil after writing
-      # the appropriate 403/401 response itself.
-      def self.authenticate!(socket, headers, allowed_origins, db_pool)
-        credential = Handshake.credential_from(headers)
-
+      # the appropriate 403/401 response itself -- or 503, when verify
+      # couldn't answer (docs/plan-ws-connect-ceiling.md, item 3).
+      def self.authenticate!(socket, headers, credential, allowed_origins, db_pool)
         if credential&.fetch(:via) == :cookie && allowed_origins
           origin = headers["origin"]
           unless origin && allowed_origins.include?(origin)
@@ -251,12 +386,26 @@ module Monk
           end
         end
 
-        subject = credential && verify(credential[:token], db_pool)
+        subject = credential && handshake_verify(credential[:token], db_pool)
+        return unavailable!(socket, headers) if subject.equal?(VERIFY_FAILED)
+
         socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n") unless subject
         subject
       end
       private_class_method :authenticate!
 
+      # A handshake whose verify couldn't answer: 503, and nil. Browser JS
+      # never sees a failed handshake's status or headers (only onclose,
+      # 1006), so Retry-After is for the other clients, the ones without
+      # an Origin, and jittered, so a wave of them doesn't come back as
+      # one.
+      def self.unavailable!(socket, headers)
+        retry_after = headers["origin"] ? "" : "Retry-After: #{rand(1..10)}\r\n"
+        socket.write("HTTP/1.1 503 Service Unavailable\r\n#{retry_after}\r\n")
+        nil
+      end
+      private_class_method :unavailable!
+
       # authenticate: :optional's Origin check, before anything is verified.
       # A Bearer connection is exempt, as under true. A cookie needs an
       # allowed Origin, as under true. An anonymous connection that sends an
@@ -264,10 +413,9 @@ module Monk
       # stops another site's page from opening sockets here on its
       # visitors' behalf; one that sends none (not a browser) gets in.
       # Writes the 403 itself and returns false when refused.
-      def self.origin_allowed_for_optional?(socket, headers, allowed_origins)
+      def self.origin_allowed_for_optional?(socket, headers, credential, allowed_origins)
         return true unless allowed_origins
 
-        credential = Handshake.credential_from(headers)
         return true if credential&.fetch(:via) == :bearer
 
         origin = headers["origin"]
@@ -280,13 +428,26 @@ module Monk
       private_class_method :origin_allowed_for_optional?
 
       # The subject of a valid session, or nil -- no credential, or one
-      # Monk::Auth no longer accepts, both mean anonymous under :optional.
-      def self.identify(headers, db_pool)
-        credential = Handshake.credential_from(headers)
-        credential && verify(credential[:token], db_pool)
+      # Monk::Auth no longer accepts, both mean anonymous under :optional --
+      # or VERIFY_FAILED.
+      def self.identify(credential, db_pool)
+        credential && handshake_verify(credential[:token], db_pool)
       end
       private_class_method :identify
 
+      # verify, or VERIFY_FAILED with one line on stderr if it raised:
+      # under a burst, a backtrace per refused connection buries the log.
+      # The message's first line only (a PG error adds the query and a
+      # caret), and not Kernel#warn, which -W0 silences.
+      def self.handshake_verify(raw_token, db_pool)
+        verify(raw_token, db_pool)
+      rescue StandardError => e
+        reason = e.message.lines.first&.chomp
+        $stderr.puts "monk-websocket: handshake refused with 503, verify failed: #{e.class}: #{reason}" # rubocop:disable Style/StderrPuts
+        VERIFY_FAILED
+      end
+      private_class_method :handshake_verify
+
       # Sleeps in `interval`-sized steps, re-running Monk::Auth.verify
       # against the same raw credential each time, until it comes back
       # nil (revoked or expired) -- then closes the connection with the
@@ -319,15 +480,18 @@ module Monk
       end
       private_class_method :verify
 
+      # The request head, nil if the client went away or was too slow, or
+      # :too_large past Handshake::MAX_REQUEST_SIZE.
+      #
+      # One String, read through the socket's own buffer: Connection's
+      # reads use that buffer after it, so a frame the client sent right
+      # behind its request isn't lost (a readpartial would take it into
+      # the request and drop it). A read(1) loop made a String a byte.
       def self.read_handshake_request(socket)
-        request = +""
-        until request.end_with?("\r\n\r\n")
-          chunk = socket.read(1)
-          return nil if chunk.nil?
+        request = socket.gets("\r\n\r\n", Handshake::MAX_REQUEST_SIZE)
+        return request if request&.end_with?("\r\n\r\n")
 
-          request << chunk
-        end
-        request
+        :too_large if request && request.bytesize >= Handshake::MAX_REQUEST_SIZE
       end
       private_class_method :read_handshake_request
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: monkrb
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: 0.22.1
 platform: ruby
 authors:
 - Matteo Folin
@@ -186,6 +186,7 @@ files:
 - lib/monk/auth/rate_limiter.rb
 - lib/monk/auth/send_login_link.rb
 - lib/monk/auth/session.rb
+- lib/monk/auth/sweep.rb
 - lib/monk/base.rb
 - lib/monk/bin.rb
 - lib/monk/cli.rb
@@ -210,9 +211,13 @@ files:
 - lib/monk/jobs/adapters/pg.rb
 - lib/monk/jobs/args.rb
 - lib/monk/jobs/claim.rb
+- lib/monk/jobs/cron.rb
 - lib/monk/jobs/errors.rb
+- lib/monk/jobs/firing.rb
 - lib/monk/jobs/job.rb
+- lib/monk/jobs/recurring_task.rb
 - lib/monk/jobs/runtime.rb
+- lib/monk/jobs/scheduler.rb
 - lib/monk/jobs/worker.rb
 - lib/monk/live.rb
 - lib/monk/live/client/idiomorph.LICENSE
@@ -254,6 +259,8 @@ files:
 - lib/monk/templates/auth/config/auth.rb
 - lib/monk/templates/auth/db/migrate/00000000000001_create_auth_tables.down.sql
 - lib/monk/templates/auth/db/migrate/00000000000001_create_auth_tables.up.sql
+- lib/monk/templates/auth/db/migrate/00000000000004_create_login_requests.down.sql
+- lib/monk/templates/auth/db/migrate/00000000000004_create_login_requests.up.sql
 - lib/monk/templates/auth/setup.md
 - lib/monk/templates/auth/test/auth_test.rb
 - lib/monk/templates/base/.dockerignore
@@ -288,8 +295,11 @@ files:
 - lib/monk/templates/jobs/app/routes/jobs.rb
 - lib/monk/templates/jobs/bin/jobs
 - lib/monk/templates/jobs/config/jobs.rb
+- lib/monk/templates/jobs/config/recurring.rb
 - lib/monk/templates/jobs/db/migrate/00000000000002_create_jobs_tables.down.sql
 - lib/monk/templates/jobs/db/migrate/00000000000002_create_jobs_tables.up.sql
+- lib/monk/templates/jobs/db/migrate/00000000000003_create_recurring_tables.down.sql
+- lib/monk/templates/jobs/db/migrate/00000000000003_create_recurring_tables.up.sql
 - lib/monk/templates/jobs/setup.md
 - lib/monk/templates/jobs/test/jobs_test.rb
 - lib/monk/templates/live/agents.md