serverengine 2.3.2 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7350069b97302faf3d6c50118875bf055a7094bc31845185cd4e6b4d3e0a9f61
-  data.tar.gz: 02e1c1e261b6ea6dde6a77a984c8f94dccf237093dda7949174acd2658c9897b
+  metadata.gz: 516efcdedbeba3c7d7988c91cc121a37b52efca99726d833447fe374680cac43
+  data.tar.gz: b4265b4b425eb2d165881d579f6b2115c8e7bf2298c56f1cf75d6ed1cb804993
 SHA512:
-  metadata.gz: 66ae31cea3ff4779a648d64c00e54ff01e76912cd2a4ce1f6105b83657bdbdaea0e9b2c54972cc171ed2aa2a01794dd4c2c3b2e73fdb1754a7013cee2fd0c1a5
-  data.tar.gz: 8b023a4052650036480ce2f899ac367279dcc341a3d98536c7a3fc91cf227622a3707f7315a9eead3a0a2335792700723dda3a527aa422c5aa06d2ff27824b11
+  metadata.gz: cc1d3263b08bd87e645563000b46a79a58437617006eb993b959d50105a875d1592bdce2e45c9bd1a4fbfd7740ab672d1e61a13e39654cdae798dbe0fe18f606
+  data.tar.gz: 62fca1438f15432ce570216dce14eabef6235835c2f43f875ba220b3ba6d6a86e8aaedcfa4e5752a7ba5529a55d6bfeca9618ce0f59924dd6ef9666e69d00cf2

data/.github/workflows/{linux.yml → test.yml}

@@ -1,4 +1,4 @@
-name: Testing on Ubuntu
+name: Test
 on:
   push:
     branches: [master]
@@ -11,10 +11,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
-        os:
-          - ubuntu-latest
-    name: Unit testing with Ruby ${{ matrix.ruby }} on ${{ matrix.os }}
+        ruby: [ '3.3', '3.2', '3.1', '3.0', '2.7' ]
+        os: ['ubuntu-latest', 'windows-latest']
+    name: Ruby ${{ matrix.ruby }} on ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby
@@ -22,9 +21,7 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby }}
       - name: Install dependencies
-        run: |
-          gem install bundler rake
-          bundle install --jobs 4 --retry 3
+        run: bundle install --jobs 4 --retry 3
       - name: Run tests
         env:
           CI: true

data/Changelog

@@ -1,3 +1,8 @@
+2024-10-22 version 2.4.0
+
+* Add missing base64 and logger gem as dependency for Ruby 3.4 and 3.5.
+* socket_manager: add feature to share sockets with another server.
+
 2023-03-14 version 2.3.2
 
 * Accept `nil` for `ServerEngine::SocketManager::Server.open` to select path automatically

data/README.md

@@ -413,8 +413,15 @@ se = ServerEngine.create(MyServer, MyWorker, {
 se.run
 ```
 
-See also [examples](https://github.com/fluent/serverengine/tree/master/examples).
+Other features:
+
+- `socket_manager_server = SocketManager::Server.share_sockets_with_another_server(path)`
+  - It starts a new manager server that shares all UDP/TCP sockets with the existing manager.
+  - We can use this for live restart for network servers.
+  - The old process should stop without removing the file for the socket after the new process starts.
+  - Limitation: This feature would not work well if the process opens new TCP ports frequently.
 
+See also [examples](https://github.com/fluent/serverengine/tree/master/examples).
 
 ## Module API
 

data/lib/serverengine/socket_manager.rb

@@ -96,19 +96,32 @@ module ServerEngine
         end
       end
 
-      def initialize(path)
+      def self.share_sockets_with_another_server(path)
+        raise NotImplementedError, "Not supported on Windows." if ServerEngine.windows?
+        server = new(path, start: false)
+        server.share_sockets_with_another_server
+        server
+      end
+
+      def initialize(path, start: true)
         @tcp_sockets = {}
         @udp_sockets = {}
         @mutex = Mutex.new
-        @path = start_server(path)
+        @path = start ? start_server(path) : path
       end
 
       attr_reader :path
+      attr_reader :tcp_sockets, :udp_sockets # for tests
 
       def new_client
         Client.new(@path)
       end
 
+      def start
+        start_server(path)
+        nil
+      end
+
       def close
         stop_server
         nil
@@ -159,9 +172,9 @@ module ServerEngine
           res = SocketManager.recv_peer(peer)
           return if res.nil?
 
-          pid, method, bind, port = *res
+          pid, method, *opts = res
           begin
-            send_socket(peer, pid, method, bind, port)
+            send_socket(peer, pid, method, *opts)
           rescue => e
             SocketManager.send_peer(peer, e)
           end

data/lib/serverengine/socket_manager_unix.rb

@@ -47,6 +47,38 @@ module ServerEngine
     end
 
     module ServerModule
+      def share_sockets_with_another_server
+        another_server = UNIXSocket.new(@path)
+        begin
+          idx = 0
+          while true
+            SocketManager.send_peer(another_server, [Process.pid, :share_udp, idx])
+            key = SocketManager.recv_peer(another_server)
+            break if key.nil?
+            @udp_sockets[key] = another_server.recv_io UDPSocket
+            idx += 1
+          end
+
+          idx = 0
+          while true
+            SocketManager.send_peer(another_server, [Process.pid, :share_tcp, idx])
+            key = SocketManager.recv_peer(another_server)
+            break if key.nil?
+            @tcp_sockets[key] = another_server.recv_io TCPServer
+            idx += 1
+          end
+
+          SocketManager.send_peer(another_server, [Process.pid, :share_unix])
+          res = SocketManager.recv_peer(another_server)
+          raise res if res.is_a?(Exception)
+          @server = another_server.recv_io UNIXServer
+
+          start_server(@path)
+        ensure
+          another_server.close
+        end
+      end
+
       private
 
       def listen_tcp_new(bind_ip, port)
@@ -77,15 +109,17 @@ module ServerEngine
       end
 
       def start_server(path)
-        # return absolute path so that client can connect to this path
-        # when client changed working directory
-        path = File.expand_path(path)
+        unless @server
+          # return absolute path so that client can connect to this path
+          # when client changed working directory
+          path = File.expand_path(path)
 
-        begin
-          old_umask = File.umask(0077) # Protect unix socket from other users
-          @server = UNIXServer.new(path)
-        ensure
-          File.umask(old_umask)
+          begin
+            old_umask = File.umask(0077) # Protect unix socket from other users
+            @server = UNIXServer.new(path)
+          ensure
+            File.umask(old_umask)
+          end
         end
 
         @thread = Thread.new do
@@ -111,19 +145,34 @@ module ServerEngine
         @thread.join if RUBY_VERSION >= "2.2"
       end
 
-      def send_socket(peer, pid, method, bind, port)
-        sock = case method
-               when :listen_tcp
-                 listen_tcp(bind, port)
-               when :listen_udp
-                 listen_udp(bind, port)
-               else
-                 raise ArgumentError, "Unknown method: #{method.inspect}"
-               end
-
-        SocketManager.send_peer(peer, nil)
-
-        peer.send_io sock
+      def send_socket(peer, pid, method, *opts)
+        case method
+        when :listen_tcp
+          bind, port = opts
+          sock = listen_tcp(bind, port)
+          SocketManager.send_peer(peer, nil)
+          peer.send_io sock
+        when :listen_udp
+          bind, port = opts
+          sock = listen_udp(bind, port)
+          SocketManager.send_peer(peer, nil)
+          peer.send_io sock
+        when :share_tcp
+          idx, = opts
+          key = @tcp_sockets.keys[idx]
+          SocketManager.send_peer(peer, key)
+          peer.send_io(@tcp_sockets.values[idx]) if key
+        when :share_udp
+          idx, = opts
+          key = @udp_sockets.keys[idx]
+          SocketManager.send_peer(peer, key)
+          peer.send_io(@udp_sockets.values[idx]) if key
+        when :share_unix
+          SocketManager.send_peer(peer, nil)
+          peer.send_io @server
+        else
+          raise ArgumentError, "Unknown method: #{method.inspect}"
+        end
       end
     end
 

data/lib/serverengine/version.rb

@@ -1,3 +1,3 @@
 module ServerEngine
-  VERSION = "2.3.2"
+  VERSION = "2.4.0"
 end

data/serverengine.gemspec

@@ -19,6 +19,8 @@ Gem::Specification.new do |gem|
   gem.required_ruby_version = ">= 2.3.0"
 
   gem.add_dependency "sigdump", ["~> 0.2.2"]
+  gem.add_dependency "base64", ["~> 0.1"]
+  gem.add_dependency "logger", ["~> 1.4"]
 
   # rake v12.x doesn't work with rspec 2. rspec should be updated to 3
   gem.add_development_dependency "rake", ["~> 13.0"]

data/spec/socket_manager_spec.rb

@@ -55,6 +55,15 @@ describe ServerEngine::SocketManager do
         expect(server.path).to be_between(49152, 65535)
       end
     end
+
+    context 'Server.share_sockets_with_another_server' do
+      it 'not supported' do
+        server = SocketManager::Server.open(server_path)
+        expect { SocketManager::Server.share_sockets_with_another_server(server_path) }.to raise_error(NotImplementedError)
+      ensure
+        server.close
+      end
+    end
   else
     context 'Server.generate_path' do
       it 'returns socket path under /tmp' do
@@ -76,6 +85,165 @@ describe ServerEngine::SocketManager do
         expect(server.path).to include('/tmp/SERVERENGINE_SOCKETMANAGER_')
       end
     end
+
+    context 'Server.share_sockets_with_another_server' do
+      it 'shares listen sockets to another server' do
+        server = SocketManager::Server.open(server_path)
+
+        client = SocketManager::Client.new(server_path)
+        tcp1 = client.listen_tcp('127.0.0.1', 55551)
+        udp1 = client.listen_udp('127.0.0.1', 55561)
+        udp2 = client.listen_udp('127.0.0.1', 55562)
+
+        another_server = SocketManager::Server.share_sockets_with_another_server(server_path)
+
+        expect([
+          another_server.tcp_sockets.keys,
+          another_server.tcp_sockets.values.map(&:addr),
+          another_server.udp_sockets.keys,
+          another_server.udp_sockets.values.map(&:addr),
+        ]).to eq([
+          server.tcp_sockets.keys,
+          server.tcp_sockets.values.map(&:addr),
+          server.udp_sockets.keys,
+          server.udp_sockets.values.map(&:addr),
+        ])
+      ensure
+        tcp1&.close
+        udp1&.close
+        udp2&.close
+        server&.close
+        another_server&.close
+      end
+
+      it 'takes over TCP sockets without downtime' do
+        manager_server = SocketManager::Server.open(server_path)
+        manager_client = SocketManager::Client.new(server_path)
+
+        has_server_started = false
+        # The old server starts listening
+        thread_server = Thread.new do
+          server = manager_client.listen_tcp('127.0.0.1', test_port)
+          has_server_started = true
+          while socket = server.accept
+            incr_test_state(:count)
+            socket.close
+          end
+        ensure
+          server&.close
+        end
+
+        sleep 0.1 until has_server_started
+
+        # The client starts sending data
+        thread_client = Thread.new do
+          100.times do |i|
+            socket = TCPSocket.new('127.0.0.1', test_port)
+            begin
+              socket.write("Hello #{i}\n")
+            ensure
+              socket.close
+            end
+            sleep 0.01
+          end
+        end
+
+        sleep 0.5
+
+        # The new server shares the sockets and starts listening in parallel with the old one
+        thread_new_server = Thread.new do
+          new_manager_server = SocketManager::Server.share_sockets_with_another_server(server_path)
+          server = manager_client.listen_tcp('127.0.0.1', test_port)
+          while socket = server.accept
+            incr_test_state(:count)
+            socket.close
+          end
+        ensure
+          new_manager_server&.close
+          server&.close
+        end
+
+        # Stop the old server
+        sleep 0.1
+        thread_server.kill
+        thread_server.join
+
+        thread_client.join
+        wait_for_stop
+
+        # Confirm that server switching was completed without data loss
+        expect(test_state(:count)).to eq(100)
+      ensure
+        manager_server&.close
+        thread_server&.kill
+        thread_new_server&.kill
+        thread_server&.join
+        thread_new_server&.join
+      end
+
+      it 'takes over UDP sockets without downtime' do
+        manager_server = SocketManager::Server.open(server_path)
+        manager_client = SocketManager::Client.new(server_path)
+
+        has_server_started = false
+        # The old server starts listening
+        thread_server = Thread.new do
+          server = manager_client.listen_udp('127.0.0.1', test_port)
+          has_server_started = true
+          while server.recv(10)
+            incr_test_state(:count)
+          end
+        ensure
+          server&.close
+        end
+
+        sleep 0.1 until has_server_started
+
+        # The client starts sending data
+        thread_client = Thread.new do
+          100.times do |i|
+            socket = UDPSocket.new
+            begin
+              socket.send("Hello #{i}\n", 0, "127.0.0.1", test_port)
+            ensure
+              socket.close
+            end
+            sleep 0.01
+          end
+        end
+
+        sleep 0.5
+
+        # The new server shares the sockets and starts listening in parallel with the old one
+        thread_new_server = Thread.new do
+          new_manager_server = SocketManager::Server.share_sockets_with_another_server(server_path)
+          server = manager_client.listen_udp('127.0.0.1', test_port)
+          while server.recv(10)
+            incr_test_state(:count)
+          end
+        ensure
+          new_manager_server&.close
+          server&.close
+        end
+
+        # Stop the old server
+        sleep 0.1
+        thread_server.kill
+        thread_server.join
+
+        thread_client.join
+        wait_for_stop
+
+        # Confirm that server switching was completed without data loss
+        expect(test_state(:count)).to eq(100)
+      ensure
+        manager_server&.close
+        thread_server&.kill
+        thread_new_server&.kill
+        thread_server&.join
+        thread_new_server&.join
+      end
+    end
   end
 
   context 'with thread' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: serverengine
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.4.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-14 00:00:00.000000000 Z
+date: 2024-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sigdump
@@ -24,6 +24,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -115,8 +143,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/linux.yml"
-- ".github/workflows/windows.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - Changelog
@@ -167,7 +194,7 @@ homepage: https://github.com/fluent/serverengine
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -182,8 +209,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
-signing_key: 
+rubygems_version: 3.5.11
+signing_key:
 specification_version: 4
 summary: ServerEngine - multiprocess server framework
 test_files:

data/.github/workflows/windows.yml

@@ -1,42 +0,0 @@
-name: Testing on Windows
-on:
-  push:
-    branches: [master]
-  pull_request:
-    branches: [master]
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '3.2', '3.1', '2.7' ]
-        os:
-          - windows-latest
-        include:
-          - ruby: '3.0.3'
-            os: windows-latest
-            # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
-            # error code. In addition, we have to specify the path of fiddle by RUBYLIB
-            # because RubyInstaller loads Ruby's bundled fiddle before initializing gem.
-            # See also:
-            # * https://github.com/ruby/fiddle/issues/72
-            # * https://bugs.ruby-lang.org/issues/17813
-            # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
-            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib
-
-    name: Unit testing with Ruby ${{ matrix.ruby }} on ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-      - name: Add Fiddle 1.1.0
-        if: ${{ matrix.ruby == '3.0.3' }}
-        run: gem install fiddle --version 1.1.0
-      - name: Install dependencies
-        run: ridk exec bundle install --jobs 4 --retry 3
-      - name: Run tests
-        run: bundle exec rake spec ${{ matrix.ruby-lib-opt }}