nonnative 3.36.0 → 3.40.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6969c5c21e416e67752d3421af108506fe44730230537c9b426529e28dad7934
-  data.tar.gz: 148ab5d39ef4f367501cca5786de14bc50be175616048a19a792671eee27fc1a
+  metadata.gz: f3513aa619cd9dab8134d9c4e8d22d333dcd2300f92e271cb8dee10ed7c0de2c
+  data.tar.gz: 4aeae3fd49ad6bd1da6b929bfa666c17c3c346a73a56b430ca7214a82ee19698
 SHA512:
-  metadata.gz: 85218dfe59a108dad82130912aaaa1bb0a79e60017595fd089383aafc005e4b9c9e72c74341cd38efc97ac2f31dbf460967cb2dc4eb2fcb34c565ad663b12e41
-  data.tar.gz: 8125a4ef344c00e450f8b03df400cbe2df991f8e2905420673fa2fbd09032c1623f52b540c92cf7ab08c60967b7dfb1d9239c68fcf737341e97c21e1feffa10a
+  metadata.gz: d5c02d41c403e663ba1ffe2e8f36e67fe4a2a36910b60a921d0e02c3ac84b1c68be01158ff223c40ee3e6990e429d79faf79eada9e538195a26652523d89813a
+  data.tar.gz: 6be1f9067df0fd1c121d0604813d3f91b4e9bbe9cbecfcd10aacd8a96a06a76f27a0a42bb3a08398cb7f0c3e68678a42c0f55b62390f304461814332a1e7e1bf

data/README.md

@@ -78,8 +78,9 @@ Common runner fields:
 Process/server fields:
 - `ports`: client-facing ports. These are also used for readiness/shutdown port checks.
 - `timeout`: max time (seconds) for each readiness/shutdown check. For processes, the same value also
-  bounds optional HTTP/gRPC probes and graceful child exit after the stop signal. Defaults to `1.0`.
-  A value of `0` fails immediately; setting it to `nil` programmatically does the same.
+  bounds optional HTTP/gRPC probes and graceful child exit after the stop signal. For servers, it
+  also bounds worker-thread cleanup after the stop hook returns. Defaults to `1.0`. A value of `0`
+  fails immediately; setting it to `nil` programmatically does the same.
 - `wait`: small sleep (seconds) between lifecycle steps.
 - `log`: per-runner log file used by process output redirection or server implementations.
 
@@ -181,8 +182,9 @@ expect(response.code).to eq(200)
 ```
 
 HTTP error statuses are returned as response objects so callers can inspect `.code` and `.body`.
-Request timeouts and broken connections raise their RestClient exceptions; observability requests are
-not retried automatically.
+Broken connections raise their RestClient exceptions, and a request timeout raises either a RestClient
+timeout exception or `Timeout::Error` (the latter when `read_timeout` bounds a Net::HTTP retry of an
+idempotent verb); observability requests are not retried automatically.
 
 `Nonnative.grpc_health` is a helper for the standard gRPC health checking protocol:
 
@@ -437,6 +439,13 @@ module Nonnative
 end
 ```
 
+Rollback calls `perform_stop` for every successfully constructed server, even when a later
+constructor fails before any server thread starts. Keep `perform_stop` safe in that state and use it
+to release resources acquired during initialization. After `perform_stop` requests shutdown,
+Nonnative waits up to the server `timeout` for the owned thread to finish. A thread that exceeds the
+timeout is terminated and causes `Nonnative.stop` to raise `Nonnative::StopError`; the server can
+still be started again afterward.
+
 Set it up programmatically:
 
 ```ruby
@@ -600,8 +609,10 @@ end
 The system allows you to define an in-process HTTP forward proxy server for external systems, e.g. `api.github.com`. This is a server implementation, not a fault-injection service proxy.
 
 The upstream scheme defaults to HTTPS on the scheme's default port; pass `scheme:`/`port:` to
-`Nonnative::HTTPProxyServer.new` to target an `http://` upstream or a non-default port. The proxy
-forwards the request path and query for `GET`, `HEAD`, `POST`, `PUT`, `PATCH`, `DELETE`, and
+`Nonnative::HTTPProxyServer.new` to target an `http://` upstream or a non-default port. The whole
+upstream call defaults to a one-second bound, including Net::HTTP's built-in retry of an
+unresponsive idempotent request; pass `upstream_timeout:` to tune it for a slow dependency. The
+proxy forwards the request path and query for `GET`, `HEAD`, `POST`, `PUT`, `PATCH`, `DELETE`, and
 `OPTIONS`, while removing proxy credentials, `Host`, `Accept-Encoding`, hop-by-hop headers, and
 headers nominated by `Connection` before the upstream request.
 

data/lib/nonnative/configuration.rb

@@ -93,6 +93,8 @@ module Nonnative
       process = Nonnative::ConfigurationProcess.new
       yield process
 
+      raise ArgumentError, "Process '#{process.name}' requires 'command' or 'go'" if process.command.nil?
+
       processes << process
     end
 
@@ -157,7 +159,7 @@ module Nonnative
         tools = go.tools || []
 
         -> { Nonnative.go_argv(tools, go.output, go.executable, go.command, *params) }
-      else
+      elsif process.command
         -> { process.command }
       end
     end

data/lib/nonnative/configuration_server.rb

@@ -14,7 +14,8 @@ module Nonnative
     # @return [Class] a class that implements `#initialize(service)` and the hooks expected by {Nonnative::Server}
     attr_accessor :klass
 
-    # @return [Numeric] readiness timeout (seconds) used when waiting for ports to open/close (defaults to `1.0`)
+    # @return [Numeric] lifecycle timeout (seconds) used for port checks and worker-thread cleanup
+    #   (defaults to `1.0`)
     attr_accessor :timeout
 
     # @return [String] log file path used by server implementations (for example Puma/gRPC log files)

data/lib/nonnative/fault_injection_proxy.rb

@@ -52,6 +52,8 @@ module Nonnative
   # @see Nonnative::Proxy
   # @see Nonnative::SocketPairFactory
   class FaultInjectionProxy < Nonnative::Proxy
+    # Used both to flush the accept-queue barrier and as the aggregate deadline for draining active
+    # worker threads during stop.
     STOP_DRAIN_TIMEOUT = 1
 
     class Connection
@@ -62,10 +64,9 @@ module Nonnative
         @socket = socket
       end
 
-      def close
+      def close_sockets
         pair&.close
         socket.close unless socket.closed?
-        thread&.terminate
       end
     end
 
@@ -88,6 +89,7 @@ module Nonnative
     # @return [void]
     def start
       mutex.synchronize { @stopping = false }
+      logger
       @tcp_server = ::TCPServer.new(service.host, service.port)
       @thread = Thread.new { perform_start }
 
@@ -100,7 +102,7 @@ module Nonnative
     def stop
       server = tcp_server
       mark_stopping
-      close_connections
+      drain_workers(close_connections)
       close_queued_connections(server)
       server&.close
 
@@ -113,6 +115,7 @@ module Nonnative
 
       @tcp_server = nil
       @thread = nil
+      close_logger
 
       Nonnative.logger.info "stopped with host '#{service.host}' and port '#{service.port}' for proxy 'fault_injection'"
     end
@@ -230,7 +233,7 @@ module Nonnative
 
     private
 
-    attr_reader :tcp_server, :thread, :connections, :mutex, :state, :logger, :stopping
+    attr_reader :tcp_server, :thread, :connections, :mutex, :state, :stopping
 
     def perform_start
       loop do
@@ -240,10 +243,9 @@ module Nonnative
           local_socket.close
           next
         end
-        connection_thread = Thread.start(local_socket) do |accepted_socket|
-          accept_connection id, accepted_socket
+        Thread.start(local_socket) do |accepted_socket|
+          accept_connection(id, accepted_socket) if register_connection_thread(id)
         end
-        attach_connection_thread(id, connection_thread)
       end
     rescue IOError, Errno::EBADF
       nil
@@ -281,8 +283,29 @@ module Nonnative
       active_connections.each do |id, connection|
         close_connection(id, connection)
       end
+
+      active_connections.filter_map { |_id, connection| connection.thread }
+    end
+
+    def drain_workers(workers)
+      deadline = monotonic_now + STOP_DRAIN_TIMEOUT
+      workers.each do |worker|
+        worker.join(remaining_time(deadline))
+        terminate_worker(worker)
+      end
     end
 
+    def terminate_worker(worker)
+      return unless worker.alive?
+
+      worker.kill
+      worker.join
+    end
+
+    def remaining_time(deadline) = [deadline - monotonic_now, 0].max
+
+    def monotonic_now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
     def close_queued_connections(server)
       return unless server
 
@@ -309,9 +332,7 @@ module Nonnative
       wait
     end
 
-    def read_state
-      mutex.synchronize { state }
-    end
+    def read_state = mutex.synchronize { state }
 
     def register_connection(id, socket)
       mutex.synchronize do
@@ -322,26 +343,34 @@ module Nonnative
       end
     end
 
-    def mark_stopping
-      mutex.synchronize { @stopping = true }
-    end
+    def mark_stopping = mutex.synchronize { @stopping = true }
 
-    def attach_connection_thread(id, thread)
-      mutex.synchronize { connections[id]&.thread = thread }
-    end
+    def register_connection_thread(id)
+      mutex.synchronize do
+        connection = connections[id]
+        return false unless connection && !stopping
 
-    def attach_connection_pair(id, pair)
-      mutex.synchronize { connections[id]&.pair = pair }
+        connection.thread = Thread.current
+        true
+      end
     end
 
-    def delete_connection(id)
-      mutex.synchronize { connections.delete(id) }
-    end
+    def attach_connection_pair(id, pair) = mutex.synchronize { connections[id]&.pair = pair }
+
+    def delete_connection(id) = mutex.synchronize { connections.delete(id) }
 
     def close_connection(id, connection)
       Nonnative.logger.info "closing connection for '#{id}' for proxy 'fault_injection'"
 
-      connection.close
+      connection.close_sockets
     end
+
+    def close_logger
+      @logger&.close
+    ensure
+      @logger = nil
+    end
+
+    def logger = (@logger ||= Logger.new(service.proxy.log))
   end
 end

data/lib/nonnative/grpc_server.rb

@@ -45,11 +45,14 @@ module Nonnative
       server.run
     end
 
-    # Stops the gRPC server.
+    # Stops the gRPC server after it has started.
+    #
+    # Construction rollback can call this hook before `GRPC::RpcServer#run`; gRPC rejects `stop` in
+    # that state and has not acquired its listener yet, so there is nothing to release.
     #
     # @return [void]
     def perform_stop
-      server.stop
+      server.stop unless server.running_state == :not_started
     end
 
     # Waits until the gRPC server reports it is running, or the configured timeout elapses.

data/lib/nonnative/http_client.rb

@@ -18,7 +18,8 @@ module Nonnative
       @host = host
       @exceptions = [
         RestClient::Exceptions::Timeout,
-        RestClient::ServerBrokeConnection
+        RestClient::ServerBrokeConnection,
+        ::Timeout::Error
       ]
     end
 
@@ -40,7 +41,7 @@ module Nonnative
     # @param opts [Hash] RestClient request options (e.g. `headers`, `read_timeout`, `open_timeout`)
     # @return [RestClient::Response, String] response for non-2xx errors, otherwise the RestClient result
     def get(pathname, opts = {})
-      with_exception do
+      with_exception(opts) do
         resource(pathname, opts).get
       end
     end
@@ -52,7 +53,7 @@ module Nonnative
     # @param opts [Hash] RestClient request options
     # @return [RestClient::Response, String] response for non-2xx errors, otherwise the RestClient result
     def post(pathname, payload, opts = {})
-      with_exception do
+      with_exception(opts) do
         resource(pathname, opts).post(payload)
       end
     end
@@ -63,7 +64,7 @@ module Nonnative
     # @param opts [Hash] RestClient request options
     # @return [RestClient::Response, String] response for non-2xx errors, otherwise the RestClient result
     def delete(pathname, opts = {})
-      with_exception do
+      with_exception(opts) do
         resource(pathname, opts).delete
       end
     end
@@ -75,7 +76,7 @@ module Nonnative
     # @param opts [Hash] RestClient request options
     # @return [RestClient::Response, String] response for non-2xx errors, otherwise the RestClient result
     def put(pathname, payload, opts = {})
-      with_exception do
+      with_exception(opts) do
         resource(pathname, opts).put(payload)
       end
     end
@@ -87,7 +88,7 @@ module Nonnative
     # @param opts [Hash] RestClient request options
     # @return [RestClient::Response, String] response for non-2xx errors, otherwise the RestClient result
     def patch(pathname, payload, opts = {})
-      with_exception do
+      with_exception(opts) do
         resource(pathname, opts).patch(payload)
       end
     end
@@ -98,7 +99,7 @@ module Nonnative
     # @param opts [Hash] RestClient request options (e.g. `headers`, `read_timeout`, `open_timeout`)
     # @return [RestClient::Response, String] response for non-2xx errors, otherwise the RestClient result
     def head(pathname, opts = {})
-      with_exception do
+      with_exception(opts) do
         resource(pathname, opts).head
       end
     end
@@ -109,7 +110,7 @@ module Nonnative
     # @param opts [Hash] RestClient request options (e.g. `headers`, `read_timeout`, `open_timeout`)
     # @return [RestClient::Response, String] response for non-2xx errors, otherwise the RestClient result
     def options(pathname, opts = {})
-      with_exception do
+      with_exception(opts) do
         RestClient::Request.execute(opts.merge(method: :options, url: request_url(pathname)))
       end
     end
@@ -131,8 +132,12 @@ module Nonnative
       URI.join(host, pathname).to_s
     end
 
-    def with_exception
-      yield
+    # Net::HTTP silently retries idempotent verbs once on a read timeout, so bound the whole call
+    # (including any retry) by the wall clock rather than trusting `:read_timeout` alone.
+    def with_exception(opts = {}, &)
+      timeout = opts[:read_timeout]
+
+      timeout ? ::Timeout.timeout(timeout, &) : yield
     rescue *exceptions => e
       raise e
     rescue RestClient::Exception => e

data/lib/nonnative/http_proxy_server.rb

@@ -59,7 +59,8 @@ module Nonnative
     ].freeze
 
     GATEWAY_TIMEOUT_ERRORS = [
-      RestClient::Exceptions::Timeout
+      RestClient::Exceptions::Timeout,
+      ::Timeout::Error
     ].freeze
 
     GATEWAY_CONNECTION_ERRORS = [
@@ -112,10 +113,12 @@ module Nonnative
     # @return [RestClient::Response] response for error statuses, otherwise RestClient return value
     # @raise [Sinatra::Halt] with a gateway status when the upstream is unavailable or times out
     def api_response(method:, url:, headers:, payload: nil)
-      options = { method:, url:, headers: }
+      options = { method:, url:, headers:, open_timeout: settings.upstream_timeout, read_timeout: settings.upstream_timeout }
       options[:payload] = payload unless payload.nil?
 
-      RestClient::Request.execute(options)
+      # Net::HTTP retries idempotent verbs once on a read timeout, so bound the whole
+      # call (including any retry) by the wall clock rather than trusting read_timeout alone.
+      ::Timeout.timeout(settings.upstream_timeout) { RestClient::Request.execute(options) }
     rescue *GATEWAY_TIMEOUT_ERRORS
       halt 504
     rescue *GATEWAY_CONNECTION_ERRORS
@@ -246,11 +249,13 @@ module Nonnative
     # @param service [Nonnative::ConfigurationServer] server configuration
     # @param scheme [String] upstream scheme, `"http"` or `"https"`
     # @param port [Integer, nil] upstream port; `nil` uses the scheme's default port
-    def initialize(host, service, scheme: 'https', port: nil)
+    # @param upstream_timeout [Numeric] maximum seconds for each upstream operation (defaults to `1.0`)
+    def initialize(host, service, scheme: 'https', port: nil, upstream_timeout: ConfigurationRunner::DEFAULT_TIMEOUT)
       http_service = Class.new(Nonnative::HTTPProxy) do
         set :host, host
         set :scheme, scheme
         set :port, port
+        set :upstream_timeout, upstream_timeout
       end
 
       super(http_service.new, service)

data/lib/nonnative/http_server.rb

@@ -72,9 +72,14 @@ module Nonnative
 
     # Gracefully shuts down the Puma server.
     #
+    # Signals shutdown through Puma's own control pipe (`Puma::Server#stop`) rather than calling
+    # `graceful_shutdown` directly, so the accept loop wakes up reliably; calling `graceful_shutdown`
+    # without that signal leaves the accept loop dependent on noticing a closed socket, which is not
+    # guaranteed to happen promptly on every platform.
+    #
     # @return [void]
     def perform_stop
-      server.graceful_shutdown
+      server.stop(true)
     ensure
       close_log
     end

data/lib/nonnative/observability.rb

@@ -14,8 +14,9 @@ module Nonnative
   #
   # Requests are performed using {Nonnative::HTTPClient}, so callers may pass RestClient options
   # such as `headers`, `open_timeout`, and `read_timeout`. HTTP error statuses are returned as response
-  # objects; timeouts and broken connections raise their RestClient exceptions. Requests are not
-  # retried automatically.
+  # objects; broken connections raise their RestClient exceptions, and a timeout raises either a
+  # RestClient timeout exception or `Timeout::Error` (the latter when `read_timeout` bounds a Net::HTTP
+  # retry of an idempotent verb). Requests are not retried automatically.
   #
   # @example
   #   Nonnative.configure do |config|

data/lib/nonnative/pool.rb

@@ -46,8 +46,8 @@ module Nonnative
 
     # Stops all configured runners and collects lifecycle and shutdown errors.
     #
-    # Processes and servers are stopped first and checked for shutdown, then service proxies are
-    # stopped when configured.
+    # Processes and servers are stopped first and checked for shutdown, including bounded server
+    # thread cleanup, then service proxies are stopped when configured.
     #
     # @return [Array<String>] lifecycle and shutdown-check errors collected while stopping
     def stop
@@ -240,7 +240,7 @@ module Nonnative
       id, stopped = Array(values).then { |v| [v.first, v.fetch(1, true)] }
       errors = []
       errors << "Stopped #{runner.name} with id #{id}, though did not respond in time for #{port.description}" unless ready
-      errors << "Stopped #{runner.name} with id #{id}, though the process did not exit in time" unless stopped
+      errors << "Stopped #{runner.name} with id #{id}, though the #{exit_subject(runner)} did not exit in time" unless stopped
       errors
     end
 
@@ -248,7 +248,7 @@ module Nonnative
       id, stopped = Array(values).then { |v| [v.first, v.fetch(1, true)] }
       errors = []
       errors << "Rollback failed for #{runner.name} with id #{id}, because it did not stop in time for #{port.description}" unless ready
-      errors << "Rollback failed for #{runner.name} with id #{id}, because the process did not exit in time" unless stopped
+      errors << "Rollback failed for #{runner.name} with id #{id}, because the #{exit_subject(runner)} did not exit in time" unless stopped
       errors
     end
 
@@ -265,6 +265,10 @@ module Nonnative
       "Started #{runner_name(service)}, though did not respond in time for readiness: #{checks.map(&:endpoint).join(', ')}"
     end
 
+    def exit_subject(runner)
+      runner.is_a?(Nonnative::Server) ? 'server thread' : 'process'
+    end
+
     def runner_name(type)
       name = type.name
       return "runner '#{name}'" if name

data/lib/nonnative/server.rb

@@ -12,6 +12,9 @@ module Nonnative
   # - {#perform_start} (to bind/listen and begin serving), and
   # - {#perform_stop} (to gracefully shut down).
   #
+  # {#stop} calls the stop hook once for every successfully constructed lifecycle, even if startup
+  # never created a worker thread. This lets rollback release resources acquired by a constructor.
+  #
   # The underlying configuration is a {Nonnative::ConfigurationServer}.
   #
   # @see Nonnative::ConfigurationServer
@@ -22,17 +25,22 @@ module Nonnative
       super
 
       @timeout = Nonnative::Timeout.new(service.timeout)
+      @cleanup_required = true
     end
 
-    # Starts the server thread if it is not already started.
+    # Starts the server thread if it is not already running.
+    #
+    # A thread retained from a stop that exceeded its timeout is treated as not running, so a server
+    # can be restarted even if a prior {#stop} never observed that thread's actual termination.
     #
     # @return [Array<(Integer, TrueClass)>]
     #   a tuple of:
     #   - a stable identifier for this server instance (`object_id`)
     #   - `true` (thread creation itself is considered started; readiness is checked separately)
     def start
-      unless thread
+      unless thread&.alive?
         @error = nil
+        @cleanup_required = true
         @thread = Thread.new do
           perform_start
         rescue StandardError => e
@@ -49,23 +57,27 @@ module Nonnative
       [object_id, true]
     end
 
-    # Stops the server if it is running.
+    # Stops the server if its current lifecycle still requires cleanup.
     #
-    # Calls {#perform_stop}, terminates the server thread, and waits briefly.
+    # Calls {#perform_stop} even if startup never created a worker thread, then waits up to the
+    # configured timeout for any owned thread to finish. A thread that exceeds the timeout is
+    # terminated and reported through {Nonnative::Pool}. A later stop retries draining a retained
+    # thread without calling {#perform_stop} again.
     #
-    # @return [Integer] the server identifier (`object_id`)
+    # @return [Integer, Array<(Integer, FalseClass)>] the server identifier when cleanup finishes, or
+    #   the identifier and `false` when the owned thread exceeds the configured timeout
     def stop
-      if thread
-        perform_stop
-        thread.terminate
+      owned_thread = thread
+      return object_id unless @cleanup_required || owned_thread
 
-        @thread = nil
-        wait_stop
+      perform_cleanup
+      stopped = drain_thread(owned_thread) == :drained
+      wait_stop
+      @thread = nil unless owned_thread&.alive?
 
-        Nonnative.logger.info "stopped server '#{service.name}'"
-      end
+      Nonnative.logger.info "stopped server '#{service.name}'" if stopped
 
-      object_id
+      stopped ? object_id : [object_id, false]
     end
 
     # Describes how the server thread terminated before becoming ready, for lifecycle diagnostics.
@@ -85,5 +97,20 @@ module Nonnative
     private
 
     attr_reader :thread, :timeout, :error
+
+    def perform_cleanup
+      return unless @cleanup_required
+
+      perform_stop
+      @cleanup_required = false
+    end
+
+    def drain_thread(owned_thread)
+      return :drained if owned_thread.nil? || !owned_thread.alive?
+      return :drained if timeout.perform { owned_thread.join }
+
+      owned_thread.terminate
+      :timed_out
+    end
   end
 end

data/lib/nonnative/version.rb

@@ -4,5 +4,5 @@ module Nonnative
   # The current gem version.
   #
   # @return [String]
-  VERSION = '3.36.0'
+  VERSION = '3.40.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nonnative
 version: !ruby/object:Gem::Version
-  version: 3.36.0
+  version: 3.40.0
 platform: ruby
 authors:
 - Alejandro Falkowski