solid_objects 0.7.3 → 0.9.0

data/lib/solid_objects/supervisor.rb

@@ -4,7 +4,10 @@ module SolidObjects
   class Supervisor
     # @rbs @components: Array[Worker | EffectExecutor | ReminderScheduler | BroadcastExecutor]
     # @rbs @threads: Array[Thread]
+    # @rbs @monitor: Thread?
     # @rbs @started: bool
+    # @rbs @cleaned_up_at: Float
+    # @rbs @lifecycle: Thread::Mutex
 
     # @rbs (?worker_count: Integer, ?effect_worker_count: Integer, ?broadcast_worker_count: Integer, ?reminder_scheduler_count: Integer) -> void
     def initialize(
@@ -20,7 +23,10 @@ module SolidObjects
         reminder_scheduler_count:
       )
       @threads = []
+      @monitor = nil
       @started = false
+      @cleaned_up_at = nil
+      @lifecycle = Thread::Mutex.new
     end
 
     # @rbs () -> void
@@ -36,7 +42,8 @@ module SolidObjects
       return if @started
 
       @started = true
-      @threads = components.map { |component| Thread.new { component.run } }
+      @threads = components.map { |component| supervise(component) }
+      @monitor = Thread.new { monitor_loop }
       SolidObjects.instrument(:"supervisor.started", component_count: components.length)
     end
 
@@ -44,17 +51,129 @@ module SolidObjects
     def stop
       return unless @started
 
-      components.each(&:request_shutdown)
-      join_until_timeout
-      components.reject(&:stopped?).each(&:stop)
-      @started = false
-      SolidObjects.instrument(:"supervisor.stopped", component_count: components.length)
+      begin
+        # Flipping the flag under the same lock replacement takes means a
+        # replacement either completes before shutdown reads the component
+        # list, or never starts.
+        @lifecycle.synchronize { @started = false }
+        stop_monitor
+        components.each(&:request_shutdown)
+        join_until_timeout
+        components.reject(&:stopped?).each(&:stop)
+      ensure
+        # Connections held outside the pool must be released even when a
+        # component fails to stop, or they accumulate across restarts.
+        release_wake_up
+        @monitor = nil
+        SolidObjects.instrument(:"supervisor.stopped", component_count: components.length)
+      end
     end
 
     private
 
     attr_reader :components, :threads
 
+    # A role that raises leaves its thread dead. Without replacement the
+    # process keeps running while quietly doing less work, so the supervisor
+    # watches its threads and restarts any that stopped before shutdown.
+    # A failing pass must not stop supervision, and must not retry without
+    # pacing either: a persistently failing database would otherwise spin.
+    # @rbs () -> void
+    def monitor_loop
+      while @started
+        begin
+          replace_dead_roles
+          cleanup_dead_processes
+        rescue => error
+          SolidObjects.instrument(
+            :"supervisor.monitor_failed",
+            error_class: error.class.name,
+            error_message: error.message
+          )
+        end
+        sleep SolidObjects.configuration.supervisor_monitor_interval
+      end
+    end
+
+    # A role that raises runs its own shutdown cleanup on the way out, so a
+    # crashed component reports itself stopped exactly like one that was asked
+    # to stop. While the supervisor is still running, a dead thread can only
+    # mean a crash, so replacement keys on the supervisor rather than on the
+    # component. The crashed instance has already released its process record,
+    # so a fresh one takes its place.
+    # @rbs () -> void
+    def replace_dead_roles
+      components.each_with_index do |component, index|
+        thread = threads[index]
+        next if thread&.alive?
+
+        replaced = @lifecycle.synchronize do
+          next false unless @started
+
+          replacement = component.class.new
+          components[index] = replacement
+          threads[index] = supervise(replacement)
+          replacement
+        end
+        break unless replaced
+
+        SolidObjects.instrument(
+          :"supervisor.role_replaced",
+          role: replaced.class.name,
+          error_class: thread_error(thread)
+        )
+      end
+    end
+
+    # @rbs (Thread?) -> String?
+    def thread_error(thread)
+      thread&.join
+      nil
+    rescue => error
+      error.class.name
+    end
+
+    # @rbs () -> void
+    def cleanup_dead_processes
+      interval = SolidObjects.configuration.dead_process_cleanup_interval
+      return unless interval.positive?
+      return if @cleaned_up_at && monotonic_now - @cleaned_up_at < interval
+
+      @cleaned_up_at = monotonic_now
+      ProcessRegistry.cleanup_dead
+    end
+
+    # @rbs (untyped) -> Thread
+    def supervise(component)
+      Thread.new { component.run }
+    end
+
+    # The monitor only performs maintenance, so shutdown must never return while
+    # it is still alive: a pass blocked on the database would otherwise outlive
+    # the supervisor that owns it.
+    # @rbs () -> void
+    def stop_monitor
+      monitor = @monitor
+      @monitor = nil
+      return unless monitor
+
+      monitor.join(SolidObjects.configuration.shutdown_timeout)
+      monitor.kill if monitor.alive?
+      monitor.join(SolidObjects.configuration.supervisor_monitor_interval)
+    end
+
+    # A wake-up adapter may hold connections outside the pool, which would
+    # otherwise accumulate across restarts in one process.
+    # @rbs () -> void
+    def release_wake_up
+      wake_up = SolidObjects.wake_up
+      return unless wake_up.respond_to?(:stop)
+
+      wake_up.stop
+    rescue
+      nil
+    end
+
     # @rbs (worker_count: Integer, effect_worker_count: Integer, broadcast_worker_count: Integer, reminder_scheduler_count: Integer) -> Array[Worker | EffectExecutor | ReminderScheduler | BroadcastExecutor]
     def build_components(
       worker_count:,

data/lib/solid_objects/version.rb

@@ -1,5 +1,5 @@
 # rbs_inline: enabled
 
 module SolidObjects
-  VERSION = "0.7.3"
+  VERSION = "0.9.0"
 end

data/lib/solid_objects/wake_up_adapters/postgresql.rb

@@ -0,0 +1,136 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  module WakeUpAdapters
+    # Wakes runtime roles across processes using PostgreSQL notifications.
+    #
+    # The in-process wake-up cannot reach another process, so a commit in a web
+    # process leaves a worker waiting out its polling interval. This adapter
+    # keeps that polling interval as the upper bound and delivers a notification
+    # when one is available, so a missed or failed notification costs latency
+    # rather than correctness.
+    class Postgresql
+      CHANNEL = "solid_objects_wake_up"
+      FAILED_WAIT_INTERVAL = 0.05
+
+      # @rbs @channel: String
+      # @rbs @mutex: Thread::Mutex
+      # @rbs @connections: Array[untyped]
+
+      attr_reader :channel
+
+      # @rbs (?channel: String) -> void
+      def initialize(channel: CHANNEL)
+        @channel = channel
+        @mutex = Thread::Mutex.new
+        @connections = []
+      end
+
+      # @rbs () -> bool
+      def signal
+        notify_channel
+        true
+      rescue => error
+        instrument_failure(:signal, error)
+        false
+      end
+
+      # @rbs (timeout: Numeric) -> bool
+      def wait(timeout:)
+        connection = listening_connection
+        !connection.raw_connection.wait_for_notify(timeout.to_f).nil?
+      rescue => error
+        instrument_failure(:wait, error)
+        pace_after_failure(timeout)
+        false
+      end
+
+      # Starts listening before a caller blocks, so a notification sent between
+      # startup and the first wait is not missed.
+      # @rbs () -> bool
+      def listen
+        listening_connection
+        true
+      rescue => error
+        instrument_failure(:listen, error)
+        false
+      end
+
+      # @rbs () -> bool
+      def stop
+        open = mutex.synchronize do
+          listening = connections.dup
+          connections.clear
+          listening
+        end
+        Thread.current[thread_key] = nil
+        open.each { |connection| disconnect(connection) }
+        open.any?
+      end
+
+      private
+
+      attr_reader :mutex, :connections
+
+      # @rbs () -> void
+      def notify_channel
+        Record.connection_pool.with_connection do |connection|
+          connection.execute("NOTIFY #{connection.quote_table_name(channel)}")
+        end
+      end
+
+      # A listening connection is dedicated and per thread. `LISTEN` is per
+      # connection, a blocking wait must not hold a connection the rest of the
+      # runtime needs, and one connection cannot serve concurrent waiters: the
+      # supervisor shares one adapter across roles, and a notification consumed
+      # by one waiter would leave the others asleep until their poll expired.
+      # @rbs () -> untyped
+      def listening_connection
+        connection = Thread.current[thread_key]
+        return connection if connection&.active?
+
+        open_listening_connection
+      end
+
+      # @rbs () -> untyped
+      def open_listening_connection
+        connection = Record.connection_pool.send(:new_connection)
+        connection.execute("LISTEN #{connection.quote_table_name(channel)}")
+        Thread.current[thread_key] = connection
+        mutex.synchronize { connections << connection }
+        connection
+      end
+
+      # @rbs () -> Symbol
+      def thread_key
+        :"solid_objects_wake_up_#{object_id}"
+      end
+
+      # @rbs (untyped) -> void
+      def disconnect(connection)
+        connection.disconnect!
+      rescue
+        nil
+      end
+
+      # @rbs (Numeric) -> void
+      def pace_after_failure(timeout)
+        interval = [ timeout.to_f, FAILED_WAIT_INTERVAL ].min
+        return unless interval.positive?
+
+        sleep interval
+      end
+
+      # @rbs (Symbol, Exception) -> void
+      def instrument_failure(operation, error)
+        SolidObjects.instrument(
+          :"wake_up.failed",
+          adapter: "postgresql",
+          operation: operation.to_s,
+          error_class: error.class.name,
+          error_message: error.message
+        )
+      end
+    end
+  end
+end

data/lib/solid_objects/wake_up_adapters/redis.rb

@@ -0,0 +1,183 @@
+# rbs_inline: enabled
+
+require "timeout"
+
+module SolidObjects
+  module WakeUpAdapters
+    # Wakes runtime roles across processes using Redis publish/subscribe.
+    #
+    # MySQL has no notification primitive, so this is the cross-process option
+    # for applications that cannot use PostgreSQL notifications. It is optional
+    # in every sense: the `redis` gem is not a dependency of this gem, and the
+    # polling interval remains the upper bound, so a missed or failed
+    # notification costs latency rather than correctness.
+    class Redis
+      CHANNEL = "solid_objects_wake_up"
+      FAILED_WAIT_INTERVAL = 0.05
+      SUBSCRIBE_TIMEOUT = 5.0
+
+      # @rbs @channel: String
+      # @rbs @url: String?
+      # @rbs @client: untyped
+      # @rbs @mutex: Thread::Mutex
+      # @rbs @condition: Thread::ConditionVariable
+      # @rbs @subscriber: Thread?
+      # @rbs @subscription: untyped
+      # @rbs @signalled: Integer
+
+      attr_reader :channel
+
+      # @rbs (?channel: String, ?url: String?, ?client: untyped) -> void
+      def initialize(channel: CHANNEL, url: nil, client: nil)
+        @channel = channel
+        @url = url
+        @client = client
+        @mutex = Thread::Mutex.new
+        @condition = Thread::ConditionVariable.new
+        @subscriber = nil
+        @subscription = nil
+        @signalled = 0
+        validate_client!
+      end
+
+      # @rbs () -> bool
+      def signal
+        publisher.publish(channel, "1")
+        true
+      rescue => error
+        instrument_failure(:signal, error)
+        false
+      end
+
+      # The counter is snapshotted before subscribing, and re-checked before
+      # blocking, so a signal delivered while this caller was still getting
+      # ready is observed rather than absorbed into the new baseline.
+      # @rbs (timeout: Numeric) -> bool
+      def wait(timeout:)
+        signalled = mutex.synchronize { @signalled }
+        return paced_failure(timeout) unless listen
+
+        mutex.synchronize do
+          return true unless @signalled == signalled
+
+          condition.wait(mutex, timeout.to_f)
+          @signalled != signalled
+        end
+      end
+
+      # Redis delivers to a subscribed connection only, and a subscribed
+      # connection cannot serve other callers, so one background subscription
+      # per process fans out to every waiting role in memory. Subscribing
+      # eagerly also closes the window where a signal sent during startup would
+      # be missed.
+      # @rbs () -> bool
+      def listen
+        mutex.synchronize do
+          return true if @subscriber&.alive?
+
+          ready = Queue.new
+          @subscriber = Thread.new { subscribe_loop(ready) }
+          Timeout.timeout(SUBSCRIBE_TIMEOUT) { ready.pop } == :subscribed
+        end
+      rescue => error
+        instrument_failure(:listen, error)
+        false
+      end
+
+      # @rbs () -> bool
+      def stop
+        subscriber = mutex.synchronize do
+          thread = @subscriber
+          @subscriber = nil
+          thread
+        end
+        return false unless subscriber
+
+        disconnect(@subscription)
+        subscriber.join(SUBSCRIBE_TIMEOUT)
+        subscriber.kill if subscriber.alive?
+        true
+      end
+
+      private
+
+      attr_reader :mutex, :condition, :url
+
+      # @rbs (Queue) -> void
+      def subscribe_loop(ready)
+        connection = build_client
+        @subscription = connection
+        connection.subscribe(channel) do |on|
+          on.subscribe { ready << :subscribed }
+          on.message { broadcast }
+        end
+      rescue => error
+        instrument_failure(:subscribe, error)
+        ready << :failed
+      end
+
+      # @rbs () -> void
+      def broadcast
+        mutex.synchronize do
+          @signalled += 1
+          condition.broadcast
+        end
+      end
+
+      # @rbs (Numeric) -> bool
+      def paced_failure(timeout)
+        pace_after_failure(timeout)
+        false
+      end
+
+      # @rbs () -> untyped
+      def publisher
+        @publisher ||= build_client
+      end
+
+      # @rbs () -> untyped
+      def build_client
+        return @client.call if @client.respond_to?(:call)
+
+        require "redis"
+        url ? ::Redis.new(url:) : ::Redis.new
+      rescue LoadError
+        raise ArgumentError,
+          "the redis gem is required for SolidObjects::WakeUpAdapters::Redis"
+      end
+
+      # @rbs () -> void
+      def validate_client!
+        return if @client.nil? || @client.respond_to?(:call)
+
+        raise ArgumentError, "client must respond to call and return a Redis client"
+      end
+
+      # @rbs (untyped) -> void
+      def disconnect(connection)
+        connection&.close
+      rescue
+        nil
+      end
+
+      # @rbs (Numeric) -> void
+      def pace_after_failure(timeout)
+        interval = [ timeout.to_f, FAILED_WAIT_INTERVAL ].min
+        return unless interval.positive?
+
+        sleep interval
+      end
+
+      # @rbs (Symbol, Exception) -> void
+      def instrument_failure(operation, error)
+        SolidObjects.instrument(
+          :"wake_up.failed",
+          adapter: "redis",
+          operation: operation.to_s,
+          error_class: error.class.name,
+          error_message: error.message
+        )
+      end
+    end
+  end
+end

data/lib/solid_objects/wake_up_adapters.rb

@@ -0,0 +1,23 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  module WakeUpAdapters
+    module_function
+
+    # Returns the best wake-up strategy for a connection: cross-process
+    # notifications where the database provides them, and the in-process
+    # default everywhere else.
+    #
+    # This is deliberately not the default. A notification adapter opens a
+    # connection per waiting thread outside the pool, and `LISTEN` does not
+    # survive a transaction-pooling proxy such as PgBouncer, so adopting it is
+    # a deployment decision rather than an upgrade side effect.
+    #
+    # @rbs (?untyped) -> untyped
+    def for(connection = Record.connection)
+      return Postgresql.new if connection.adapter_name.match?(/postgres/i)
+
+      WakeUp.new
+    end
+  end
+end

data/lib/solid_objects.rb

@@ -46,6 +46,9 @@ require "solid_objects/actor_view"
 require "solid_objects/actor_channel"
 require "solid_objects/action_cable_broadcast_adapter"
 require "solid_objects/wake_up"
+require "solid_objects/wake_up_adapters/postgresql"
+require "solid_objects/wake_up_adapters/redis"
+require "solid_objects/wake_up_adapters"
 require "solid_objects/effect_registry"
 require "solid_objects/commit_action_registry"
 require "solid_objects/lease"

data/sig/generated/lib/solid_objects/configuration.rbs

@@ -2,10 +2,12 @@
 
 module SolidObjects
   class Configuration
-    @process_alive_threshold: Float
-
     @shutdown_timeout: Float
 
+    @supervisor_monitor_interval: Float
+
+    @dead_process_cleanup_interval: Float
+
     @message_retention: Numeric
 
     @message_retention_by_actor_type: Hash[String, Numeric]
@@ -82,6 +84,8 @@ module SolidObjects
 
     @process_heartbeat_interval: Float
 
+    @process_alive_threshold: Float
+
     attr_accessor table_name_prefix: untyped
 
     attr_accessor polling_interval: untyped
@@ -120,6 +124,10 @@ module SolidObjects
 
     attr_accessor shutdown_timeout: untyped
 
+    attr_accessor supervisor_monitor_interval: untyped
+
+    attr_accessor dead_process_cleanup_interval: untyped
+
     attr_accessor message_retention: untyped
 
     attr_accessor message_retention_by_actor_type: untyped

data/sig/generated/lib/solid_objects/database_adapter.rbs

@@ -16,6 +16,23 @@ module SolidObjects
     # @rbs (untyped) -> void
     def initialize: (untyped) -> void
 
+    # The oldest server the adapter has been exercised against. Reported rather
+    # than enforced: refusing to boot on an untested server would be a worse
+    # failure than running on one.
+    # @rbs () -> Gem::Version?
+    def minimum_server_version: () -> Gem::Version?
+
+    # @rbs () -> Gem::Version
+    def server_version: () -> Gem::Version
+
+    # One observed version decides both the status and the message. Reading it
+    # again could let a transient failure replace an already determined result.
+    # @rbs (?Gem::Version?) -> Array[String]
+    def unsupported_server_reasons: (?Gem::Version?) -> Array[String]
+
+    # @rbs () -> Array[String]
+    def additional_server_reasons: () -> Array[String]
+
     # @rbs () -> bool
     def supports_skip_locked?: () -> bool
 

data/sig/generated/lib/solid_objects/database_adapters/mysql.rbs

@@ -9,6 +9,17 @@ module SolidObjects
       # @rbs () -> String
       def claim_lock: () -> String
 
+      # A non-transactional engine would silently break fenced commits, so the
+      # storage engine is verified rather than assumed.
+      # @rbs () -> Array[String]
+      def additional_server_reasons: () -> Array[String]
+
+      # @rbs () -> Array[String]
+      def non_innodb_tables: () -> Array[String]
+
+      # @rbs () -> Gem::Version?
+      def minimum_server_version: () -> Gem::Version?
+
       # @rbs () -> String
       def current_time_expression: () -> String
 

data/sig/generated/lib/solid_objects/database_adapters/postgresql.rbs

@@ -3,6 +3,14 @@
 module SolidObjects
   module DatabaseAdapters
     class Postgresql < DatabaseAdapter
+      # @rbs () -> Gem::Version?
+      def minimum_server_version: () -> Gem::Version?
+
+      # PostgreSQL reports a packed integer, 170010 for 17.10, so comparing it
+      # directly would make every server look newer than any minimum.
+      # @rbs () -> Gem::Version
+      def server_version: () -> Gem::Version
+
       # @rbs () -> bool
       def supports_skip_locked?: () -> bool
 

data/sig/generated/lib/solid_objects/database_adapters/sqlite.rbs

@@ -11,6 +11,9 @@ module SolidObjects
 
       LOCK_RETRY_CONDITION: untyped
 
+      # @rbs () -> Gem::Version?
+      def minimum_server_version: () -> Gem::Version?
+
       # @rbs () -> String
       def current_time_expression: () -> String
 
@@ -52,6 +55,8 @@ module SolidObjects
       # @rbs (Integer) -> void
       def wait_before_busy_retry: (Integer) -> void
 
+      # The deadline can expire between the check above and this wait, which
+      # would otherwise ask for a negative interval.
       # @rbs () -> void
       def wait_before_retry: () -> void
     end

data/sig/generated/lib/solid_objects/doctor.rbs

@@ -72,6 +72,9 @@ module SolidObjects
     # @rbs () -> Check
     def check_authorization: () -> Check
 
+    # @rbs () -> Check
+    def check_database_server: () -> Check
+
     # @rbs () -> Check
     def check_runtime: () -> Check