solid_objects 0.13.0 → 0.13.2

data/lib/solid_objects/configuration.rb

@@ -4,6 +4,7 @@ module SolidObjects
   class Configuration
     # @rbs @table_name_prefix: String
     # @rbs @polling_interval: Float
+    # @rbs @idle_polling_interval: Float
     # @rbs @sync_polling_interval: Float
     # @rbs @lease_duration: Float
     # @rbs @lease_renewal_interval: Float
@@ -49,6 +50,7 @@ module SolidObjects
 
     attr_accessor :table_name_prefix,
       :polling_interval,
+      :idle_polling_interval,
       :sync_polling_interval,
       :lease_duration,
       :lease_renewal_interval,
@@ -99,6 +101,7 @@ module SolidObjects
     def initialize
       @table_name_prefix = "solid_objects_"
       @polling_interval = 0.1
+      @idle_polling_interval = 1.0
       @sync_polling_interval = 0.05
       @lease_duration = 30.0
       @lease_renewal_interval = 10.0
@@ -241,6 +244,7 @@ module SolidObjects
     def positive_values
       {
         polling_interval:,
+        idle_polling_interval:,
         sync_polling_interval:,
         lease_duration:,
         lease_renewal_interval:,

data/lib/solid_objects/effect_executor.rb

@@ -1,5 +1,7 @@
 # rbs_inline: enabled
 
+require "solid_objects/polling_backoff"
+
 module SolidObjects
   EffectContext = Data.define(:id, :attempt, :source_message_id, :actor_type, :actor_id)
 
@@ -10,6 +12,7 @@ module SolidObjects
     # @rbs @database_adapter: DatabaseAdapter
     # @rbs @stopped: bool
     # @rbs @shutdown_requested: bool
+    # @rbs @polling_backoff: PollingBackoff
 
     # @rbs (?process_registry: ProcessRegistry, ?database_adapter: DatabaseAdapter) -> void
     def initialize(
@@ -21,6 +24,17 @@ module SolidObjects
       process_registry.register(kind: "effect")
       @stopped = false
       @shutdown_requested = false
+      @polling_backoff = PollingBackoff.new(
+        minimum_interval: SolidObjects.configuration.polling_interval,
+        maximum_interval: SolidObjects.configuration.idle_polling_interval,
+        on_change: ->(transition) do
+          SolidObjects.instrument(
+            :"polling.interval_changed",
+            role: "effects",
+            **transition
+          )
+        end
+      )
     end
 
     # @rbs () -> bool
@@ -50,11 +64,23 @@ module SolidObjects
 
     # @rbs () -> void
     def run
+      ProcessRegistry.warn_if_polling_is_only_cross_process_wake_up
+
       until shutdown_requested?
+        wake_up = SolidObjects.wake_up
+        watch = wake_up.respond_to?(:watch) ? wake_up.watch : wake_up
         worked = run_once
-        next if worked
+        if worked
+          polling_backoff.reset(:work)
+          next
+        end
 
-        SolidObjects.wake_up.wait(timeout: SolidObjects.configuration.polling_interval)
+        notified = watch.wait(timeout: current_polling_interval)
+        if notified == false
+          polling_backoff.record_idle
+        else
+          polling_backoff.reset(:wake_up)
+        end
       end
     ensure
       stop
@@ -76,9 +102,14 @@ module SolidObjects
       @shutdown_requested
     end
 
+    # @rbs () -> Float
+    def current_polling_interval
+      polling_backoff.current_interval
+    end
+
     private
 
-    attr_reader :process_registry, :database_adapter
+    attr_reader :process_registry, :database_adapter, :polling_backoff
 
     # @rbs () -> Effect?
     def claim_next

data/lib/solid_objects/executor.rb

@@ -223,10 +223,10 @@ module SolidObjects
     end
 
     # A reminder is one named alarm per actor, so scheduling a name that is
-    # already armed moves it rather than adding a second. An actor that arms a
-    # reminder per queued item therefore keeps only the last, and nothing else
-    # about that is visible: the write succeeds and the earlier wake-up simply
-    # never happens.
+    # already armed moves it rather than adding a second. Without a key that
+    # name is the operation, so an actor arming a reminder per queued item keeps
+    # only the last; passing schedule a key gives each item its own name and so
+    # its own alarm.
     # Moves are returned rather than reported here, so the report happens after
     # the turn commits. A rolled back turn would otherwise announce an alarm
     # that never moved, which is the opposite of the visibility this event
@@ -239,7 +239,7 @@ module SolidObjects
         reminder.assign_attributes(
           actor_type: instance.actor_type,
           actor_id: instance.actor_id,
-          operation: intent.name,
+          operation: intent.operation,
           arguments: intent.arguments,
           next_run_at: intent.at,
           interval_seconds: intent.interval_seconds,

data/lib/solid_objects/polling_backoff.rb

@@ -0,0 +1,45 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class PollingBackoff
+    # @rbs @minimum_interval: Float
+    # @rbs @maximum_interval: Float
+    # @rbs @current_interval: Float
+    # @rbs @on_change: Proc?
+
+    attr_reader :current_interval
+
+    # @rbs (minimum_interval: Numeric, maximum_interval: Numeric, ?on_change: Proc?) -> void
+    def initialize(minimum_interval:, maximum_interval:, on_change: nil)
+      @minimum_interval = minimum_interval.to_f
+      @maximum_interval = [ @minimum_interval, maximum_interval.to_f ].max
+      @current_interval = @minimum_interval
+      @on_change = on_change
+    end
+
+    # @rbs () -> void
+    def record_idle
+      change([ current_interval * 2, @maximum_interval ].min, :idle)
+    end
+
+    # @rbs (:work | :wake_up) -> void
+    def reset(reason)
+      change(@minimum_interval, reason)
+    end
+
+    private
+
+    # @rbs (Float, :idle | :work | :wake_up) -> void
+    def change(interval, reason)
+      return if interval == current_interval
+
+      previous_interval = current_interval
+      @current_interval = interval
+      @on_change&.call(
+        previous_interval:,
+        current_interval: interval,
+        reason:
+      )
+    end
+  end
+end

data/lib/solid_objects/process_registry.rb

@@ -4,6 +4,9 @@ require "socket"
 
 module SolidObjects
   class ProcessRegistry
+    @polling_warning_mutex = Mutex.new
+    @polling_warning_emitted = false
+
     class << self
       # @rbs (?now: Time) -> Integer
       def cleanup_dead(now: SolidObjects.database_adapter.database_now)
@@ -53,8 +56,56 @@ module SolidObjects
         true
       end
 
+      # @rbs () -> void
+      def warn_if_polling_is_only_cross_process_wake_up
+        return if SolidObjects.configuration.wake_up_adapter
+
+        polling_warning_mutex.synchronize do
+          return if polling_warning_emitted?
+          return unless another_live_process?
+
+          payload = {
+            event: "solid_objects.polling_only_cross_process_wake_up",
+            polling_interval: SolidObjects.configuration.polling_interval,
+            idle_polling_interval: SolidObjects.configuration.idle_polling_interval
+          }
+          SolidObjects.configuration.logger.warn(payload)
+          SolidObjects.instrument(
+            :"polling.only_cross_process_wake_up",
+            **payload.except(:event)
+          )
+          @polling_warning_emitted = true
+        end
+      end
+
+      # @rbs () -> void
+      def reset_polling_warning!
+        polling_warning_mutex.synchronize { @polling_warning_emitted = false }
+      end
+
       private
 
+      # @rbs () -> bool
+      def another_live_process?
+        alive_after = SolidObjects.database_adapter.database_now -
+          SolidObjects.configuration.process_alive_threshold
+        Process
+          .where.not(shutdown_state: "stopped")
+          .where(last_heartbeat_at: alive_after..)
+          .where("hostname <> ? OR pid <> ?", Socket.gethostname, ::Process.pid)
+          .exists?
+      end
+
+      # @rbs () -> bool
+      def polling_warning_emitted?
+        @polling_warning_emitted
+      end
+
+      # @rbs () -> Mutex
+      def polling_warning_mutex
+        @polling_warning_mutex ||= Mutex.new
+      end
+
       # @rbs (Process, Time) -> void
       def cleanup_process(process_record, now)
         deregister(process_record, now:)

data/lib/solid_objects/reminder_scheduler.rb

@@ -1,11 +1,14 @@
 # rbs_inline: enabled
 
+require "solid_objects/polling_backoff"
+
 module SolidObjects
   class ReminderScheduler
     # @rbs @process_registry: ProcessRegistry
     # @rbs @database_adapter: DatabaseAdapter
     # @rbs @stopped: bool
     # @rbs @shutdown_requested: bool
+    # @rbs @polling_backoff: PollingBackoff
 
     # @rbs (?process_registry: ProcessRegistry, ?database_adapter: DatabaseAdapter) -> void
     def initialize(
@@ -17,6 +20,17 @@ module SolidObjects
       process_registry.register(kind: "reminder")
       @stopped = false
       @shutdown_requested = false
+      @polling_backoff = PollingBackoff.new(
+        minimum_interval: SolidObjects.configuration.polling_interval,
+        maximum_interval: SolidObjects.configuration.idle_polling_interval,
+        on_change: ->(transition) do
+          SolidObjects.instrument(
+            :"polling.interval_changed",
+            role: "reminders",
+            **transition
+          )
+        end
+      )
     end
 
     # @rbs (?now: Time?) -> bool
@@ -45,11 +59,23 @@ module SolidObjects
 
     # @rbs () -> void
     def run
+      ProcessRegistry.warn_if_polling_is_only_cross_process_wake_up
+
       until shutdown_requested?
+        wake_up = SolidObjects.wake_up
+        watch = wake_up.respond_to?(:watch) ? wake_up.watch : wake_up
         worked = run_once
-        next if worked
-
-        SolidObjects.wake_up.wait(timeout: SolidObjects.configuration.polling_interval)
+        if worked
+          polling_backoff.reset(:work)
+          next
+        end
+
+        notified = watch.wait(timeout: current_polling_interval)
+        if notified == false
+          polling_backoff.record_idle
+        else
+          polling_backoff.reset(:wake_up)
+        end
       end
     ensure
       stop
@@ -71,9 +97,14 @@ module SolidObjects
       @shutdown_requested
     end
 
+    # @rbs () -> Float
+    def current_polling_interval
+      polling_backoff.current_interval
+    end
+
     private
 
-    attr_reader :process_registry, :database_adapter
+    attr_reader :process_registry, :database_adapter, :polling_backoff
 
     # @rbs (now: Time?) -> Reminder?
     def claim_next(now:)

data/lib/solid_objects/version.rb

@@ -1,5 +1,5 @@
 # rbs_inline: enabled
 
 module SolidObjects
-  VERSION = "0.13.0"
+  VERSION = "0.13.2"
 end

data/lib/solid_objects/wake_up.rb

@@ -2,23 +2,55 @@
 
 module SolidObjects
   class WakeUp
+    class Watch
+      # @rbs @wake_up: WakeUp
+      # @rbs @generation: Integer
+
+      # @rbs (WakeUp, Integer) -> void
+      def initialize(wake_up, generation)
+        @wake_up = wake_up
+        @generation = generation
+      end
+
+      # @rbs (timeout: Numeric) -> bool
+      def wait(timeout:)
+        @wake_up.wait(timeout:, generation: @generation)
+      end
+    end
+
     # @rbs @mutex: Mutex
     # @rbs @condition: Thread::ConditionVariable
+    # @rbs @generation: Integer
 
     # @rbs () -> void
     def initialize
       @mutex = Mutex.new
       @condition = Thread::ConditionVariable.new
+      @generation = 0
     end
 
     # @rbs () -> void
     def signal
-      mutex.synchronize { condition.broadcast }
+      mutex.synchronize do
+        @generation += 1
+        condition.broadcast
+      end
     end
 
-    # @rbs (timeout: Numeric) -> void
-    def wait(timeout:)
-      mutex.synchronize { condition.wait(mutex, timeout) }
+    # @rbs () -> Watch
+    def watch
+      mutex.synchronize { Watch.new(self, @generation) }
+    end
+
+    # @rbs (timeout: Numeric, ?generation: Integer?) -> bool
+    def wait(timeout:, generation: nil)
+      mutex.synchronize do
+        return true if generation && @generation != generation
+
+        generation ||= @generation
+        condition.wait(mutex, timeout)
+        @generation != generation
+      end
     end
 
     private

data/lib/solid_objects/wake_up_adapters/postgresql.rb

@@ -45,6 +45,12 @@ module SolidObjects
         false
       end
 
+      # @rbs () -> self
+      def watch
+        listen
+        self
+      end
+
       # Starts listening before a caller blocks, so a notification sent between
       # startup and the first wait is not missed.
       # @rbs () -> bool

data/lib/solid_objects/wake_up_adapters/redis.rb

@@ -12,6 +12,22 @@ module SolidObjects
     # polling interval remains the upper bound, so a missed or failed
     # notification costs latency rather than correctness.
     class Redis
+      class Watch
+        # @rbs @adapter: Redis
+        # @rbs @generation: Integer
+
+        # @rbs (Redis, Integer) -> void
+        def initialize(adapter, generation)
+          @adapter = adapter
+          @generation = generation
+        end
+
+        # @rbs (timeout: Numeric) -> bool
+        def wait(timeout:)
+          @adapter.wait(timeout:, generation: @generation)
+        end
+      end
+
       CHANNEL = "solid_objects_wake_up"
       FAILED_WAIT_INTERVAL = 0.05
       SUBSCRIBE_TIMEOUT = 5.0
@@ -52,9 +68,9 @@ module SolidObjects
       # The counter is snapshotted before subscribing, and re-checked before
       # blocking, so a signal delivered while this caller was still getting
       # ready is observed rather than absorbed into the new baseline.
-      # @rbs (timeout: Numeric) -> bool
-      def wait(timeout:)
-        signalled = mutex.synchronize { @signalled }
+      # @rbs (timeout: Numeric, ?generation: Integer?) -> bool
+      def wait(timeout:, generation: nil)
+        signalled = generation || mutex.synchronize { @signalled }
         return paced_failure(timeout) unless listen
 
         mutex.synchronize do
@@ -65,6 +81,12 @@ module SolidObjects
         end
       end
 
+      # @rbs () -> Watch
+      def watch
+        listen
+        mutex.synchronize { Watch.new(self, @signalled) }
+      end
+
       # Redis delivers to a subscribed connection only, and a subscribed
       # connection cannot serve other callers, so one background subscription
       # per process fans out to every waiting role in memory. Subscribing

data/lib/solid_objects/worker.rb

@@ -5,6 +5,7 @@ require "solid_objects/activation_manager"
 require "solid_objects/activation"
 require "solid_objects/executor"
 require "solid_objects/lease_renewer"
+require "solid_objects/polling_backoff"
 
 module SolidObjects
   class Worker
@@ -13,6 +14,7 @@ module SolidObjects
     # @rbs @activations: Hash[Integer, Activation]
     # @rbs @stopped: bool
     # @rbs @shutdown_requested: bool
+    # @rbs @polling_backoff: PollingBackoff
 
     # @rbs (?process_registry: ProcessRegistry) -> void
     def initialize(process_registry: ProcessRegistry.new)
@@ -22,6 +24,23 @@ module SolidObjects
       @activations = {}
       @stopped = false
       @shutdown_requested = false
+      @polling_backoff = PollingBackoff.new(
+        minimum_interval: [
+          SolidObjects.configuration.polling_interval,
+          SolidObjects.configuration.lease_renewal_interval
+        ].min,
+        maximum_interval: [
+          SolidObjects.configuration.idle_polling_interval,
+          SolidObjects.configuration.lease_renewal_interval
+        ].min,
+        on_change: ->(transition) do
+          SolidObjects.instrument(
+            :"polling.interval_changed",
+            role: "actors",
+            **transition
+          )
+        end
+      )
     end
 
     # @rbs () -> Integer
@@ -70,11 +89,23 @@ module SolidObjects
 
     # @rbs () -> void
     def run
+      ProcessRegistry.warn_if_polling_is_only_cross_process_wake_up
+
       until shutdown_requested?
+        wake_up = SolidObjects.wake_up
+        watch = wake_up.respond_to?(:watch) ? wake_up.watch : wake_up
         processed = run_once
-        next if processed.positive?
+        if processed.positive?
+          polling_backoff.reset(:work)
+          next
+        end
 
-        SolidObjects.wake_up.wait(timeout: SolidObjects.configuration.polling_interval)
+        notified = watch.wait(timeout: current_polling_interval)
+        if notified == false
+          polling_backoff.record_idle
+        else
+          polling_backoff.reset(:wake_up)
+        end
       end
     ensure
       stop
@@ -107,9 +138,14 @@ module SolidObjects
       @shutdown_requested
     end
 
+    # @rbs () -> Float
+    def current_polling_interval
+      polling_backoff.current_interval
+    end
+
     private
 
-    attr_reader :process_registry, :activation_manager, :activations
+    attr_reader :process_registry, :activation_manager, :activations, :polling_backoff
 
     # @rbs () -> Activation?
     def cached_ready_activation

data/lib/solid_objects.rb

@@ -31,6 +31,7 @@ require "solid_objects/dead_letter_manager"
 require "solid_objects/message_pruner"
 require "solid_objects/instance_pruner"
 require "solid_objects/process_pruner"
+require "solid_objects/administration"
 require "solid_objects/stream_name"
 require "solid_objects/dom_identity"
 require "solid_objects/stream_token"
@@ -52,6 +53,7 @@ require "solid_objects/wake_up"
 require "solid_objects/wake_up_adapters/postgresql"
 require "solid_objects/wake_up_adapters/redis"
 require "solid_objects/wake_up_adapters"
+require "solid_objects/polling_backoff"
 require "solid_objects/effect_registry"
 require "solid_objects/commit_action_registry"
 require "solid_objects/lease"
@@ -136,6 +138,11 @@ module SolidObjects
       @dead_letters ||= DeadLetterManager.new
     end
 
+    # @rbs () -> Administration
+    def administration
+      @administration ||= Administration.new
+    end
+
     # @rbs (String | Symbol) -> String
     def table_name(name)
       "#{configuration.table_name_prefix}#{name}"
@@ -153,6 +160,7 @@ module SolidObjects
 
     # @rbs () -> void
     def reset!
+      ProcessRegistry.reset_polling_warning! if defined?(ProcessRegistry)
       @configuration = Configuration.new
       @registry = ActorRegistry.new
       @client = nil
@@ -162,6 +170,7 @@ module SolidObjects
       @effect_registry = EffectRegistry.new
       @commit_action_registry = CommitActionRegistry.new
       @dead_letters = nil
+      @administration = nil
     end
 
     # @rbs () -> DatabaseAdapter

data/sig/generated/lib/solid_objects/actor.rbs

@@ -32,9 +32,16 @@ module SolidObjects
       def members: () -> [ :name, :arguments ]
     end
 
+    # The reminders table holds a name in 191 characters.
+    REMINDER_NAME_LIMIT: ::Integer
+
+    REMINDER_KEY_SEPARATOR: ::String
+
     class ReminderIntent < Data
       attr_reader name(): untyped
 
+      attr_reader operation(): untyped
+
       attr_reader at(): untyped
 
       attr_reader arguments(): untyped
@@ -43,12 +50,12 @@ module SolidObjects
 
       attr_reader missed_policy(): untyped
 
-      def self.new: (untyped name, untyped at, untyped arguments, untyped interval_seconds, untyped missed_policy) -> instance
-                  | (name: untyped, at: untyped, arguments: untyped, interval_seconds: untyped, missed_policy: untyped) -> instance
+      def self.new: (untyped name, untyped operation, untyped at, untyped arguments, untyped interval_seconds, untyped missed_policy) -> instance
+                  | (name: untyped, operation: untyped, at: untyped, arguments: untyped, interval_seconds: untyped, missed_policy: untyped) -> instance
 
-      def self.members: () -> [ :name, :at, :arguments, :interval_seconds, :missed_policy ]
+      def self.members: () -> [ :name, :operation, :at, :arguments, :interval_seconds, :missed_policy ]
 
-      def members: () -> [ :name, :at, :arguments, :interval_seconds, :missed_policy ]
+      def members: () -> [ :name, :operation, :at, :arguments, :interval_seconds, :missed_policy ]
     end
 
     class OutboundMessageIntent < Data
@@ -157,8 +164,30 @@ module SolidObjects
     # @rbs (Symbol | String, **untyped) -> nil
     def commit_action: (Symbol | String, **untyped) -> nil
 
-    # @rbs (at: Time, ?every: Numeric?, ?missed: Symbol | String) -> OperationDispatcher
-    def schedule: (at: Time, ?every: Numeric?, ?missed: Symbol | String) -> OperationDispatcher
+    # A reminder is identified by its name, and without a key that name is the
+    # operation, so one actor holds one alarm per operation. A key gives an actor
+    # an alarm per item it is waiting on, which is what an actor holding a queue
+    # of scheduled work needs; the key is the caller's own identifier for the
+    # item, and scheduling the same key again moves that item's alarm.
+    # @rbs (at: Time, ?every: Numeric?, ?missed: Symbol | String, ?key: (String | Symbol | Integer)?) -> OperationDispatcher
+    def schedule: (at: Time, ?every: Numeric?, ?missed: Symbol | String, ?key: (String | Symbol | Integer)?) -> OperationDispatcher
+
+    # @rbs ((String | Symbol | Integer)?) -> String?
+    def validated_reminder_key: ((String | Symbol | Integer)?) -> String?
+
+    # A keyed name is the operation, a colon, and the key, so an operation
+    # holding a colon of its own would make two different schedules produce one
+    # name: an unkeyed "deliver:item" and a "deliver" keyed "item" would share a
+    # row, and the second would silently take the first one's alarm. Refusing a
+    # colon in the operation keeps unkeyed names free of colons, which leaves
+    # the two kinds of name disjoint and lets a key hold colons of its own.
+    #
+    # The length is checked on the composed name rather than the key alone,
+    # because a long operation and a short key can exceed the column just as
+    # easily as the reverse. Both are refused here rather than at the insert,
+    # once the turn is already doing work.
+    # @rbs (operation: Symbol | String, key: String?) -> String
+    def reminder_name: (operation: Symbol | String, key: String?) -> String
 
     # @rbs (Reference, ?available_at: Time?, ?idempotency_key: String?) -> OperationDispatcher
     def send_to: (Reference, ?available_at: Time?, ?idempotency_key: String?) -> OperationDispatcher

data/sig/generated/lib/solid_objects/administration.rbs

@@ -0,0 +1,13 @@
+# Generated from lib/solid_objects/administration.rb with RBS::Inline
+
+module SolidObjects
+  class Administration
+    # @rbs (?authorization_context: untyped) -> Array[Hash[Symbol, untyped]]
+    def processes: (?authorization_context: untyped) -> Array[Hash[Symbol, untyped]]
+
+    private
+
+    # @rbs (?authorization_context: untyped) -> void
+    def authorize!: (?authorization_context: untyped) -> void
+  end
+end

data/sig/generated/lib/solid_objects/broadcast_executor.rbs

@@ -2,6 +2,8 @@
 
 module SolidObjects
   class BroadcastExecutor
+    @polling_backoff: PollingBackoff
+
     @shutdown_requested: bool
 
     @stopped: bool
@@ -31,12 +33,17 @@ module SolidObjects
     # @rbs () -> bool
     def shutdown_requested?: () -> bool
 
+    # @rbs () -> Float
+    def current_polling_interval: () -> Float
+
     private
 
     attr_reader process_registry: untyped
 
     attr_reader database_adapter: untyped
 
+    attr_reader polling_backoff: untyped
+
     # @rbs () -> Broadcast?
     def claim_next: () -> Broadcast?