solid_objects 0.12.1 → 0.13.1

data/lib/solid_objects/polling_backoff.rb

@@ -0,0 +1,45 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class PollingBackoff
+    # @rbs @minimum_interval: Float
+    # @rbs @maximum_interval: Float
+    # @rbs @current_interval: Float
+    # @rbs @on_change: Proc?
+
+    attr_reader :current_interval
+
+    # @rbs (minimum_interval: Numeric, maximum_interval: Numeric, ?on_change: Proc?) -> void
+    def initialize(minimum_interval:, maximum_interval:, on_change: nil)
+      @minimum_interval = minimum_interval.to_f
+      @maximum_interval = [ @minimum_interval, maximum_interval.to_f ].max
+      @current_interval = @minimum_interval
+      @on_change = on_change
+    end
+
+    # @rbs () -> void
+    def record_idle
+      change([ current_interval * 2, @maximum_interval ].min, :idle)
+    end
+
+    # @rbs (:work | :wake_up) -> void
+    def reset(reason)
+      change(@minimum_interval, reason)
+    end
+
+    private
+
+    # @rbs (Float, :idle | :work | :wake_up) -> void
+    def change(interval, reason)
+      return if interval == current_interval
+
+      previous_interval = current_interval
+      @current_interval = interval
+      @on_change&.call(
+        previous_interval:,
+        current_interval: interval,
+        reason:
+      )
+    end
+  end
+end

data/lib/solid_objects/process_registry.rb

@@ -4,6 +4,9 @@ require "socket"
 
 module SolidObjects
   class ProcessRegistry
+    @polling_warning_mutex = Mutex.new
+    @polling_warning_emitted = false
+
     class << self
       # @rbs (?now: Time) -> Integer
       def cleanup_dead(now: SolidObjects.database_adapter.database_now)
@@ -53,8 +56,56 @@ module SolidObjects
         true
       end
 
+      # @rbs () -> void
+      def warn_if_polling_is_only_cross_process_wake_up
+        return if SolidObjects.configuration.wake_up_adapter
+
+        polling_warning_mutex.synchronize do
+          return if polling_warning_emitted?
+          return unless another_live_process?
+
+          payload = {
+            event: "solid_objects.polling_only_cross_process_wake_up",
+            polling_interval: SolidObjects.configuration.polling_interval,
+            idle_polling_interval: SolidObjects.configuration.idle_polling_interval
+          }
+          SolidObjects.configuration.logger.warn(payload)
+          SolidObjects.instrument(
+            :"polling.only_cross_process_wake_up",
+            **payload.except(:event)
+          )
+          @polling_warning_emitted = true
+        end
+      end
+
+      # @rbs () -> void
+      def reset_polling_warning!
+        polling_warning_mutex.synchronize { @polling_warning_emitted = false }
+      end
+
       private
 
+      # @rbs () -> bool
+      def another_live_process?
+        alive_after = SolidObjects.database_adapter.database_now -
+          SolidObjects.configuration.process_alive_threshold
+        Process
+          .where.not(shutdown_state: "stopped")
+          .where(last_heartbeat_at: alive_after..)
+          .where("hostname <> ? OR pid <> ?", Socket.gethostname, ::Process.pid)
+          .exists?
+      end
+
+      # @rbs () -> bool
+      def polling_warning_emitted?
+        @polling_warning_emitted
+      end
+
+      # @rbs () -> Mutex
+      def polling_warning_mutex
+        @polling_warning_mutex ||= Mutex.new
+      end
+
       # @rbs (Process, Time) -> void
       def cleanup_process(process_record, now)
         deregister(process_record, now:)

data/lib/solid_objects/reminder_scheduler.rb

@@ -1,11 +1,14 @@
 # rbs_inline: enabled
 
+require "solid_objects/polling_backoff"
+
 module SolidObjects
   class ReminderScheduler
     # @rbs @process_registry: ProcessRegistry
     # @rbs @database_adapter: DatabaseAdapter
     # @rbs @stopped: bool
     # @rbs @shutdown_requested: bool
+    # @rbs @polling_backoff: PollingBackoff
 
     # @rbs (?process_registry: ProcessRegistry, ?database_adapter: DatabaseAdapter) -> void
     def initialize(
@@ -17,6 +20,17 @@ module SolidObjects
       process_registry.register(kind: "reminder")
       @stopped = false
       @shutdown_requested = false
+      @polling_backoff = PollingBackoff.new(
+        minimum_interval: SolidObjects.configuration.polling_interval,
+        maximum_interval: SolidObjects.configuration.idle_polling_interval,
+        on_change: ->(transition) do
+          SolidObjects.instrument(
+            :"polling.interval_changed",
+            role: "reminders",
+            **transition
+          )
+        end
+      )
     end
 
     # @rbs (?now: Time?) -> bool
@@ -45,11 +59,23 @@ module SolidObjects
 
     # @rbs () -> void
     def run
+      ProcessRegistry.warn_if_polling_is_only_cross_process_wake_up
+
       until shutdown_requested?
+        wake_up = SolidObjects.wake_up
+        watch = wake_up.respond_to?(:watch) ? wake_up.watch : wake_up
         worked = run_once
-        next if worked
-
-        SolidObjects.wake_up.wait(timeout: SolidObjects.configuration.polling_interval)
+        if worked
+          polling_backoff.reset(:work)
+          next
+        end
+
+        notified = watch.wait(timeout: current_polling_interval)
+        if notified == false
+          polling_backoff.record_idle
+        else
+          polling_backoff.reset(:wake_up)
+        end
       end
     ensure
       stop
@@ -71,9 +97,14 @@ module SolidObjects
       @shutdown_requested
     end
 
+    # @rbs () -> Float
+    def current_polling_interval
+      polling_backoff.current_interval
+    end
+
     private
 
-    attr_reader :process_registry, :database_adapter
+    attr_reader :process_registry, :database_adapter, :polling_backoff
 
     # @rbs (now: Time?) -> Reminder?
     def claim_next(now:)

data/lib/solid_objects/version.rb

@@ -1,5 +1,5 @@
 # rbs_inline: enabled
 
 module SolidObjects
-  VERSION = "0.12.1"
+  VERSION = "0.13.1"
 end

data/lib/solid_objects/wake_up.rb

@@ -2,23 +2,55 @@
 
 module SolidObjects
   class WakeUp
+    class Watch
+      # @rbs @wake_up: WakeUp
+      # @rbs @generation: Integer
+
+      # @rbs (WakeUp, Integer) -> void
+      def initialize(wake_up, generation)
+        @wake_up = wake_up
+        @generation = generation
+      end
+
+      # @rbs (timeout: Numeric) -> bool
+      def wait(timeout:)
+        @wake_up.wait(timeout:, generation: @generation)
+      end
+    end
+
     # @rbs @mutex: Mutex
     # @rbs @condition: Thread::ConditionVariable
+    # @rbs @generation: Integer
 
     # @rbs () -> void
     def initialize
       @mutex = Mutex.new
       @condition = Thread::ConditionVariable.new
+      @generation = 0
     end
 
     # @rbs () -> void
     def signal
-      mutex.synchronize { condition.broadcast }
+      mutex.synchronize do
+        @generation += 1
+        condition.broadcast
+      end
     end
 
-    # @rbs (timeout: Numeric) -> void
-    def wait(timeout:)
-      mutex.synchronize { condition.wait(mutex, timeout) }
+    # @rbs () -> Watch
+    def watch
+      mutex.synchronize { Watch.new(self, @generation) }
+    end
+
+    # @rbs (timeout: Numeric, ?generation: Integer?) -> bool
+    def wait(timeout:, generation: nil)
+      mutex.synchronize do
+        return true if generation && @generation != generation
+
+        generation ||= @generation
+        condition.wait(mutex, timeout)
+        @generation != generation
+      end
     end
 
     private

data/lib/solid_objects/wake_up_adapters/postgresql.rb

@@ -45,6 +45,12 @@ module SolidObjects
         false
       end
 
+      # @rbs () -> self
+      def watch
+        listen
+        self
+      end
+
       # Starts listening before a caller blocks, so a notification sent between
       # startup and the first wait is not missed.
       # @rbs () -> bool

data/lib/solid_objects/wake_up_adapters/redis.rb

@@ -12,6 +12,22 @@ module SolidObjects
     # polling interval remains the upper bound, so a missed or failed
     # notification costs latency rather than correctness.
     class Redis
+      class Watch
+        # @rbs @adapter: Redis
+        # @rbs @generation: Integer
+
+        # @rbs (Redis, Integer) -> void
+        def initialize(adapter, generation)
+          @adapter = adapter
+          @generation = generation
+        end
+
+        # @rbs (timeout: Numeric) -> bool
+        def wait(timeout:)
+          @adapter.wait(timeout:, generation: @generation)
+        end
+      end
+
       CHANNEL = "solid_objects_wake_up"
       FAILED_WAIT_INTERVAL = 0.05
       SUBSCRIBE_TIMEOUT = 5.0
@@ -52,9 +68,9 @@ module SolidObjects
       # The counter is snapshotted before subscribing, and re-checked before
       # blocking, so a signal delivered while this caller was still getting
       # ready is observed rather than absorbed into the new baseline.
-      # @rbs (timeout: Numeric) -> bool
-      def wait(timeout:)
-        signalled = mutex.synchronize { @signalled }
+      # @rbs (timeout: Numeric, ?generation: Integer?) -> bool
+      def wait(timeout:, generation: nil)
+        signalled = generation || mutex.synchronize { @signalled }
         return paced_failure(timeout) unless listen
 
         mutex.synchronize do
@@ -65,6 +81,12 @@ module SolidObjects
         end
       end
 
+      # @rbs () -> Watch
+      def watch
+        listen
+        mutex.synchronize { Watch.new(self, @signalled) }
+      end
+
       # Redis delivers to a subscribed connection only, and a subscribed
       # connection cannot serve other callers, so one background subscription
       # per process fans out to every waiting role in memory. Subscribing

data/lib/solid_objects/web/action.rb

@@ -0,0 +1,109 @@
+# rbs_inline: enabled
+
+require "erb"
+require "rack/request"
+require "rack/utils"
+
+module SolidObjects
+  class Web
+    # One request. A route handler runs inside an instance of this class, so a
+    # handler and the template it renders share the same helpers and the same
+    # request.
+    class Action
+      include Helpers
+
+      # @rbs @env: Hash[String, untyped]
+      # @rbs @route: Route
+      # @rbs @captures: Hash[Symbol, String?]
+      # @rbs @request: untyped
+      # @rbs @layout_rendered: bool
+      # @rbs @response_status: Integer
+
+      attr_reader :env, :route, :response_status
+
+      # @rbs (env: Hash[String, untyped], route: Route) -> void
+      def initialize(env:, route:)
+        @env = env
+        @route = route
+        @captures = route.capture(env["PATH_INFO"].to_s)
+        @layout_rendered = false
+        @response_status = 200
+      end
+
+      # Sets the status a rendered page is served with. `halt` and `redirect`
+      # stop the handler, so a page that must render its own body and still
+      # report a failure needs this instead.
+      # @rbs (Integer) -> void
+      def status(code)
+        @response_status = code
+      end
+
+      # @rbs () -> untyped
+      def request
+        @request ||= ::Rack::Request.new(env)
+      end
+
+      # @rbs () -> Hash[untyped, untyped]?
+      def session
+        env["rack.session"]
+      end
+
+      # @rbs (Symbol) -> String?
+      def route_params(key)
+        @captures[key]
+      end
+
+      # @rbs (String) -> untyped
+      def url_params(key)
+        request.params[key]
+      end
+
+      # @rbs () -> untyped
+      def call
+        instance_exec(&route.handler)
+      end
+
+      # The layout is rendered once per request. The flag is raised before the
+      # page body runs so a partial the body renders returns its own fragment
+      # rather than a second whole page. The layout reads the page it wraps
+      # from `locals.fetch(:content)`.
+      # @rbs (Symbol, ?Hash[Symbol, untyped]) -> String
+      def erb(name, locals = {})
+        return evaluate(Web.template(name), locals) if @layout_rendered
+
+        @layout_rendered = true
+        content = evaluate(Web.template(name), locals)
+        evaluate(Web.template(:layout), { content: })
+      end
+
+      # @rbs (Integer, ?String) -> void
+      def halt(status, body = ::Rack::Utils::HTTP_STATUS_CODES.fetch(status, "Error"))
+        throw :halt, [ status, { "content-type" => "text/plain" }, [ body ] ]
+      end
+
+      # @rbs (String) -> void
+      def redirect(path)
+        throw :halt, [ 302, { "location" => path_to(path) }, [] ]
+      end
+
+      # @rbs (untyped) -> void
+      def json(payload)
+        throw :halt, [
+          200,
+          { "content-type" => "application/json", "cache-control" => "private, no-store" },
+          [ JSON.generate(payload) ]
+        ]
+      end
+
+      private
+
+      # `locals` is a local variable of this method, so a template reads it by
+      # name, and every helper is reachable because the template runs against
+      # this object.
+      # @rbs (untyped, Hash[Symbol, untyped]) -> String
+      def evaluate(template, locals)
+        template.result(binding)
+      end
+    end
+  end
+end

data/lib/solid_objects/web/application.rb

@@ -0,0 +1,239 @@
+# rbs_inline: enabled
+
+module SolidObjects
+  class Web
+    # The pages. Each route states the administration policy it needs, and
+    # `call` asks that policy before the handler runs, so a page cannot read
+    # the actor tables on behalf of an unauthorized request.
+    class Application
+      extend Router
+
+      SCRIPT_PLACEHOLDER = "!script-src!"
+      CONTENT_SECURITY_POLICY = [
+        "default-src 'self'",
+        "base-uri 'self'",
+        "form-action 'self'",
+        "frame-ancestors 'none'",
+        "img-src 'self' data:",
+        "style-src 'self'",
+        "script-src #{SCRIPT_PLACEHOLDER}",
+        "connect-src 'self'",
+        "object-src 'none'"
+      ].join("; ").freeze
+
+      MAILBOX_MEMBERSHIPS = %w[ready claimed].freeze
+      RECENT_LIMIT = 10
+      CHART_TYPE_LIMIT = 12
+
+      head "/", policy: { action: "index", resource: "dashboard" } do
+        # The cheapest liveness check available: it proves the dashboard can
+        # reach the database the actors run on, and returns no body.
+        ReadyMessage.count
+        ""
+      end
+
+      get "/", policy: { action: "index", resource: "dashboard" } do
+        @statistics = statistics.to_h
+        @processes = Process.order(last_heartbeat_at: :desc).limit(RECENT_LIMIT)
+        @dead_letters = DeadLetter.order(last_failed_at: :desc, id: :desc).limit(RECENT_LIMIT)
+        # The only chart that costs its own query, and the only one the poller
+        # cannot refresh, because the other two read what `/stats` already
+        # returns. Bounded so a runtime with many actor types draws a readable
+        # chart rather than every type it has ever seen.
+        @instances_by_type = Instance
+          .group(:actor_type)
+          .order(Arel.sql("COUNT(*) DESC"))
+          .limit(CHART_TYPE_LIMIT)
+          .count
+        erb(:dashboard)
+      end
+
+      get "/stats", policy: { action: "index", resource: "dashboard" } do
+        json(statistics.to_h)
+      end
+
+      get "/instances", policy: { action: "index", resource: "instances" } do
+        @paginator = paginate(filtered_instances)
+        # Suggestions come from the registry rather than a DISTINCT over the
+        # instances table, which no adapter can answer from an index. The field
+        # stays free text, so an actor type that is no longer registered is
+        # still reachable.
+        @actor_types = SolidObjects.registry.to_h.keys.sort
+        erb(:instances)
+      end
+
+      get "/instances/:id", policy: { action: "show", resource: "instances" } do
+        @instance = find_instance
+        @ready_messages = @instance.messages
+          .where(id: ReadyMessage.select(:message_id))
+          .order(sequence: :asc)
+          .limit(RECENT_LIMIT)
+        @claimed_messages = @instance.messages
+          .where(id: ClaimedMessage.select(:message_id))
+          .order(sequence: :asc)
+          .limit(RECENT_LIMIT)
+        @recent_messages = @instance.messages.order(sequence: :desc).limit(RECENT_LIMIT)
+        @reminders = Reminder.where(instance_id: @instance.id).order(next_run_at: :asc).limit(RECENT_LIMIT)
+        @effects = Effect.where(instance_id: @instance.id).order(id: :desc).limit(RECENT_LIMIT)
+        @broadcasts = Broadcast.where(instance_id: @instance.id).order(id: :desc).limit(RECENT_LIMIT)
+        @dead_letters = DeadLetter.where(instance_id: @instance.id).order(last_failed_at: :desc).limit(RECENT_LIMIT)
+        erb(:instance)
+      end
+
+      # Pausing stops the activation manager from claiming the instance again.
+      # A pass already in flight finishes its turn, and a synchronous caller
+      # waiting on this instance times out rather than being answered, so this
+      # is an operator brake and not a delivery guarantee.
+      post "/instances/:id/pause", policy: { action: "pause", resource: "instances" } do
+        instance = find_instance
+        instance.update!(paused_at: SolidObjects.database_adapter.database_now)
+        redirect("/instances/#{instance.id}")
+      end
+
+      post "/instances/:id/resume", policy: { action: "resume", resource: "instances" } do
+        instance = find_instance
+        instance.update!(paused_at: nil)
+        redirect("/instances/#{instance.id}")
+      end
+
+      get "/mailbox", policy: { action: "index", resource: "messages" } do
+        @membership = filter_value(MAILBOX_MEMBERSHIPS, default: "ready")
+        @paginator = paginate(mailbox_messages(@membership))
+        erb(:mailbox)
+      end
+
+      get "/messages/:id", policy: { action: "show", resource: "messages" } do
+        @message = Message.find_by(id: route_params(:id))
+        halt(404) unless @message
+        erb(:message)
+      end
+
+      get "/reminders", policy: { action: "index", resource: "reminders" } do
+        @status = filter_value(Statistics::REMINDER_STATUSES)
+        relation = Reminder.order(next_run_at: :asc, id: :asc)
+        @paginator = paginate(@status ? relation.where(status: @status) : relation)
+        erb(:reminders)
+      end
+
+      get "/effects", policy: { action: "index", resource: "effects" } do
+        @status = filter_value(Statistics::EFFECT_STATUSES)
+        relation = Effect.order(id: :desc)
+        @paginator = paginate(@status ? relation.where(status: @status) : relation)
+        erb(:effects)
+      end
+
+      get "/broadcasts", policy: { action: "index", resource: "broadcasts" } do
+        @status = filter_value(Statistics::BROADCAST_STATUSES)
+        relation = Broadcast.order(id: :desc)
+        @paginator = paginate(@status ? relation.where(status: @status) : relation)
+        erb(:broadcasts)
+      end
+
+      get "/dead_letters", policy: { action: "index", resource: "dead_letters" } do
+        @paginator = paginate(DeadLetter.order(last_failed_at: :desc, id: :desc))
+        erb(:dead_letters)
+      end
+
+      get "/dead_letters/:id", policy: { action: "show", resource: "dead_letters" } do
+        @dead_letter = DeadLetter.find_by(id: route_params(:id))
+        halt(404) unless @dead_letter
+        erb(:dead_letter)
+      end
+
+      # A retry re-enters the mailbox, which refuses work the runtime cannot
+      # accept: an actor class that no longer exists, a full mailbox, a payload
+      # over the cap. The operator who pressed the button is told which,
+      # instead of being handed a bare 500 from the Rack handler.
+      post "/dead_letters/:id/retry", policy: { action: "retry", resource: "dead_letters" } do
+        SolidObjects.dead_letters.retry(route_params(:id).to_i, authorization_context: self)
+        redirect("/dead_letters")
+      rescue Unauthorized
+        raise
+      rescue SolidObjects::Error => error
+        @dead_letter = DeadLetter.find_by(id: route_params(:id))
+        halt(404) unless @dead_letter
+        @error = "#{error.class.name.split("::").last}: #{error.message}"
+        status(422)
+        erb(:dead_letter)
+      end
+
+      get "/processes", policy: { action: "index", resource: "processes" } do
+        @status = filter_value(Statistics::PROCESS_STATES)
+        relation = Process.order(last_heartbeat_at: :desc)
+        @paginator = paginate(@status ? relation.where(shutdown_state: @status) : relation)
+        # Counted in one grouped query rather than once per row, because this
+        # page is read while the runtime is already under load.
+        @activated_counts = Instance
+          .where(activation_owner_id: @paginator.records.map(&:id))
+          .group(:activation_owner_id)
+          .count
+        erb(:processes)
+      end
+
+      # @rbs (Hash[String, untyped]) -> Array[untyped]
+      def call(env)
+        route = self.class.match(env["REQUEST_METHOD"].to_s, env["PATH_INFO"].to_s)
+        return not_found unless route
+
+        action = Action.new(env:, route:)
+        return forbidden unless authorized?(action)
+
+        respond(action, catch(:halt) { action.call })
+      rescue Unauthorized
+        forbidden
+      end
+
+      private
+
+      # @rbs (Action, untyped) -> Array[untyped]
+      def respond(action, result)
+        return result if result.is_a?(Array)
+
+        [ action.response_status, page_headers(action.env), [ result.to_s ] ]
+      end
+
+      # @rbs (Action) -> bool
+      def authorized?(action)
+        policy = action.route.policy
+        SolidObjects.configuration.authorize_administration.call(
+          action: policy.fetch(:action),
+          resource: policy.fetch(:resource),
+          resource_id: action.route_params(:id),
+          authorization_context: action
+        )
+      end
+
+      # @rbs (Hash[String, untyped]) -> Hash[String, String]
+      def page_headers(env)
+        {
+          "content-type" => "text/html; charset=utf-8",
+          "cache-control" => "private, no-store",
+          "content-security-policy" => content_security_policy(env),
+          "x-content-type-options" => "nosniff",
+          "referrer-policy" => "same-origin"
+        }
+      end
+
+      # The chart host is named only when one is configured, so a deployment
+      # that vendors the library or turns charts off never advertises a third
+      # party origin it does not use.
+      # @rbs (Hash[String, untyped]) -> String
+      def content_security_policy(env)
+        sources = [ "'self'", "'nonce-#{env[Web::NONCE_KEY]}'", Web.chart_library_origin ].compact
+        CONTENT_SECURITY_POLICY.sub(SCRIPT_PLACEHOLDER, sources.join(" "))
+      end
+
+      # The cascade header lets a host application serve its own 404 for a path
+      # below the mount that the dashboard does not define.
+      # @rbs () -> Array[untyped]
+      def not_found
+        [ 404, { "content-type" => "text/plain", "x-cascade" => "pass" }, [ "Not Found" ] ]
+      end
+
+      # @rbs () -> Array[untyped]
+      def forbidden
+        [ 403, { "content-type" => "text/plain" }, [ "Forbidden" ] ]
+      end
+    end
+  end
+end