pg_eventstore 3.0.1 → 3.1.2

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module PgEventstore
-  # This class pulls events from db and feeds given SubscriptionRunners
+  # This class decides which SubscriptionRunners to feed, pulls events from db and feeds them
   # @!visibility private
   class SubscriptionRunnersFeeder
     # @param config_name [Symbol]
@@ -12,11 +12,15 @@ module PgEventstore
     # @param runners [Array<PgEventstore::SubscriptionRunner>]
     # @return [void]
     def feed(runners)
-      runners = runners.select(&:running?).select(&:time_to_feed?)
-      return if runners.empty?
+      runners_query_options = runners.select { |runner| runner.running? && time_to_feed?(runner) }.to_h do |runner|
+        [runner, runner.next_chunk_query_opts]
+      end
+      # A runner which has enough events in its queue estimates the number of events to fetch to 0
+      runners_query_options = runners_query_options.select { |_, query_options| query_options[:max_count] > 0 }
+      return if runners_query_options.empty?
 
       feed_strategies_collection = SubscriptionFeedStrategy::Collection.create(
-        runners,
+        runners_query_options,
         connection,
         QueryStrategy::Async.new(connection)
       )
@@ -29,6 +33,13 @@ module PgEventstore
 
     private
 
+    # @param runner [PgEventstore::SubscriptionRunner]
+    # @return [Boolean]
+    def time_to_feed?(runner)
+      subscription = runner.subscription
+      subscription.last_chunk_fed_at + subscription.chunk_query_interval <= Time.now.utc
+    end
+
     # @return [PgEventstore::Connection]
     def connection
       PgEventstore.connection(@config_name)

data/lib/pg_eventstore/subscriptions/subscriptions_manager.rb

@@ -245,6 +245,11 @@ module PgEventstore
           restart_terminator:,
           failed_subscription_notifier:
         ),
+        # Catch-all recovery strategy that only reports an error. It is only invoked in case previous strategies
+        # rejected to recover an error. Must be stayed last.
+        RunnerRecoveryStrategies::ReportSubscriptionUnrecoverableError.new(
+          subscription:, failed_subscription_notifier:
+        ),
       ]
     end
 

data/lib/pg_eventstore/utils.rb

@@ -110,6 +110,8 @@ module PgEventstore
       end
       # rubocop:enable Lint/SuppressedException
 
+      # Wraps subscription handler exception to correctly differentiate it from other errors that may happen inside
+      # the implementation. This is needed to correctly handle subscription handler errors vs internal errors.
       # @param exception [StandardError]
       # @param extra [Hash] additional exception info
       # @return [PgEventstore::WrappedException]

data/lib/pg_eventstore/version.rb

@@ -2,5 +2,5 @@
 
 module PgEventstore
   # @return [String]
-  VERSION = '3.0.1'
+  VERSION = '3.1.2'
 end

data/lib/pg_eventstore/web/application.rb

@@ -176,6 +176,13 @@ module PgEventstore
           streams || []
         end
 
+        # Only maintainable roles can delete a stream or an event. This helper only supposes to hide delete buttons -
+        # backend has its own checks.
+        # @return [Boolean]
+        def can_delete?
+          PgEventstore::Config::NodeRole.maintainable.include?(PgEventstore.config(current_config).eventstore_role)
+        end
+
         private
 
         # @param hash [Hash]

data/lib/pg_eventstore/web/metrics/application.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Metrics
+      # Standalone rack application serving subscription metrics in the Prometheus text exposition format.
+      #
+      # It ships without any authentication - how the endpoint is protected is up to the application mounting it,
+      # the same way it is for the Admin UI (see docs/admin_ui.md#authorization).
+      #
+      # Which pg_eventstore database is queried is chosen per request with the "config" query param, so one mounted
+      # app can serve metrics of every configured store:
+      #
+      #   GET /subscriptions/latency?config=db1
+      #
+      # An absent config means the default one; an unknown config is answered with 404 rather than silently served from
+      # the default store, so a misconfigured scrape shows up as a failing target instead of as wrong data. Add "set"
+      # params to report only some subscription sets - repeat the param for several: "?set=SetA&set=SetB".
+      class Application < Sinatra::Base
+        set :environment, -> { (ENV['RACK_ENV'] || ENV['RAILS_ENV'] || ENV['APP_ENV'])&.to_sym || :development }
+        set :logging, false
+        set :sessions, false
+        set :host_authorization, { allow_if: ->(_env) { true } }
+
+        helpers(Helpers) do
+          # @return [PgEventstore::Connection]
+          def metrics_connection
+            PgEventstore.connection(config_name)
+          end
+
+          # Name of the config to query: the "config" param when given, the default config otherwise. Halts with 404
+          # for a name that is not configured.
+          # @return [Symbol]
+          def config_name
+            requested = params[:config].to_s
+            return PgEventstore::DEFAULT_CONFIG if requested.empty?
+            return requested.to_sym if PgEventstore.available_configs.include?(requested.to_sym)
+
+            halt 404, { 'content-type' => 'text/plain' }, "Unknown config #{requested.inspect}"
+          end
+        end
+
+        get('/subscriptions') do
+          metrics_response(
+            [Collectors::SubscriptionsLatency, Collectors::SubscriptionsHealth, Collectors::SubscriptionsThroughput]
+          )
+        end
+
+        # The only route querying event positions - one index range scan per subscription.
+        get('/subscriptions/latency') do
+          metrics_response([Collectors::SubscriptionsLatency])
+        end
+
+        get('/subscriptions/health') do
+          metrics_response([Collectors::SubscriptionsHealth])
+        end
+
+        get('/subscriptions/throughput') do
+          metrics_response([Collectors::SubscriptionsThroughput])
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/metrics/collectors/base.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Metrics
+      module Collectors
+        # Base class for metric collectors. A collector runs read-only queries and returns an array of
+        # {MetricFamily} objects.
+        class Base
+          # Milliseconds. Guards the store from a metrics query that got stuck - a scrape then fails visibly instead
+          # of piling up on the database.
+          # @return [Integer]
+          STATEMENT_TIMEOUT = 5_000
+
+          # @!attribute connection
+          #   @return [PgEventstore::Connection]
+          attr_reader :connection
+          # @!attribute sets
+          #   @return [Array<String>]
+          attr_reader :sets
+          private :connection, :sets
+
+          # @param connection [PgEventstore::Connection]
+          # @param sets [Array<String>] subscription sets to report on; an empty array means all of them
+          def initialize(connection, sets: [])
+            @connection = connection
+            @sets = sets
+          end
+
+          # @return [Array<PgEventstore::Web::Metrics::MetricFamily>]
+          def call
+            raise NotImplementedError, "#{self.class} must implement #call"
+          end
+
+          private
+
+          # @return [PgEventstore::SQLBuilder]
+          def subscriptions_sql_builder
+            sql_builder = SQLBuilder.new.from('subscriptions', table_alias: 's')
+            sql_builder.order('s.set, s.name')
+            sql_builder.where('s.set = any(?::varchar[])', sets) if sets.any?
+            sql_builder
+          end
+
+          # @return [Array<Hash>]
+          def with_safe_conn
+            transaction_queries.transaction(:read_committed, read_only: true) do
+              connection.with do |conn|
+                conn.exec("set local statement_timeout to #{STATEMENT_TIMEOUT}")
+                yield conn
+              end
+            end.to_a
+          end
+
+          # @param row [Hash]
+          # @return [Hash<Symbol => String>]
+          def subscription_labels(row)
+            { set: row['set'], name: row['name'] }
+          end
+
+          # @return [PgEventstore::TransactionQueries]
+          def transaction_queries
+            TransactionQueries.new(connection)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/metrics/collectors/subscriptions_health.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Metrics
+      module Collectors
+        # Liveness and error state of each reported subscription.
+        #
+        # The state column alone can not be trusted: a subscription killed without a graceful shutdown keeps
+        # state "running" and its lock forever. heartbeat_age_seconds is the discriminator - a subscription is
+        # really running only while its heartbeat stays below
+        # {PgEventstore::SubscriptionsLifecycle::HEARTBEAT_INTERVAL}.
+        class SubscriptionsHealth < Base
+          # @return [Array<PgEventstore::Web::Metrics::MetricFamily>]
+          def call
+            state = MetricFamily.new(
+              name: 'pg_eventstore_subscription_state',
+              type: 'gauge',
+              help: 'Last recorded state of the subscription. May be stale - correlate with ' \
+                    'pg_eventstore_subscription_heartbeat_age_seconds.'
+            )
+            locked = MetricFamily.new(
+              name: 'pg_eventstore_subscription_locked',
+              type: 'gauge',
+              help: 'Whether the subscription is locked by a subscriptions set.'
+            )
+            heartbeat_age = MetricFamily.new(
+              name: 'pg_eventstore_subscription_heartbeat_age_seconds',
+              type: 'gauge',
+              help: 'Seconds since the subscription row was last touched by its runner. A locked subscription ' \
+                    'with a stale heartbeat is a dead process that did not shut down gracefully.'
+            )
+            restarts = MetricFamily.new(
+              name: 'pg_eventstore_subscription_restarts_total',
+              type: 'counter',
+              help: 'Number of times the subscription was restarted after a failure.'
+            )
+            last_error_age = MetricFamily.new(
+              name: 'pg_eventstore_subscription_last_error_age_seconds',
+              type: 'gauge',
+              help: 'Seconds since the last error occurred. Absent when the subscription never failed.'
+            )
+            subscription_rows.each do |row|
+              labels = subscription_labels(row)
+              state.add_sample(labels: labels.merge(state: row['state']), value: 1)
+              locked.add_sample(labels:, value: row['locked'])
+              heartbeat_age.add_sample(labels:, value: row['heartbeat_age_seconds'])
+              restarts.add_sample(labels:, value: row['restart_count'])
+              last_error_age.add_sample(labels:, value: row['last_error_age_seconds']) if row['last_error_age_seconds']
+            end
+            [state, locked, heartbeat_age, restarts, last_error_age]
+          end
+
+          private
+
+          # @return [Array<Hash>]
+          def subscription_rows
+            builder = subscriptions_sql_builder
+            builder.select(<<~SQL)
+              s.set,
+              s.name,
+              s.state,
+              (s.locked_by is not null)::int as locked,
+              extract(epoch from ((now() at time zone 'utc') - s.updated_at))::float8 as heartbeat_age_seconds,
+              s.restart_count,
+              case when s.last_error_occurred_at is not null
+                   then extract(epoch from ((now() at time zone 'utc') - s.last_error_occurred_at))::float8
+              end as last_error_age_seconds
+            SQL
+            with_safe_conn do |conn|
+              conn.exec_params(*builder.to_exec_params)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/metrics/collectors/subscriptions_latency.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Metrics
+      module Collectors
+        # How far behind each subscription is.
+        #
+        # A subscription checkpoint is measured in the same units as the subscription positions frontier, not in
+        # events.global_position units - the global position sequence contains gaps and runs ahead of the frontier,
+        # so comparing a checkpoint against it over-reports by orders of magnitude. Lag is therefore always measured
+        # against the frontier.
+        #
+        # - lag_events: how much of the store the subscription still has to walk through before it reaches the
+        #   frontier - that is, before it starts processing newly appended events.
+        # - lag_seconds: age of the oldest event the subscription has not processed yet; 0 when caught up.
+        #
+        # The frontier only advances while at least one subscriptions process is running. When they are all down lag
+        # stops growing - that situation is reported by the heartbeat metric of the health collector, not here.
+        class SubscriptionsLatency < Base
+          # @return [Array<PgEventstore::Web::Metrics::MetricFamily>]
+          def call
+            lag_events = MetricFamily.new(
+              name: 'pg_eventstore_subscription_lag_events',
+              type: 'gauge',
+              help: 'How many events the subscription still has to catch up on before it reaches the edge of the ' \
+                    '"all" stream.'
+            )
+            lag_seconds = MetricFamily.new(
+              name: 'pg_eventstore_subscription_lag_seconds',
+              type: 'gauge',
+              help: 'Age in seconds of the oldest event the subscription has not processed yet. 0 when caught up.'
+            )
+            frontier_position, head_global_position = *positions
+            rows = subscription_rows(frontier_position)
+            created_at_by_position = resolve_created_at(rows)
+            now = Time.now.utc
+            rows.each { add_samples(_1, created_at_by_position, now, lag_events, lag_seconds) }
+            [lag_events, lag_seconds, *store_families(frontier_position, head_global_position)]
+          end
+
+          private
+
+          # @param row [Hash]
+          # @param created_at_by_position [Hash<Integer => Time>]
+          # @param now [Time]
+          # @param lag_events [PgEventstore::Web::Metrics::MetricFamily]
+          # @param lag_seconds [PgEventstore::Web::Metrics::MetricFamily]
+          # @return [void]
+          def add_samples(row, created_at_by_position, now, lag_events, lag_seconds)
+            labels = subscription_labels(row)
+            lag_events.add_sample(labels:, value: row['lag_events'])
+            position = row['event_global_position']
+            # Caught up - nothing is left to process, so there is no unprocessed event to age. Deliberately a float:
+            # the metric is seconds everywhere else, and a gauge that renders as "0" here and "12.5" there is a wart.
+            return lag_seconds.add_sample(labels:, value: 0.0) if position.nil?
+
+            created_at = created_at_by_position[position]
+            # An unprocessed position whose event no longer exists (the event or its stream was deleted). Reporting 0
+            # would read as "caught up", the opposite of the truth, so report nothing - lag_events still carries the
+            # backlog.
+            return if created_at.nil?
+
+            lag_seconds.add_sample(labels:, value: [(now - created_at).to_f, 0].max)
+          end
+
+          # One index range scan per subscription over idx_event_subscription_positions_sposition_n_gposition. The
+          # cost does not grow with the size of the backlog: the oldest unprocessed position is taken with
+          # "order by subscription_position limit 1" rather than by aggregating over the whole backlog.
+          # @param frontier_position [Integer]
+          # @return [Array<Hash>]
+          def subscription_rows(frontier_position)
+            builder = subscriptions_sql_builder
+            builder.select(<<~SQL)
+              s.set,
+              s.name,
+              greatest(#{frontier_position} - coalesce(s.current_position, 0), 0) as lag_events,
+              next_event.global_position as event_global_position,
+              next_event.event_type_partition_id as event_type_partition_id
+            SQL
+            builder.join(<<~SQL)
+              left join lateral (
+                  select egi.global_position, egi.event_type_partition_id
+                  from event_subscription_positions esp
+                  join events_global_index egi on egi.global_position = esp.global_position
+                  where esp.subscription_position > coalesce(s.current_position, 0)
+                  order by esp.subscription_position
+                  limit 1
+                ) next_event on true
+            SQL
+            with_safe_conn do |conn|
+              conn.exec_params(*builder.to_exec_params)
+            end
+          end
+
+          # Resolves the creation time of every subscription's oldest unprocessed event.
+          #
+          # The events table is partitioned, and these events are known only by global position - which is not the
+          # partition key. Looking them up in events by global position alone would have to visit every partition and
+          # lock all of them, which stops being viable long before a store reaches five figures of partitions.
+          # events_global_index records the partition of each event, so the read API can resolve them partition-wise
+          # instead.
+          # @param subscription_rows [Array<Hash>]
+          # @return [Hash<Integer => Time>]
+          def resolve_created_at(subscription_rows)
+            indexes = subscription_rows.filter_map do |attrs|
+              next if attrs['event_global_position'].nil?
+
+              EventGlobalIndex::ReadApiRepr.new(
+                global_position: attrs['event_global_position'],
+                event_type_partition_id: attrs['event_type_partition_id']
+              )
+            end
+            return {} if indexes.empty?
+
+            resolved = events_global_index_queries.resolve_indexes(indexes, resolve_link_tos: false)
+            resolved.to_h { [_1['global_position'], _1['created_at']] }
+          end
+
+          # @return [PgEventstore::EventsGlobalIndexQueries]
+          def events_global_index_queries
+            EventsGlobalIndexQueries.new(connection, QueryStrategy::Foreground.new(connection))
+          end
+
+          # @param frontier_position [Integer]
+          # @param head_global_position [Integer]
+          # @return [Array<PgEventstore::Web::Metrics::MetricFamily>]
+          def store_families(frontier_position, head_global_position)
+            frontier = MetricFamily.new(
+              name: 'pg_eventstore_store_frontier_position',
+              type: 'gauge',
+              help: 'Latest assigned subscription position. Subscription checkpoints are measured against this.'
+            )
+            frontier.add_sample(labels: {}, value: frontier_position)
+            head = MetricFamily.new(
+              name: 'pg_eventstore_store_head_global_position',
+              type: 'gauge',
+              help: 'Global position of the newest event in the store. Contains gaps; do not compare subscription ' \
+                    'checkpoints against it.'
+            )
+            head.add_sample(labels: {}, value: head_global_position)
+            [frontier, head]
+          end
+
+          # @return [Array[Integer]]
+          def positions
+            res = with_safe_conn do |conn|
+              conn.exec(<<~SQL)
+                select
+                  (select coalesce(max(subscription_position), 0) from event_subscription_positions)
+                    as frontier_position,
+                  (select coalesce(max(global_position), 0) from events_global_index) as head_global_position
+              SQL
+            end
+            res.first.values_at('frontier_position', 'head_global_position')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/metrics/collectors/subscriptions_throughput.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Metrics
+      module Collectors
+        # Processing volume and speed of each reported subscription.
+        #
+        # Two deliberately different numbers:
+        # - processed_events_total is a counter; rate() over it gives the actual current throughput and correctly
+        #   drops to 0 when no matching events arrive.
+        # - capacity_events_per_second derives from the average handler execution time of the last
+        #   {PgEventstore::SubscriptionHandlerPerformance::TIMINGS_TO_KEEP} processed events - whenever they
+        #   happened. It answers "how fast can this handler go when fed", is sticky while the subscription is idle
+        #   and must not be read as current throughput.
+        class SubscriptionsThroughput < Base
+          # @return [Array<PgEventstore::Web::Metrics::MetricFamily>]
+          def call
+            processed = MetricFamily.new(
+              name: 'pg_eventstore_subscription_processed_events_total',
+              type: 'counter',
+              help: 'Total number of events processed by the subscription. Use rate() for current throughput.'
+            )
+            capacity = MetricFamily.new(
+              name: 'pg_eventstore_subscription_capacity_events_per_second',
+              type: 'gauge',
+              help: 'Average processing speed over the last few processed events, whenever they happened. ' \
+                    'Sticky while idle - this is handler capacity, not current throughput.'
+            )
+            subscription_rows.each do |row|
+              labels = subscription_labels(row)
+              processed.add_sample(labels:, value: row['total_processed_events'])
+              capacity.add_sample(labels:, value: row['capacity_eps']) if row['capacity_eps']
+            end
+            [processed, capacity]
+          end
+
+          private
+
+          # @return [Array<Hash>]
+          def subscription_rows
+            builder = subscriptions_sql_builder
+            builder.select(<<~SQL)
+              s.set,
+              s.name,
+              s.total_processed_events,
+              case when s.average_event_processing_time > 0
+                   then (1.0 / s.average_event_processing_time)::float8
+              end as capacity_eps
+            SQL
+            with_safe_conn do |conn|
+              conn.exec_params(*builder.to_exec_params)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/metrics/formatter.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Metrics
+      # Renders metric families into the Prometheus text exposition format (version 0.0.4).
+      class Formatter
+        # @return [String]
+        CONTENT_TYPE = 'text/plain; version=0.0.4; charset=utf-8'
+
+        # @param families [Array<PgEventstore::Web::Metrics::MetricFamily>]
+        # @return [String]
+        def call(families)
+          "#{families.map { |family| format_family(family) }.join("\n")}\n"
+        end
+
+        private
+
+        # @param family [PgEventstore::Web::Metrics::MetricFamily]
+        # @return [String]
+        def format_family(family)
+          lines = ["# HELP #{family.name} #{family.help}", "# TYPE #{family.name} #{family.type}"]
+          family.samples.each do |sample|
+            lines.push("#{family.name}#{format_labels(sample[:labels])} #{sample[:value]}")
+          end
+          lines.join("\n")
+        end
+
+        # @param labels [Hash<Symbol => String>]
+        # @return [String]
+        def format_labels(labels)
+          return '' if labels.empty?
+
+          "{#{labels.map { |name, value| %(#{name}="#{escape_label_value(value.to_s)}") }.join(',')}}"
+        end
+
+        # @param value [String]
+        # @return [String]
+        def escape_label_value(value)
+          value.gsub('\\', '\\\\\\\\').gsub("\n", '\\n').gsub('"', '\\"')
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/metrics/helpers.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Metrics
+      # Route helpers of {Metrics::Application}.
+      module Helpers
+        # @param collector_classes [Array<Class<PgEventstore::Web::Metrics::Collectors::Base>>]
+        # @return [String]
+        def metrics_response(collector_classes)
+          connection = metrics_connection
+          families = collector_classes.flat_map { _1.new(connection, sets: requested_sets).call }
+          content_type(Formatter::CONTENT_TYPE)
+          Formatter.new.call(families)
+        end
+
+        # Subscription sets a scrape asks for. Accepts a repeated param ("?set=A&set=B" - what Prometheus emits for
+        # `params: {set: [A, B]}`) or a comma separated list ("?set=A,B"). Empty means every set.
+        #
+        # The query string is parsed directly because Sinatra's `params` keeps only the last value of a repeated key
+        # unless it is written as "set[]", which Prometheus does not do.
+        # @return [Array<String>]
+        def requested_sets
+          query = Rack::Utils.parse_query(request.query_string)
+          Array(query['set'] || query['set[]']).flat_map { _1.to_s.split(',') }.map(&:strip).reject(&:empty?)
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/metrics/metric_family.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Metrics
+      # A collection of samples of a single Prometheus metric.
+      class MetricFamily
+        # @!attribute name
+        #   @return [String]
+        attr_reader :name
+        # @!attribute type
+        #   @return [String] "gauge" or "counter"
+        attr_reader :type
+        # @!attribute help
+        #   @return [String]
+        attr_reader :help
+        # @!attribute samples
+        #   @return [Array<Hash>]
+        attr_reader :samples
+
+        # @param name [String]
+        # @param type [String]
+        # @param help [String]
+        def initialize(name:, type:, help:)
+          @name = name
+          @type = type
+          @help = help
+          @samples = []
+        end
+
+        # @param labels [Hash<Symbol => String>]
+        # @param value [Integer, Float]
+        # @return [void]
+        def add_sample(labels:, value:)
+          @samples.push({ labels:, value: })
+        end
+      end
+    end
+  end
+end