pg_eventstore 1.13.4 → 3.0.0

data/lib/pg_eventstore/web/paginator/events_collection.rb

@@ -19,57 +19,46 @@ module PgEventstore
         #   count instead because of the potential performance degradation.
         MAX_NUMBER_TO_COUNT = 10_000
 
-        # @param config_name [Symbol]
-        # @param starting_id [String, Integer, nil]
-        # @param per_page [Integer]
-        # @param order [Symbol] :asc or :desc
-        # @param options [Hash] additional options to filter the collection
-        # @param system_stream [String, nil] a name of system stream
-        def initialize(config_name, starting_id:, per_page:, order:, options: {}, system_stream: nil)
-          super(config_name, starting_id: starting_id, per_page: per_page, order: order, options: options)
-          @stream = system_stream ? PgEventstore::Stream.system_stream(system_stream) : PgEventstore::Stream.all_stream
-        end
-
         # @return [Array<PgEventstore::Event>]
         def collection
-          @collection ||= PgEventstore.client(config_name).read(
-            @stream,
-            options: options.merge(from_position: starting_id, max_count: per_page, direction: order)
-          )
+          _collection[0...per_page]
         end
 
         # @return [Integer, nil]
         def next_page_starting_id
           return unless collection.size == per_page
 
-          from_position = event_global_position(collection.first)
-          sql_builder = QueryBuilders::EventsFiltering.events_filtering(
-            @stream,
-            options.merge(from_position: from_position, max_count: 1, direction: order)
-          ).to_sql_builder.unselect.select('global_position').offset(per_page)
-          global_position(sql_builder)
+          event_global_position(_collection[per_page])
         end
 
         # @return [Integer, nil]
         def prev_page_starting_id
-          from_position = event_global_position(collection.first) || starting_id
-          sql_builder = QueryBuilders::EventsFiltering.events_filtering(
-            @stream,
-            options.merge(from_position: from_position, max_count: per_page, direction: order == :asc ? :desc : :asc)
-          ).to_sql_builder.unselect.select('global_position').offset(1)
-          sql, params = sql_builder.to_exec_params
-          sql = "SELECT * FROM (#{sql}) #{Event::PRIMARY_TABLE_NAME} ORDER BY global_position #{order} LIMIT 1"
-          connection.with do |conn|
-            conn.exec_params(sql, params)
-          end.to_a.dig(0, 'global_position')
+          starting_position = event_global_position(_collection.first) || starting_id
+          return unless starting_position
+
+          order = self.order == :asc ? :desc : :asc
+          starting_position = self.order == :asc ? starting_position - 1 : starting_position + 1
+
+          # Not ideal from the perspective of the amount of records we load here, but this is the only way to read from
+          # the db in the correct way. Direct usage of query builders may result in heavy, unoptimized queries.
+          options = self.options.merge(
+            from_position: starting_position, max_count: per_page, direction: order, resolve_link_tos: false
+          )
+          PgEventstore.client(config_name).read(
+            PgEventstore::Stream.all_stream,
+            options:,
+            middlewares: []
+          ).last&.global_position
         end
 
         # @return [Integer]
         def total_count
           @total_count ||=
             begin
-              sql_builder = QueryBuilders::EventsFiltering.events_filtering(@stream, options).to_sql_builder
-              sql_builder.remove_limit.remove_group.remove_order
+              filters_collection = QueryBuilders::Filters::Collection.from_options(options)
+              sql_builder = QueryBuilders::IndexBasedEventsFiltering.sql_builder_for_estimate_count(
+                filters_collection.collection
+              )
               count = estimate_count(sql_builder)
               return count if count > MAX_NUMBER_TO_COUNT
 
@@ -79,6 +68,13 @@ module PgEventstore
 
         private
 
+        def _collection
+          @_collection ||= PgEventstore.client(config_name).read(
+            PgEventstore::Stream.all_stream,
+            options: options.merge(from_position: starting_id, max_count: per_page + 1, direction: order)
+          )
+        end
+
         # @param event [PgEventstore::Event, nil]
         # @return [Integer, nil]
         def event_global_position(event)

data/lib/pg_eventstore/web/paginator/helpers.rb

@@ -41,13 +41,13 @@ module PgEventstore
         # @param per_page [String] string representation of items per page. E.g. "10", "20", etc.
         # @return [String]
         def per_page_url(per_page)
-          build_path(params.merge(per_page: per_page))
+          build_path(params.merge(per_page:))
         end
 
         # @param order [String] "asc"/"desc"
         # @return [String]
         def sort_url(order)
-          build_path(params.merge(order: order))
+          build_path(params.merge(order:))
         end
 
         def resolve_link_tos_url(should_resolve)
@@ -80,24 +80,31 @@ module PgEventstore
           parts.join(delimiter)
         end
 
-        # @param event [PgEventstore::Event]
+        # @param stream [PgEventstore::Stream]
         # @return [String]
-        def stream_path(event)
+        def stream_path(stream)
           build_path(
             {
               filter: {
                 streams: [
                   {
-                    context: escape_empty_string(event.stream.context),
-                    stream_name: escape_empty_string(event.stream.stream_name),
-                    stream_id: escape_empty_string(event.stream.stream_id),
+                    context: escape_empty_string(stream.context),
+                    stream_name: escape_empty_string(stream.stream_name),
+                    stream_id: escape_empty_string(stream.stream_id),
                   },
                 ],
               },
-            }
+            },
+            base_url: '/'
           )
         end
 
+        # @param marker [String]
+        # @return [String]
+        def event_marker_path(marker)
+          build_path({ filter: { markers: [escape_empty_string(marker)] } }, base_url: '/')
+        end
+
         # @param str [String]
         # @return [String]
         def empty_characters_fallback(str)
@@ -113,16 +120,16 @@ module PgEventstore
         def build_starting_id_link(starting_id)
           return 'javascript: void(0);' unless starting_id
 
-          build_path(params.merge(starting_id: starting_id))
+          build_path(params.merge(starting_id:))
         end
 
         # @param params [Hash, Array]
         # @return [String]
-        def build_path(params)
+        def build_path(params, base_url: request.path)
           encoded_params = Rack::Utils.build_nested_query(params)
-          return request.path if encoded_params.empty?
+          return base_url if encoded_params.empty?
 
-          "#{request.path}?#{encoded_params}"
+          "#{base_url}?#{encoded_params}"
         end
       end
     end

data/lib/pg_eventstore/web/paginator/markers_collection.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Paginator
+      class MarkersCollection < BaseCollection
+        # @return [Integer]
+        PER_PAGE = 10
+
+        # @return [Array<Hash<String => String>>]
+        def collection
+          @collection ||=
+            begin
+              sql_builder = SQLBuilder.new.select('name').from('event_markers')
+              sql_builder.where("name #{direction_operator} ?", starting_id) if starting_id
+              sql_builder.where('name like ?', "#{options[:query]}%")
+              sql_builder.order("name #{order}").limit(per_page)
+              connection.with do |conn|
+                conn.exec_params(*sql_builder.to_exec_params)
+              end.to_a
+            end
+        end
+
+        # @return [String, nil]
+        def next_page_starting_id
+          return unless collection.size == per_page
+
+          starting_id = collection.first['name']
+          sql_builder = SQLBuilder.new.select('name').from('event_markers')
+          sql_builder.where("name #{direction_operator} ?", starting_id)
+          sql_builder.where('name like ?', "#{options[:query]}%")
+          sql_builder.order("name #{order}").limit(1).offset(per_page)
+
+          connection.with do |conn|
+            conn.exec_params(*sql_builder.to_exec_params)
+          end.to_a.dig(0, 'name')
+        end
+
+        private
+
+        # @return [String]
+        def direction_operator
+          order == :asc ? '>=' : '<='
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/paginator/stream_contexts_collection.rb

@@ -14,7 +14,7 @@ module PgEventstore
               sql_builder = SQLBuilder.new.select('context').from('partitions')
               sql_builder.where('stream_name is null and event_type is null')
               sql_builder.where("context #{direction_operator} ?", starting_id) if starting_id
-              sql_builder.where('context ilike ?', "%#{options[:query]}%")
+              compute_context_filter(sql_builder)
               sql_builder.limit(per_page).order("context #{order}")
               connection.with do |conn|
                 conn.exec_params(*sql_builder.to_exec_params)
@@ -30,7 +30,7 @@ module PgEventstore
           sql_builder = SQLBuilder.new.select('context').from('partitions')
           sql_builder.where('stream_name is null and event_type is null')
           sql_builder.where("context #{direction_operator} ?", starting_id)
-          sql_builder.where('context ilike ?', "%#{options[:query]}%")
+          compute_context_filter(sql_builder)
           sql_builder.limit(1).offset(per_page).order("context #{order}")
 
           connection.with do |conn|
@@ -44,6 +44,17 @@ module PgEventstore
         def direction_operator
           order == :asc ? '>=' : '<='
         end
+
+        # @param builder [PgEventstore::SQLBuilder]
+        # @return [void]
+        def compute_context_filter(builder)
+          query = options[:query].to_s
+          if query.size < MIN_QUERY_SIZE_FOR_ADVANCE_SEARCH
+            builder.where('context like ?', "#{query}%")
+          else
+            builder.where('context ilike ?', "%#{query}%")
+          end
+        end
       end
     end
   end

data/lib/pg_eventstore/web/paginator/stream_ids_collection.rb

@@ -11,11 +11,9 @@ module PgEventstore
         def collection
           @collection ||=
             begin
-              sql_builder = SQLBuilder.new.select('stream_id').from(Event::PRIMARY_TABLE_NAME)
-              sql_builder.where('context = ? and stream_name = ?', options[:context], options[:stream_name])
-              sql_builder.where('stream_id like ?', "#{options[:query]}%")
-              sql_builder.where("stream_id #{direction_operator} ?", starting_id) if starting_id
-              sql_builder.group('stream_id').limit(per_page).order("stream_id #{order}")
+              sql_builder = streams_index_sql_builder(from_id: starting_id)
+              sql_builder.limit(per_page)
+
               connection.with do |conn|
                 conn.exec_params(*sql_builder.to_exec_params)
               end.to_a
@@ -26,12 +24,8 @@ module PgEventstore
         def next_page_starting_id
           return unless collection.size == per_page
 
-          starting_id = collection.first['stream_id']
-          sql_builder = SQLBuilder.new.select('stream_id').from(Event::PRIMARY_TABLE_NAME)
-          sql_builder.where("stream_id #{direction_operator} ?", starting_id)
-          sql_builder.where('stream_id like ?', "#{options[:query]}%")
-          sql_builder.where('context = ? and stream_name = ?', options[:context], options[:stream_name])
-          sql_builder.group('stream_id').limit(1).offset(per_page).order("stream_id #{order}")
+          sql_builder = streams_index_sql_builder(from_id: collection.first['stream_id'])
+          sql_builder.limit(1).offset(per_page)
 
           connection.with do |conn|
             conn.exec_params(*sql_builder.to_exec_params)
@@ -40,6 +34,22 @@ module PgEventstore
 
         private
 
+        def partitions_sql_builder
+          filters = QueryBuilders::Filters::Collection.from_options(
+            { filter: { streams: [{ context: options[:context], stream_name: options[:stream_name] }] } }
+          )
+          QueryBuilders::PartitionsFiltering.assemble_sql_builder(filters, scope: :stream_name).unselect.select('id')
+        end
+
+        def streams_index_sql_builder(from_id: nil)
+          sql_builder = SQLBuilder.new.select('stream_id')
+          sql_builder.from(QueryBuilders::StreamsGlobalIndexFiltering::PRIMARY_TABLE_NAME)
+          sql_builder.where('partition_id = ?', partitions_sql_builder)
+          sql_builder.where('stream_id like ?', "#{options[:query]}%")
+          sql_builder.where("stream_id #{direction_operator} ?", from_id) if from_id
+          sql_builder.order("stream_id #{order}")
+        end
+
         # @return [String]
         def direction_operator
           order == :asc ? '>=' : '<='

data/lib/pg_eventstore/web/paginator/stream_names_collection.rb

@@ -12,9 +12,9 @@ module PgEventstore
           @collection ||=
             begin
               sql_builder = SQLBuilder.new.select('stream_name').from('partitions')
-              sql_builder.where('event_type is null and context = ?', options[:context])
-              sql_builder.where('stream_name ilike ?', "%#{options[:query]}%")
+              sql_builder.where('event_type is null and stream_name is not null and context = ?', options[:context])
               sql_builder.where("stream_name #{direction_operator} ?", starting_id) if starting_id
+              compute_stream_name_filter(sql_builder)
               sql_builder.limit(per_page).order("stream_name #{order}")
               connection.with do |conn|
                 conn.exec_params(*sql_builder.to_exec_params)
@@ -29,10 +29,9 @@ module PgEventstore
           starting_id = collection.first['stream_name']
           sql_builder = SQLBuilder.new.select('stream_name').from('partitions')
           sql_builder.where("stream_name #{direction_operator} ?", starting_id)
-          sql_builder.where('stream_name ilike ?', "%#{options[:query]}%")
+          compute_stream_name_filter(sql_builder)
           sql_builder.where('event_type is null and context = ?', options[:context])
           sql_builder.limit(1).offset(per_page).order("stream_name #{order}")
-
           connection.with do |conn|
             conn.exec_params(*sql_builder.to_exec_params)
           end.to_a.dig(0, 'stream_name')
@@ -44,6 +43,21 @@ module PgEventstore
         def direction_operator
           order == :asc ? '>=' : '<='
         end
+
+        # @param builder [PgEventstore::SQLBuilder]
+        # @return [void]
+        def compute_stream_name_filter(builder)
+          query = options[:query].to_s
+          if query.size < MIN_QUERY_SIZE_FOR_ADVANCE_SEARCH
+            builder.where('stream_name like ?', "#{query}%")
+          else
+            builder.where(
+              'context || ? || stream_name ilike ?',
+              PgEventstore::Event::SYSTEM_SYMBOL,
+              "#{options[:context]}#{PgEventstore::Event::SYSTEM_SYMBOL}%#{query}%"
+            )
+          end
+        end
       end
     end
   end

data/lib/pg_eventstore/web/paginator/streams_collection.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Web
+    module Paginator
+      class StreamsCollection < BaseCollection
+        # @return [Hash<String => Symbol>] SQL directions, string-to-symbol mapping
+        SQL_DIRECTIONS = {
+          'asc' => :asc,
+          'desc' => :desc,
+        }.tap do |directions|
+          directions.default = :desc
+        end.freeze
+        # @return [Hash<String => Integer>] per page limits, string to Integer mapping
+        PER_PAGE = %w[10 20 50 100 1000].to_h { [_1, _1.to_i] }.tap do |per_page|
+          per_page.default = 10
+        end.freeze
+        # @return [Integer] max number of events after which we don't perform the exact count and keep the estimate
+        #   count instead because of the potential performance degradation.
+        MAX_NUMBER_TO_COUNT = 10_000
+
+        # @return [Array<PgEventstore::Stream>]
+        def collection
+          @collection ||= PgEventstore.client(config_name).read_streams(
+            options: { from_position: starting_id, max_count: per_page, direction: order }
+          )
+        end
+
+        # @return [Integer, nil]
+        def next_page_starting_id
+          return unless collection.size == per_page
+
+          from_position = collection.first&.starting_position
+          sql_builder = QueryBuilders::StreamsGlobalIndexFiltering.sql_builder_for_basic_pagination(
+            from_position:, max_count: 1, direction: order
+          ).unselect.select('starting_position').offset(per_page)
+          starting_position(sql_builder)
+        end
+
+        # @return [Integer, nil]
+        def prev_page_starting_id
+          from_position = collection.first&.starting_position || starting_id
+          sql_builder = QueryBuilders::StreamsGlobalIndexFiltering.sql_builder_for_basic_pagination(
+            from_position:, max_count: per_page, direction: order == :asc ? :desc : :asc
+          ).unselect.select('starting_position').offset(1)
+          sql_builder = SQLBuilder.new.select('starting_position').from(sql_builder)
+          sql_builder.order("starting_position #{order}").limit(1)
+          starting_position(sql_builder)
+        end
+
+        # @return [Integer]
+        def total_count
+          @total_count ||=
+            begin
+              sql_builder = QueryBuilders::StreamsGlobalIndexFiltering.sql_builder_for_basic_pagination({})
+              sql_builder.remove_limit.remove_group.remove_order
+              count = estimate_count(sql_builder)
+              return count if count > MAX_NUMBER_TO_COUNT
+
+              regular_count(sql_builder)
+            end
+        end
+
+        private
+
+        # @param sql_builder [PgEventstore::SQLBuilder]
+        # @return [Integer]
+        def estimate_count(sql_builder)
+          sql, params = sql_builder.to_exec_params
+          connection.with do |conn|
+            conn.exec_params("EXPLAIN #{sql}", params)
+          end.to_a.first['QUERY PLAN'].match(/rows=(\d+)/)[1].to_i
+        end
+
+        # @param sql_builder [PgEventstore::SQLBuilder]
+        # @return [Integer]
+        def regular_count(sql_builder)
+          sql_builder.unselect.select('count(*) as count_all')
+
+          connection.with do |conn|
+            conn.exec_params(*sql_builder.to_exec_params)
+          end.to_a.first['count_all']
+        end
+
+        # @param sql_builder [PgEventstore::SQLBuilder]
+        # @return [Integer, nil]
+        def starting_position(sql_builder)
+          connection.with do |conn|
+            conn.exec_params(*sql_builder.to_exec_params)
+          end.to_a.dig(0, 'starting_position')
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/web/public/javascripts/pg_eventstore.js

@@ -98,15 +98,9 @@ $(function(){
         $streamIdSelect.on('change.select2', removeDeleteBtn);
     }
 
-    let initSystemStreamFilterAutocomplete = function($filter){
-        let $streamNameSelect = $filter.find('select');
-        $streamNameSelect.select2({
-            allowClear: true
-        });
-    }
-
     let initEventTypeFilterAutocomplete = function($filter) {
-        let $eventTypeSelect = $filter.find('select');
+        let $eventTypeSelect = $filter.find('select.type-filter');
+        let $eventMarkersSelect = $filter.find('select.event-markers-filter');
         $eventTypeSelect.select2({
             ajax: {
                 url: $eventTypeSelect.data('url'),
@@ -123,6 +117,42 @@ $(function(){
             templateSelection: templateSelection,
             templateResult: templateResult,
         });
+        $eventMarkersSelect.select2({
+            ajax: {
+                url: $eventMarkersSelect.data('url'),
+                processResults: function(data, params){
+                    params.starting_id = data.pagination.starting_id;
+                    data.results.forEach(function(contextAttrs){
+                        contextAttrs.id = contextAttrs.name;
+                        contextAttrs.text = contextAttrs.name;
+                    });
+                    return data;
+                },
+            },
+            allowClear: true,
+            templateSelection: templateSelection,
+            templateResult: templateResult,
+        });
+    }
+
+    let initMarkersFilterAutocomplete = function($filter) {
+        let $markersSelect = $filter.find('select.markers-filter');
+        $markersSelect.select2({
+            ajax: {
+                url: $markersSelect.data('url'),
+                processResults: function(data, params){
+                    params.starting_id = data.pagination.starting_id;
+                    data.results.forEach(function(contextAttrs){
+                        contextAttrs.id = contextAttrs.name;
+                        contextAttrs.text = contextAttrs.name;
+                    });
+                    return data;
+                },
+            },
+            allowClear: true,
+            templateSelection: templateSelection,
+            templateResult: templateResult,
+        });
     }
 
     // Per page drop down
@@ -133,10 +163,10 @@ $(function(){
     let $filtersForm = $('#filters-form');
     // Stream filter template
     let streamFilterTmpl = $('#stream-filter-tmpl').text();
-    // System stream filter template
-    let systemStreamFilterTmpl = $('#system-stream-filter-tmpl').text();
     // Event type filter template
     let eventFilterTmpl = $('#event-type-filter-tmpl').text();
+    // Markers filter template
+    let markersFilterTmpl = $('#markers-filter-tmpl').text();
 
     // Remove filter button
     $filtersForm.on('click', '.remove-filter', function(){
@@ -148,19 +178,16 @@ $(function(){
         $filtersForm.find('.stream-filters').append(streamFilterTmpl.replace(/%NUM%/g, filtersNum));
         initStreamFilterAutocomplete($filtersForm.find('.stream-filters').children().last());
     });
-    // "Add system stream filter" button
-    $filtersForm.on('click', '.add-system-stream-filter', function(){
-        if($filtersForm.find('.system-stream-filter').children().length > 0)
-            return;
-
-        $filtersForm.find('.system-stream-filter').append(systemStreamFilterTmpl);
-        initSystemStreamFilterAutocomplete($filtersForm.find('.system-stream-filter').children().last());
-    });
     // "Add event type filter" button
     $filtersForm.on('click', '.add-event-filter', function(){
         $filtersForm.find('.event-filters').append(eventFilterTmpl);
         initEventTypeFilterAutocomplete($filtersForm.find('.event-filters').children().last());
     });
+    // "Add markers filter" button
+    $filtersForm.on('click', '.add-markers-filter', function(){
+        $filtersForm.find('.markers-filter').append(markersFilterTmpl);
+        initMarkersFilterAutocomplete($filtersForm.find('div.markers-filter').children().last());
+    });
 
     // Because zero-byte character can't be rendered into HTML properly - loop through each option, marked as containing
     // zero-byte value and assign zero-byte value explicitly. This must be done before we initialize select2 plugin.
@@ -171,14 +198,14 @@ $(function(){
     $filtersForm.find('.stream-filters').children().each(function(){
         initStreamFilterAutocomplete($(this));
     });
-    // Init select2 for system stream filter which were initially rendered
-    $filtersForm.find('.system-stream-filter').children().each(function(){
-        initSystemStreamFilterAutocomplete($(this));
-    });
     // Init select2 for event type filters which were initially rendered
     $filtersForm.find('.event-filters').children().each(function(){
         initEventTypeFilterAutocomplete($(this));
     });
+    // Init select2 for markers filter which was initially rendered
+    $filtersForm.find('.markers-filter').children().each(function(){
+        initMarkersFilterAutocomplete($(this));
+    });
 
     // Automatically refresh events list every two seconds
     let autoRefreshInterval;

data/lib/pg_eventstore/web/subscriptions/helpers.rb

@@ -9,7 +9,7 @@ module PgEventstore
         def subscriptions_url(set_name: nil)
           return url('/subscriptions') unless set_name
 
-          encoded_params = Rack::Utils.build_nested_query(set_name: set_name)
+          encoded_params = Rack::Utils.build_nested_query(set_name:)
           url("/subscriptions?#{encoded_params}")
         end
 
@@ -117,7 +117,7 @@ module PgEventstore
         # @param ids [Array<Integer>]
         # @return [String]
         def delete_all_subscriptions_url(ids)
-          encoded_params = Rack::Utils.build_nested_query(ids: ids)
+          encoded_params = Rack::Utils.build_nested_query(ids:)
           url("/delete_all_subscriptions?#{encoded_params}")
         end
 

data/lib/pg_eventstore/web/subscriptions/set_collection.rb

@@ -23,7 +23,7 @@ module PgEventstore
 
         # @return [PgEventstore::SubscriptionQueries]
         def subscription_queries
-          SubscriptionQueries.new(connection)
+          SubscriptionQueries.new(connection, QueryStrategy::Foreground.new(connection))
         end
 
         # @return [PgEventstore::SubscriptionsSetQueries]

data/lib/pg_eventstore/web/subscriptions/subscriptions.rb

@@ -27,7 +27,7 @@ module PgEventstore
 
         # @return [PgEventstore::SubscriptionQueries]
         def subscription_queries
-          SubscriptionQueries.new(connection)
+          SubscriptionQueries.new(connection, QueryStrategy::Foreground.new(connection))
         end
       end
     end

data/lib/pg_eventstore/web/subscriptions/with_state/set_collection.rb

@@ -26,7 +26,7 @@ module PgEventstore
 
           # @return [PgEventstore::SubscriptionQueries]
           def subscription_queries
-            SubscriptionQueries.new(connection)
+            SubscriptionQueries.new(connection, QueryStrategy::Foreground.new(connection))
           end
         end
       end

data/lib/pg_eventstore/web/subscriptions/with_state/subscriptions.rb

@@ -30,7 +30,7 @@ module PgEventstore
 
           # @return [PgEventstore::SubscriptionQueries]
           def subscription_queries
-            SubscriptionQueries.new(connection)
+            SubscriptionQueries.new(connection, QueryStrategy::Foreground.new(connection))
           end
         end
       end