pg_eventstore 1.11.0 → 1.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 402dc90a012b5e1ee934da4f62c4af4ba62e69693a45dac1a84d461950da5717
-  data.tar.gz: 6c5d1fdc2e9475eca72b5ef27d0081e0aec6331f76639b7172e676d8b6e63e3d
+  metadata.gz: 3de3a26ba4a2773815fdc36adc4852acf4ce1248b6add884673c9009bae8f9d1
+  data.tar.gz: 2a160259af68bb8522497f5013754215ee19a95d862388246a45498c8f24d8a0
 SHA512:
-  metadata.gz: b20c1124f2108b671bd28f3677fb3b2ae88f3b19d2470eb5a31102eb597d4bd5af1080f9bd337b1528107eef2e4bce6b882d434aa8536abc0a5a80b373bfe3a3
-  data.tar.gz: 9fd5af079cedc277e278b2e0b50e628601880a81ec5bf7c94700595f6472be389b2fa3d3a2ba2f5d311c8844da40de2c4a52e4dc6c2bda2ca3559e94b267f7f2
+  metadata.gz: 5b044fb3e689885d85b3af84caade98f5a66a020cf40991a6b0b2fd2bbf352089be24b7f31313de8c537fe65dc56c9fad2dec4b281eb792aae0bfa150c4d63be
+  data.tar.gz: '0899781c0d9fb93cd7e166897edcf099e06d5b1582a32839d016b897a690e5ef141db79c94d8fd807312bb3d03ba867ceb905dbe0e31f92a3e4be4cc5cc1f91f'

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
 
+## [1.13.0]
+
+- Introduce automatic subscriptions recovery from connection errors. This way if a subscription process loses the connection to the database - it will be trying to reconnect until the connection is restored.
+- Resolve ambiguity in usage of `PgEventstore.config` method. It now returns the frozen object.
+
+## [1.12.0]
+
+- Introduce `#read_grouped` API method that allows to group events by type
+
 ## [1.11.0]
 
 - Add a global position that caused an error to the subscription's error JSON info. This will help you understand what event caused your subscription to fail.

data/docs/reading_events.md

@@ -230,3 +230,101 @@ PgEventstore.client.read_paginated(projection_stream, options: { resolve_link_to
   end
 end
 ```
+
+## Grouping events by type
+
+`pg_eventstore` implements an ability to group events by type when reading from a stream. Example:
+
+```ruby
+stream = PgEventstore::Stream.new(context: 'FooCtx', stream_name: 'Foo', stream_id: '1')
+event1 = PgEventstore::Event.new(type: 'Foo', data: { foo: 1 })
+event2 = PgEventstore::Event.new(type: 'Foo', data: { foo: 2 })
+event3 = PgEventstore::Event.new(type: 'Bar', data: { bar: 2 })
+
+PgEventstore.client.append_to_stream(stream, [event1, event2, event3])
+
+PgEventstore.client.read_grouped(stream) # => returns event1 and event3
+```
+
+API is very similar to the API of `#read`, but it ignores `:max_count` options as the result is always returns a set of event types in your stream.
+
+Reading most recent events:
+
+```ruby
+PgEventstore.client.read_grouped(stream, options: { direction: :desc })
+```
+
+Filtering the result by stream attributes:
+
+```ruby
+PgEventstore.client.read_grouped(
+  PgEventstore::Stream.all_stream,
+  options: { filter: { streams: [{ context: 'FooCtx' }] } }
+)
+```
+
+Filtering the result by event types:
+
+```ruby
+PgEventstore.client.read_grouped(
+  PgEventstore::Stream.all_stream,
+  options: { filter: { event_types: ['Foo', 'Bar'] } }
+)
+```
+
+Filtering by stream attributes and event types:
+
+```ruby
+PgEventstore.client.read_grouped(
+  PgEventstore::Stream.all_stream,
+  options: { filter: { streams: [{ context: 'FooCtx' }], event_types: ['Foo', 'Bar'] } }
+)
+```
+
+Reading most recent events until the certain stream revision:
+
+```ruby
+PgEventstore.client.read_grouped(stream, options: { direction: :desc, from_revision: 1 })
+```
+
+Reading the oldest events from the certain stream revision:
+
+```ruby
+PgEventstore.client.read_grouped(stream, options: { direction: :asc, from_revision: 1 })
+```
+
+Reading most recent events until the certain global position:
+
+```ruby
+PgEventstore.client.read_grouped(PgEventstore::Stream.all_stream, options: { direction: :desc, from_position: 5 })
+```
+
+Reading the oldest events from the certain global position:
+
+```ruby
+PgEventstore.client.read_grouped(PgEventstore::Stream.all_stream, options: { direction: :asc, from_position: 5 })
+```
+
+### Event types list lookup
+
+If you do not provide event types filter - event types list will be determined based on the rest of arguments(a stream argument or a stream filters option).
+
+### Multiple events of same type in the result
+
+If same event type appear in different streams(different by `#context` and `#stream_name`) - those events will appear in the result. This is because even though `Event#type` value may be the same - its meaning may have different meaning in different `context`/`stream_name` couple. Example:
+
+```ruby
+stream1 = PgEventstore::Stream.new(context: 'FooCtx', stream_name: 'Foo', stream_id: '1')
+stream2 = PgEventstore::Stream.new(context: 'FooCtx', stream_name: 'Bar', stream_id: '1')
+
+event1 = PgEventstore::Event.new(type: 'Foo', data: { foo: 1 })
+event2 = PgEventstore::Event.new(type: 'Foo', data: { foo: 2 })
+
+PgEventstore.client.append_to_stream(stream1, event1)
+PgEventstore.client.append_to_stream(stream2, event2)
+
+PgEventstore.client.read_grouped(
+  PgEventstore::Stream.all_stream,
+  options: { filter: { streams: [{ context: 'FooCtx' }] } }
+) # => returns both events even though they are of "Foo" type
+```

data/lib/pg_eventstore/cli/commands/base_command.rb

@@ -3,7 +3,9 @@
 module PgEventstore
   module CLI
     module Commands
+      # @!visibility private
       class BaseCommand
+        # @!visibility private
         module BaseCommandActions
           # @return [Integer] exit code
           def call

data/lib/pg_eventstore/cli/commands/callback_handlers/start_cmd_handlers.rb

@@ -4,6 +4,7 @@ module PgEventstore
   module CLI
     module Commands
       module CallbackHandlers
+        # @!visibility private
         class StartCmdHandlers
           include Extensions::CallbackHandlersExtension
 

data/lib/pg_eventstore/cli/commands/help_command.rb

@@ -3,6 +3,7 @@
 module PgEventstore
   module CLI
     module Commands
+      # @!visibility private
       class HelpCommand
         attr_reader :options
 

data/lib/pg_eventstore/cli/commands/start_subscriptions_command.rb

@@ -5,6 +5,7 @@ require_relative 'callback_handlers/start_cmd_handlers'
 module PgEventstore
   module CLI
     module Commands
+      # @!visibility private
       class StartSubscriptionsCommand < BaseCommand
         # @return [Integer] seconds
         KEEP_ALIVE_INTERVAL = 2
@@ -55,8 +56,9 @@ module PgEventstore
                 manager.stop
               end
             end.each(&:join)
-            Utils.remove_file(options.pid_path)
+          ensure
             @running = false
+            Utils.remove_file(options.pid_path)
           end
         end
 

data/lib/pg_eventstore/cli/commands/stop_subscriptions_command.rb

@@ -3,6 +3,7 @@
 module PgEventstore
   module CLI
     module Commands
+      # @!visibility private
       class StopSubscriptionsCommand < BaseCommand
         # @return [Integer] exit code
         def call

data/lib/pg_eventstore/cli/exit_codes.rb

@@ -2,6 +2,7 @@
 
 module PgEventstore
   module CLI
+    # @!visibility private
     module ExitCodes
       # @return [Integer]
       SUCCESS = 0

data/lib/pg_eventstore/cli/parser_options/base_options.rb

@@ -3,6 +3,7 @@
 module PgEventstore
   module CLI
     module ParserOptions
+      # @!visibility private
       class BaseOptions
         include Extensions::OptionsExtension
 

data/lib/pg_eventstore/cli/parser_options/default_options.rb

@@ -3,6 +3,7 @@
 module PgEventstore
   module CLI
     module ParserOptions
+      # @!visibility private
       class DefaultOptions < BaseOptions
       end
     end

data/lib/pg_eventstore/cli/parser_options/metadata.rb

@@ -3,6 +3,7 @@
 module PgEventstore
   module CLI
     module ParserOptions
+      # @!visibility private
       class Metadata
         include Extensions::OptionsExtension
 

data/lib/pg_eventstore/cli/parser_options/subscription_options.rb

@@ -3,6 +3,7 @@
 module PgEventstore
   module CLI
     module ParserOptions
+      # @!visibility private
       class SubscriptionOptions < BaseOptions
         option(
           :pid_path,

data/lib/pg_eventstore/cli/try_to_delete_subscriptions_set.rb

@@ -2,6 +2,7 @@
 
 module PgEventstore
   module CLI
+    # @!visibility private
     class TryToDeleteSubscriptionsSet
       class << self
         def try_to_delete(...)
@@ -30,7 +31,7 @@ module PgEventstore
         # Potentially CommandsHandler can be dead exactly at the same moment we expect it to process "Ping" command.
         # Wait for potential recover plus run interval and plus another second to allow potential processing of
         # "Ping" command. "Ping" command comes in prio, so it is guaranteed it will be processed as a first command.
-        sleep CommandsHandler::RESTART_DELAY + CommandsHandler::PULL_INTERVAL + 1
+        sleep RunnerRecoveryStrategies::RestoreConnection::TIME_BETWEEN_RETRIES + CommandsHandler::PULL_INTERVAL + 1
         if subscriptions_set_commands_queries.find_by(subscriptions_set_id: subscriptions_set_id, command_name: cmd_name)
           # "Ping" command wasn't consumed. Related process must be dead.
           subscriptions_set_queries.delete(subscriptions_set_id)

data/lib/pg_eventstore/cli/try_unlock_subscriptions_set.rb

@@ -5,6 +5,7 @@ require_relative 'wait_for_subscriptions_set_shutdown'
 
 module PgEventstore
   module CLI
+    # @!visibility private
     class TryUnlockSubscriptionsSet
       class << self
         def try_unlock(...)

data/lib/pg_eventstore/cli/wait_for_subscriptions_set_shutdown.rb

@@ -2,6 +2,7 @@
 
 module PgEventstore
   module CLI
+    # @!visibility private
     class WaitForSubscriptionsSetShutdown
       class << self
         def wait_for_shutdown(...)

data/lib/pg_eventstore/client.rb

@@ -55,10 +55,11 @@ module PgEventstore
     # Read events from the specific stream or from "all" stream.
     # @param stream [PgEventstore::Stream]
     # @param options [Hash] request options
-    # @option options [String] :direction read direction - 'Forwards' or 'Backwards'
+    # @option options [String] :direction read direction. Allowed values are "Forwards", "Backwards", "asc", "desc",
+    #   :asc, :desc
     # @option options [Integer] :from_revision a starting revision number. **Use this option when stream name is a
     #   normal stream name**
-    # @option options [Integer, Symbol] :from_position a starting global position number. **Use this option when reading
+    # @option options [Integer] :from_position a starting global position number. **Use this option when reading
     #   from "all" stream**
     # @option options [Integer] :max_count max number of events to return in one response. Defaults to config.max_count
     # @option options [Boolean] :resolve_link_tos When using projections to create new events you
@@ -94,7 +95,7 @@ module PgEventstore
     #       }
     #     )
     #
-    #     # Filtering the a mix of context and event type
+    #     # Filtering a mix of context and event type
     #     PgEventstore.client.read(
     #       PgEventstore::Stream.all_stream,
     #       options: { filter: { streams: [{ context: 'User' }], event_types: ['MyAwesomeEvent'] } }
@@ -123,6 +124,23 @@ module PgEventstore
         call(stream, options: { max_count: config.max_count }.merge(options))
     end
 
+    # Takes a stream, determines a list of even types in it and returns most recent(or very first - depending on
+    # :direction option) events, one of each type. If :event_types filter is provided - uses it instead of automatic
+    # event types lookup logic. The result size is almost always less than or equal to event types list size, so passing
+    # :max_count option does not make any effect. In case if event of same type appears in different context/stream
+    # name - it will be counted as a different event, thus, may appear several times in the result.
+    # @see {#read} for the detailed docs
+    # @param stream [PgEventstore::Stream]
+    # @param options [Hash] request options
+    # @param middlewares [Array, nil]
+    # @return [Array<PgEventstore::Event>]
+    def read_grouped(stream, options: {}, middlewares: nil)
+      cmd_class = stream.all_stream? ? Commands::AllStreamReadGrouped : Commands::RegularStreamReadGrouped
+      cmd_class.
+        new(Queries.new(partitions: partition_queries, events: event_queries(middlewares(middlewares)))).
+        call(stream, options: options)
+    end
+
     # Links event from one stream into another stream. You can later access it by providing :resolve_link_tos option
     # when reading from a stream. Only existing events can be linked.
     # @param stream [PgEventstore::Stream]
@@ -149,7 +167,7 @@ module PgEventstore
     private
 
     # @param middlewares [Array, nil]
-    # @return [Array<Object<#serialize, #deserialize>>]
+    # @return [Array<PgEventstore::Middleware>]
     def middlewares(middlewares = nil)
       return config.middlewares.values unless middlewares
 

data/lib/pg_eventstore/commands/all_stream_read_grouped.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Commands
+    # @!visibility private
+    class AllStreamReadGrouped < AbstractCommand
+      # @param stream [PgEventstore::Stream]
+      # @param options [Hash] request options
+      # @option options [String] :direction read direction
+      # @option options [Integer, Symbol] :from_position. **Use this option when reading from "all" stream**
+      # @option options [Boolean] :resolve_link_tos
+      # @option options [Hash] :filter provide it to filter events
+      # @return [Array<PgEventstore::Event>]
+      # @raise [PgEventstore::StreamNotFoundError]
+      def call(stream, options: {})
+        event_types = QueryBuilders::PartitionsFiltering.extract_event_types_filter(options)
+        stream_filters = QueryBuilders::PartitionsFiltering.extract_streams_filter(options)
+        stream_ids_grouped = group_stream_ids(options)
+        options_by_event_type =
+          queries.partitions.partitions(stream_filters, event_types).flat_map do |partition|
+            stream_ids = stream_ids_grouped[[partition.context, partition.stream_name]]
+            next build_filter_options_for_streams(partition, stream_ids, options) if stream_ids
+
+            build_filter_options_for_partitions(partition, options)
+          end
+        queries.events.grouped_events(stream, options_by_event_type, **options.slice(:resolve_link_tos))
+      end
+
+      private
+
+      # @param options [Hash]
+      # @return [Hash]
+      def group_stream_ids(options)
+        event_stream_filters = QueryBuilders::EventsFiltering.extract_streams_filter(options)
+        event_stream_filters.each_with_object({}) do |attrs, res|
+          next unless attrs[:stream_id]
+
+          res[[attrs[:context], attrs[:stream_name]]] ||= []
+          res[[attrs[:context], attrs[:stream_name]]].push(attrs[:stream_id])
+        end
+      end
+
+      # @param partition [PgEventstore::Partition]
+      # @param stream_ids [Array<String>]
+      # @param options [Hash]
+      # @return [Array<Hash>]
+      def build_filter_options_for_streams(partition, stream_ids, options)
+        stream_ids.map do |stream_id|
+          filter = {
+            streams: [{ context: partition.context, stream_name: partition.stream_name, stream_id: stream_id }],
+            event_types: [partition.event_type]
+          }
+          options.merge(filter: filter, max_count: 1)
+        end
+      end
+
+      # @param partition [PgEventstore::Partition]
+      # @param options [Hash]
+      # @return [Hash]
+      def build_filter_options_for_partitions(partition, options)
+        filter = {
+          streams: [{ context: partition.context, stream_name: partition.stream_name }],
+          event_types: [partition.event_type]
+        }
+        options.merge(filter: filter, max_count: 1)
+      end
+    end
+  end
+end

data/lib/pg_eventstore/commands/regular_stream_read_grouped.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Commands
+    # @!visibility private
+    class RegularStreamReadGrouped < AbstractCommand
+      # @param stream [PgEventstore::Stream]
+      # @param options [Hash] request options
+      # @option options [String] :direction read direction
+      # @option options [Integer, Symbol] :from_revision. **Use this option when stream name is a normal stream name**
+      # @option options [Integer, Symbol] :from_position. **Use this option when reading from "all" stream**
+      # @option options [Boolean] :resolve_link_tos
+      # @option options [Hash] :filter provide it to filter events
+      # @return [Array<PgEventstore::Event>]
+      # @raise [PgEventstore::StreamNotFoundError]
+      def call(stream, options: {})
+        queries.events.stream_revision(stream) || raise(StreamNotFoundError, stream)
+
+        event_types = QueryBuilders::PartitionsFiltering.extract_event_types_filter(options)
+        stream_filters = QueryBuilders::PartitionsFiltering.extract_streams_filter(
+          filter: { streams: [{ context: stream.context, stream_name: stream.stream_name }] }
+        )
+        options_by_event_type = queries.partitions.partitions(stream_filters, event_types).map do |partition|
+          filter = { event_types: [partition.event_type] }
+          options.merge(filter: filter, max_count: 1)
+        end
+        queries.events.grouped_events(stream, options_by_event_type, **options.slice(:resolve_link_tos))
+      end
+    end
+  end
+end

data/lib/pg_eventstore/commands.rb

@@ -5,6 +5,8 @@ require_relative 'commands/event_modifiers/prepare_link_event'
 require_relative 'commands/event_modifiers/prepare_regular_event'
 require_relative 'commands/append'
 require_relative 'commands/read'
+require_relative 'commands/regular_stream_read_grouped'
+require_relative 'commands/all_stream_read_grouped'
 require_relative 'commands/regular_stream_read_paginated'
 require_relative 'commands/system_stream_read_paginated'
 require_relative 'commands/multiple'

data/lib/pg_eventstore/connection.rb

@@ -67,9 +67,9 @@ module PgEventstore
       should_retry = true
       @pool.with do |conn|
         yield conn
-      rescue PG::ConnectionBad
-        # Recover connection after fork. We do it only once and without any delay. Recover is required by both
-        # processes - child process and parent process
+      rescue PG::ConnectionBad, PG::UnableToSend
+        # Recover a connection after fork or when we lost a connection to PostgreSQL. We retry only once and without any
+        # delay.
         conn.sync_reset
         raise unless should_retry
         should_retry = false
@@ -77,6 +77,11 @@ module PgEventstore
       end
     end
 
+    # @return [void]
+    def shutdown
+      @pool.shutdown(&:close)
+    end
+
     private
 
     # @return [ConnectionPool]

data/lib/pg_eventstore/extensions/callback_handlers_extension.rb

@@ -2,6 +2,7 @@
 
 module PgEventstore
   module Extensions
+    # @!visibility private
     module CallbackHandlersExtension
       def self.included(klass)
         klass.extend(ClassMethods)

data/lib/pg_eventstore/partition.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  class Partition
+    include Extensions::OptionsExtension
+
+    # @!attribute id
+    #   @return [Integer]
+    option(:id)
+    # @!attribute context
+    #   @return [String]
+    option(:context)
+    # @!attribute stream_name
+    #   @return [String, nil]
+    option(:stream_name)
+    # @!attribute event_type
+    #   @return [String, nil]
+    option(:event_type)
+    # @!attribute table_name
+    #   @return [String]
+    option(:table_name)
+  end
+end

data/lib/pg_eventstore/pg_connection.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module PgEventstore
+  # @!visibility private
   class PgConnection < PG::Connection
     def exec(sql)
       log(sql, [])

data/lib/pg_eventstore/queries/event_queries.rb

@@ -98,6 +98,24 @@ module PgEventstore
       end
     end
 
+    # @param stream [PgEventstore::Stream]
+    # @param options_by_event_type [Array<Hash>] a set of options per an event type
+    # @param options [Hash]
+    # @option options [Boolean] :resolve_link_tos
+    # @return [Array<PgEventstore::Event>]
+    def grouped_events(stream, options_by_event_type, **options)
+      builders = options_by_event_type.map do |filter|
+        QueryBuilders::EventsFiltering.events_filtering(stream, filter)
+      end
+      final_builder = SQLBuilder.union_builders(builders.map(&:to_sql_builder))
+
+      raw_events = connection.with do |conn|
+        conn.exec_params(*final_builder.to_exec_params)
+      end.to_a
+      raw_events = links_resolver.resolve(raw_events) if options[:resolve_link_tos]
+      deserializer.deserialize_many(raw_events)
+    end
+
     private
 
     # @param stream [PgEventstore::Stream]

data/lib/pg_eventstore/queries/partition_queries.rb

@@ -176,6 +176,19 @@ module PgEventstore
       end.to_a
     end
 
+    # @param stream_filters [Array<Hash[Symbol, String]>]
+    # @param event_filters [Array<String>]
+    # @return [Array<PgEventstore::Partition>]
+    def partitions(stream_filters, event_filters)
+      partitions_filter = QueryBuilders::PartitionsFiltering.new
+      stream_filters.each { |attrs| partitions_filter.add_stream_attrs(**attrs) }
+      partitions_filter.add_event_types(event_filters)
+      partitions_filter.with_event_types
+      connection.with do |conn|
+        conn.exec_params(*partitions_filter.to_exec_params)
+      end.map(&method(:deserialize))
+    end
+
     # @param stream [PgEventstore::Stream]
     # @return [String]
     def context_partition_name(stream)
@@ -194,5 +207,13 @@ module PgEventstore
     def event_type_partition_name(stream, event_type)
       "event_types_#{Digest::MD5.hexdigest("#{stream.context}-#{stream.stream_name}-#{event_type}")[0..5]}"
     end
+
+    private
+
+    # @param attrs [Hash]
+    # @return [PgEventstore::Partition]
+    def deserialize(attrs)
+      Partition.new(**attrs.transform_keys(&:to_sym))
+    end
   end
 end

data/lib/pg_eventstore/queries.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 require_relative 'sql_builder'
+require_relative 'query_builders/basic_filtering'
 require_relative 'query_builders/events_filtering'
+require_relative 'query_builders/partitions_filtering'
 require_relative 'queries/transaction_queries'
 require_relative 'queries/event_queries'
 require_relative 'queries/partition_queries'

data/lib/pg_eventstore/query_builders/basic_filtering.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module QueryBuilders
+    # @!visibility private
+    class BasicFiltering
+      def initialize
+        @sql_builder = SQLBuilder.new.select("#{to_table_name}.*").from(to_table_name)
+      end
+
+      # @return [String]
+      def to_table_name
+        raise NotImplementedError
+      end
+
+      # @return [PgEventstore::SQLBuilder]
+      def to_sql_builder
+        @sql_builder
+      end
+
+      # @return [Array]
+      def to_exec_params
+        @sql_builder.to_exec_params
+      end
+    end
+  end
+end