pg_eventstore 2.0.0 → 3.0.0

data/lib/pg_eventstore/stream.rb

@@ -4,12 +4,10 @@ require 'digest/md5'
 
 module PgEventstore
   class Stream
-    # @return [String] a stream prefix of the system stream
-    SYSTEM_STREAM_PREFIX = '$'
     # @return [Integer]
     NON_EXISTING_STREAM_REVISION = -1
     # @return [Array<String>]
-    KNOWN_SYSTEM_STREAMS = %w[$streams].freeze
+    KNOWN_SYSTEM_STREAMS = %w[].freeze
 
     class << self
       # Produces "all" stream instance. "all" stream does not represent any specific stream. Instead, it indicates that
@@ -20,12 +18,6 @@ module PgEventstore
           stream.instance_variable_set(:@all_stream, true)
         end
       end
-
-      # @param name [String]
-      # @return [PgEventstore::Stream]
-      def system_stream(name)
-        new(context: name, stream_name: '', stream_id: '')
-      end
     end
 
     # @!attribute context
@@ -37,26 +29,31 @@ module PgEventstore
     # @!attribute stream_id
     #   @return [String]
     attr_reader :stream_id
+    # @!attribute stream_revision
+    #   @return [Integer, nil]
+    attr_reader :stream_revision
+    # @!attribute starting_position
+    #   @return [Integer, nil]
+    attr_reader :starting_position
 
     # @param context [String]
     # @param stream_name [String]
     # @param stream_id [String]
-    def initialize(context:, stream_name:, stream_id:)
+    # @param stream_revision [Integer, nil]
+    # @param starting_position [Integer, nil]
+    def initialize(context:, stream_name:, stream_id:, stream_revision: nil, starting_position: nil)
       @context = context
       @stream_name = stream_name
       @stream_id = stream_id
+      @stream_revision = stream_revision
+      @starting_position = starting_position
     end
 
     # @return [Boolean]
     def all_stream?
       !!@all_stream
     end
-
-    # Determine whether a stream is reserved by `pg_eventstore`. You can't append events to such streams.
-    # @return [Boolean]
-    def system?
-      all_stream? || context.start_with?(SYSTEM_STREAM_PREFIX)
-    end
+    alias system? all_stream?
 
     # @return [Array]
     def deconstruct

data/lib/pg_eventstore/stream_global_index.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class StreamGlobalIndex
+    include Extensions::OptionsExtension
+    include Extensions::OptionsDefaults
+
+    # @return [Integer]
+    INITIAL_STARTING_POSITION = -1
+
+    # @!attribute id
+    #   @return [Integer]
+    attribute(:id)
+    # @!attribute partition_id
+    #   @return [Integer]
+    attribute(:partition_id)
+    # @!attribute stream_id
+    #   @return [String]
+    attribute(:stream_id)
+    # @!attribute stream_revision
+    #   @return [Integer]
+    attribute(:stream_revision)
+    # @!attribute starting_position
+    #   @return [Integer]
+    attribute(:starting_position)
+  end
+end

data/lib/pg_eventstore/subscriptions/callback_handlers/events_processor_handlers.rb

@@ -6,25 +6,13 @@ module PgEventstore
     include Extensions::CallbackHandlersExtension
 
     class << self
+      # @param consumer [PgEventstore::EventsProcessorConsumer]
       # @param callbacks [PgEventstore::Callbacks]
-      # @param handler [#call]
-      # @param raw_events [Array<Hash>]
-      # @param raw_events_cond [MonitorMixin::ConditionVariable]
+      # @param events_repository [PgEventstore::Chunks::Repository]
+      # @param repository_cond [MonitorMixin::ConditionVariable]
       # @return [void]
-      def process_event(callbacks, handler, raw_events, raw_events_cond)
-        raw_event = nil
-        raw_events.synchronize do
-          raw_events_cond.wait(0.5) if raw_events.empty?
-          raw_event = raw_events.shift
-        end
-        return if raw_event.nil?
-
-        callbacks.run_callbacks(:process, Utils.original_global_position(raw_event)) do
-          handler.call(raw_event)
-        rescue => exception
-          raw_events.unshift(raw_event)
-          raise Utils.wrap_exception(exception, global_position: Utils.original_global_position(raw_event))
-        end
+      def consume_events(consumer, callbacks, events_repository, repository_cond)
+        consumer.call(callbacks, events_repository, repository_cond)
       end
 
       # @param callbacks [PgEventstore::Callbacks]

data/lib/pg_eventstore/subscriptions/callback_handlers/events_subscription_position_worker_handlers.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class EventsSubscriptionPositionWorkerHandlers
+    include Extensions::CallbackHandlersExtension
+
+    class << self
+      # @param event_subscription_position_queries [PgEventstore::EventSubscriptionPositionQueries]
+      # @param update_interval [Integer, Float]
+      # @return [void]
+      def assign_subscription_position(event_subscription_position_queries, update_interval)
+        affected_records = event_subscription_position_queries.assign_subscription_position
+        # In case if assigning subscription position process is locked by someone else or if we updated less than
+        # MAX_INDEX_RECORDS_TO_UPDATE_SUBSCRIPTION_POSITION number of records(which basically means we are on the edge
+        # of the table) - take some delay before the next attempt.
+        if affected_records.nil? ||
+           affected_records < EventSubscriptionPositionQueries::MAX_INDEX_RECORDS_TO_UPDATE_SUBSCRIPTION_POSITION
+          sleep update_interval
+        end
+      end
+
+      # @param event_subscription_position_queries [PgEventstore::EventSubscriptionPositionQueries]
+      # @param next_reindex_at [PgEventstore::EventsSubscriptionPositionWorker::ReindexTime]
+      # @return [void]
+      def reindex(event_subscription_position_queries, next_reindex_at)
+        return if next_reindex_at.time && next_reindex_at.time > Time.now
+
+        time = event_subscription_position_queries.reindex_unprocessed_positions
+        return unless time
+
+        next_reindex_at.time = time
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/callback_handlers/subscription_feeder_handlers.rb

@@ -85,6 +85,18 @@ module PgEventstore
           last_restarted_at: Time.now.utc, restart_count: subscriptions_set.restart_count + 1
         )
       end
+
+      # @param worker [PgEventstore::EventsSubscriptionPositionWorker]
+      # @return [void]
+      def start_events_subscription_position_worker(worker)
+        worker.start
+      end
+
+      # @param worker [PgEventstore::EventsSubscriptionPositionWorker]
+      # @return [void]
+      def stop_events_subscription_position_worker(worker)
+        worker.stop_async.wait_for_finish
+      end
     end
   end
 end

data/lib/pg_eventstore/subscriptions/callback_handlers/subscription_runner_handlers.rb

@@ -9,20 +9,22 @@ module PgEventstore
       # @param stats [PgEventstore::SubscriptionHandlerPerformance]
       # @param action [Proc]
       # @param _current_position [Integer]
+      # @param events_number [Integer]
       # @return [void]
-      def track_exec_time(stats, action, _current_position)
-        stats.track_exec_time { action.call }
+      def track_exec_time(stats, action, _current_position, events_number)
+        stats.track_exec_time(events_number) { action.call }
       end
 
       # @param subscription [PgEventstore::Subscription]
       # @param stats [PgEventstore::SubscriptionHandlerPerformance]
       # @param current_position [Integer]
+      # @param events_number [Integer]
       # @return [void]
-      def update_subscription_stats(subscription, stats, current_position)
+      def update_subscription_stats(subscription, stats, current_position, events_number)
         subscription.update(
           average_event_processing_time: stats.average_event_processing_time,
           current_position:,
-          total_processed_events: subscription.total_processed_events + 1
+          total_processed_events: subscription.total_processed_events + events_number
         )
       end
 
@@ -34,10 +36,12 @@ module PgEventstore
       end
 
       # @param subscription [PgEventstore::Subscription]
-      # @param global_position [Integer]
+      # @param subscription_position [Integer]
       # @return [void]
-      def update_subscription_chunk_stats(subscription, global_position)
-        subscription.update(last_chunk_fed_at: Time.now.utc, last_chunk_greatest_position: global_position)
+      def update_subscription_chunk_stats(subscription, subscription_position)
+        return if subscription.last_chunk_greatest_position == subscription_position
+
+        subscription.update(last_chunk_fed_at: Time.now.utc, last_chunk_greatest_position: subscription_position)
       end
 
       # @param subscription [PgEventstore::Subscription]
@@ -52,6 +56,16 @@ module PgEventstore
       def update_subscription_state(subscription, state)
         subscription.update(state:)
       end
+
+      # @param subscription [PgEventstore::Subscription]
+      # @param global_position [Integer]
+      # @return [void]
+      def checkpoint(subscription, global_position)
+        return if subscription.current_position == subscription.last_chunk_greatest_position &&
+                  subscription.current_position == global_position
+
+        subscription.update(current_position: global_position, last_chunk_greatest_position: global_position)
+      end
     end
   end
 end

data/lib/pg_eventstore/subscriptions/events_processor.rb

@@ -10,14 +10,16 @@ module PgEventstore
     def_delegators :@basic_runner, :state, :start, :stop, :wait_for_finish, :stop_async, :restore, :running?,
                    :within_state
 
-    # @param handler [#call]
     # @param graceful_shutdown_timeout [Integer, Float] seconds. Determines how long to wait before force-shutdown
     #   the runner when stopping it using #stop_async
+    # @param consumer [PgEventstore::EventsProcessorConsumer]
+    # @param events_repository [PgEventstore::Chunks::Repository]
     # @param recovery_strategies [Array<PgEventstore::RunnerRecoveryStrategy>]
-    def initialize(handler, graceful_shutdown_timeout:, recovery_strategies: [])
-      @handler = handler
-      @raw_events = SynchronizedArray.new
-      @raw_events_cond = @raw_events.new_cond
+    def initialize(graceful_shutdown_timeout:, consumer:, events_repository: Chunks::Repository.new,
+                   recovery_strategies: [])
+      @consumer = consumer
+      @events_repository = events_repository
+      @repository_cond = @events_repository.new_cond
       @basic_runner = BasicRunner.new(
         run_interval: 0,
         async_shutdown_time: graceful_shutdown_timeout,
@@ -26,29 +28,27 @@ module PgEventstore
       attach_runner_callbacks
     end
 
-    # @param raw_events [Array<Hash>]
+    # @param chunk [PgEventstore::Chunks::Chunk]
     # @return [void]
-    def feed(raw_events)
-      raise EmptyChunkFedError.new('Empty chunk was fed!') if raw_events.empty?
+    def feed(chunk)
+      raise EmptyChunkFedError.new('Empty chunk was fed!') if chunk.drained?
 
       within_state(:running) do
-        callbacks.run_callbacks(:feed, Utils.original_global_position(raw_events.last))
-        @raw_events.synchronize do
-          @raw_events.push(*raw_events)
-          @raw_events_cond.broadcast
-        end
+        callbacks.run_callbacks(:feed, chunk.last.subscription_position)
+        @events_repository.add_chunk(chunk, condition: @repository_cond)
       end
     end
 
     # Number of unprocessed events which are currently in a queue
     # @return [Integer]
-    def events_left_in_chunk
-      @raw_events.size
+    def events_left_in_repo
+      @events_repository.size
     end
 
     # @return [void]
-    def clear_chunk
-      @raw_events.clear
+    def clear_events_repository
+      @events_repository.clear
+      @consumer.clear_unprocessed_events
     end
 
     private
@@ -56,7 +56,9 @@ module PgEventstore
     def attach_runner_callbacks
       @basic_runner.define_callback(
         :process_async, :before,
-        EventsProcessorHandlers.setup_handler(:process_event, @callbacks, @handler, @raw_events, @raw_events_cond)
+        EventsProcessorHandlers.setup_handler(
+          :consume_events, @consumer, @callbacks, @events_repository, @repository_cond
+        )
       )
 
       @basic_runner.define_callback(

data/lib/pg_eventstore/subscriptions/events_processor_consumer/multiple.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module EventsProcessorConsumer
+    # @!visibility private
+    class Multiple
+      include EventsProcessorConsumer
+
+      # @return [Float, Integer]
+      EVENTS_WAIT_TIMEOUT = 0.5
+
+      class << self
+        # @param handler [#call]
+        # @param deserializer [PgEventstore::EventDeserializer]
+        # @return [PgEventstore::EventsProcessorConsumer::Multiple]
+        def create_consumer(handler, deserializer)
+          raw_handler = ->(raw_events) { handler.call(raw_events.map(&deserializer.method(:deserialize))) }
+          new(raw_handler)
+        end
+      end
+
+      # @param handler [#call]
+      def initialize(handler)
+        @handler = handler
+        @last_unprocessed_events = nil
+      end
+
+      # @return [void]
+      def clear_unprocessed_events
+        @last_unprocessed_events = nil
+      end
+
+      # @param callbacks [PgEventstore::Callbacks]
+      # @param events_repository [PgEventstore::Chunks::Repository]
+      # @param repository_cond [MonitorMixin::ConditionVariable]
+      # @return [void]
+      def call(callbacks, events_repository, repository_cond)
+        events_to_process =
+          @last_unprocessed_events ||
+          events_repository.wait_and_consume(
+            entities_num: nil, timeout: EVENTS_WAIT_TIMEOUT, condition: repository_cond
+          )
+        return if events_to_process.empty?
+
+        if events_to_process.first.is_a?(Chunks::SubscriptionCheckpointChunk::Checkpoint)
+          callbacks.run_callbacks(:checkpoint, events_to_process.first.subscription_position)
+          return
+        end
+
+        callbacks.run_callbacks(:process, events_to_process.last.subscription_position, events_to_process.size) do
+          @handler.call(events_to_process.map(&:attributes))
+        rescue => exception
+          @last_unprocessed_events = events_to_process
+          raise Utils.wrap_exception(
+            exception, subscription_positions: events_to_process.map(&:subscription_position)
+          )
+        else
+          clear_unprocessed_events
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/events_processor_consumer/replica.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module EventsProcessorConsumer
+    # @!visibility private
+    class Replica
+      include EventsProcessorConsumer
+
+      # @return [Float, Integer]
+      EVENTS_WAIT_TIMEOUT = 0.5
+
+      class << self
+        # @param config_name [Symbol]
+        # @param replica_config_name [Symbol]
+        # @return [PgEventstore::EventsProcessorConsumer::Replica]
+        def create_consumer(config_name, replica_config_name)
+          handler = ReplicaSubscriptionHandler.new(config_name, replica_config_name)
+          raw_handler = ->(events) { handler.call(events) }
+          new(raw_handler)
+        end
+      end
+
+      # @param handler [#call]
+      def initialize(handler)
+        @handler = handler
+        @last_unprocessed_events = nil
+      end
+
+      # @return [void]
+      def clear_unprocessed_events
+        @last_unprocessed_events = nil
+      end
+
+      # @param callbacks [PgEventstore::Callbacks]
+      # @param events_repository [PgEventstore::Chunks::Repository]
+      # @param repository_cond [MonitorMixin::ConditionVariable]
+      # @return [void]
+      def call(callbacks, events_repository, repository_cond)
+        events_to_process =
+          @last_unprocessed_events ||
+          events_repository.wait_and_consume(
+            entities_num: nil, timeout: EVENTS_WAIT_TIMEOUT, condition: repository_cond
+          )
+        return if events_to_process.empty?
+
+        if events_to_process.first.is_a?(Chunks::SubscriptionCheckpointChunk::Checkpoint)
+          callbacks.run_callbacks(:checkpoint, events_to_process.first.subscription_position)
+          return
+        end
+
+        callbacks.run_callbacks(:process, events_to_process.last.subscription_position, events_to_process.size) do
+          @handler.call(events_to_process)
+        rescue => exception
+          @last_unprocessed_events = events_to_process
+          raise Utils.wrap_exception(
+            exception,
+            subscription_position_range: [
+              events_to_process.first.subscription_position,
+              events_to_process.last.subscription_position,
+            ]
+          )
+        else
+          clear_unprocessed_events
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/events_processor_consumer/single.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module EventsProcessorConsumer
+    # @!visibility private
+    class Single
+      include EventsProcessorConsumer
+
+      # @return [Float, Integer]
+      EVENT_WAIT_TIMEOUT = 0.5
+
+      class << self
+        # @param handler [#call]
+        # @param deserializer [PgEventstore::EventDeserializer]
+        # @return [PgEventstore::EventsProcessorConsumer::Single]
+        def create_consumer(handler, deserializer)
+          raw_handler = ->(raw_event) { handler.call(deserializer.deserialize(raw_event)) }
+          new(raw_handler)
+        end
+      end
+
+      # @param handler [#call]
+      def initialize(handler)
+        @handler = handler
+        @last_unprocessed_event = nil
+      end
+
+      # @return [void]
+      def clear_unprocessed_events
+        @last_unprocessed_event = nil
+      end
+
+      # @param callbacks [PgEventstore::Callbacks]
+      # @param events_repository [PgEventstore::Chunks::Repository]
+      # @param repository_cond [MonitorMixin::ConditionVariable]
+      # @return [void]
+      def call(callbacks, events_repository, repository_cond)
+        raw_event =
+          @last_unprocessed_event ||
+          events_repository.wait_and_consume(
+            entities_num: 1, timeout: EVENT_WAIT_TIMEOUT, condition: repository_cond
+          ).first
+
+        return if raw_event.nil?
+
+        if raw_event.is_a?(Chunks::SubscriptionCheckpointChunk::Checkpoint)
+          callbacks.run_callbacks(:checkpoint, raw_event.subscription_position)
+          return
+        end
+
+        callbacks.run_callbacks(:process, raw_event.subscription_position, 1) do
+          @handler.call(raw_event.attributes)
+        rescue => exception
+          @last_unprocessed_event = raw_event
+          raise Utils.wrap_exception(exception, global_position: raw_event.subscription_position)
+        else
+          clear_unprocessed_events
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/events_processor_consumer.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  module EventsProcessorConsumer
+    class << self
+      # @param in_batches [Boolean]
+      # @return [Class]
+      def consumer_class(in_batches)
+        return Multiple if in_batches
+
+        Single
+      end
+
+      def included(othermod)
+        othermod.extend(ClassMethods)
+        super
+      end
+    end
+
+    module ClassMethods
+      def create_consumer(handler, deserializer)
+        raise NotImplementedError
+      end
+    end
+
+    # @param callbacks [PgEventstore::Callbacks]
+    # @param repository [PgEventstore::Chunks::Repository]
+    # @param repository_cond [MonitorMixin::ConditionVariable]
+    # @return [void]
+    def call(callbacks, repository, repository_cond)
+      raise NotImplementedError
+    end
+
+    # @return [void]
+    def clear_unprocessed_events
+      raise NotImplementedError
+    end
+  end
+end
+
+require_relative 'events_processor_consumer/single'
+require_relative 'events_processor_consumer/multiple'
+require_relative 'events_processor_consumer/replica'

data/lib/pg_eventstore/subscriptions/events_subscription_position_worker.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class EventsSubscriptionPositionWorker
+    extend Forwardable
+
+    attr_reader :config_name
+
+    def_delegators :@basic_runner, :start, :stop, :state, :stop_async, :wait_for_finish
+
+    class ReindexTime
+      include Extensions::OptionsExtension
+      include Extensions::OptionsDefaults
+
+      option(:time)
+    end
+
+    # @param config_name [Symbol]
+    def initialize(config_name)
+      @config_name = config_name
+      @basic_runner = BasicRunner.new(
+        run_interval: 0,
+        async_shutdown_time: 5,
+        recovery_strategies: [RunnerRecoveryStrategies::RestoreConnection.new(config_name)]
+      )
+      @next_reindex_at = ReindexTime.new
+      attach_runner_callbacks
+    end
+
+    private
+
+    # @return [void]
+    def attach_runner_callbacks
+      @basic_runner.define_callback(
+        :process_async, :before,
+        EventsSubscriptionPositionWorkerHandlers.setup_handler(
+          :assign_subscription_position,
+          event_subscription_position_queries,
+          config.events_subscription_position_update_interval
+        )
+      )
+      @basic_runner.define_callback(
+        :process_async, :before,
+        EventsSubscriptionPositionWorkerHandlers.setup_handler(
+          :reindex,
+          event_subscription_position_queries,
+          @next_reindex_at
+        )
+      )
+    end
+
+    # @return [PgEventstore::EventSubscriptionPositionQueries]
+    def event_subscription_position_queries
+      EventSubscriptionPositionQueries.new(connection)
+    end
+
+    # @return [PgEventstore::Connection]
+    def connection
+      PgEventstore.connection(config_name)
+    end
+
+    # @return [PgEventstore::Config]
+    def config
+      PgEventstore.config(config_name)
+    end
+  end
+end