pg_eventstore 2.0.0 → 3.0.0

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/replication_strategy.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module SubscriptionFeedStrategy
+    # @!visibility private
+    class ReplicationStrategy
+      include SubscriptionFeedStrategy
+
+      # Allow subscriptions to scan through up to this amount of events per a single query. This allows to make query
+      # plan more predictable.
+      # @return [Integer]
+      INDEX_LOOK_UP_DISTANCE = 100_000
+
+      # @param connection [PgEventstore::Connection]
+      # @param query_strategy [PgEventstore::QueryStrategy]
+      def initialize(connection, query_strategy)
+        @connection = connection
+        @query_strategy = query_strategy
+        @runners = []
+      end
+
+      # @param runners [Array<PgEventstore::ReplicaSubscriptionRunner>]
+      # @return [Array<PgEventstore::ReplicaSubscriptionRunner>]
+      def add(*runners)
+        @runners.push(*runners)
+      end
+
+      # @return [Integer]
+      def size
+        @runners.size
+      end
+
+      # @return [Boolean]
+      def any?
+        @runners.any?
+      end
+
+      # @return [void]
+      def feed
+        runners_query_options = @runners.to_h do |runner|
+          next_chunk_query_opts = runner.next_chunk_query_opts
+          next_chunk_query_opts[:to_position] =
+            [next_chunk_query_opts[:from_position] + INDEX_LOOK_UP_DISTANCE, safe_position].min
+          [runner.id, next_chunk_query_opts]
+        end
+        grouped_indexes = events_global_index_queries.fetch_indexes_for_subscriptions(runners_query_options)
+        @runners.each do |runner|
+          if grouped_indexes[runner.id]
+            chunk = Chunks::ReplicaEventsIndexChunk.new(grouped_indexes[runner.id])
+            runner.feed(chunk)
+          else
+            runner.feed(Chunks::SubscriptionCheckpointChunk.new(runners_query_options[runner.id][:to_position]))
+          end
+        end
+      end
+
+      private
+
+      # @return [Integer]
+      def safe_position
+        event_subscription_position_queries.max_subscription_position || 0
+      end
+
+      # @return [PgEventstore::EventsGlobalIndexQueries]
+      def events_global_index_queries
+        EventsGlobalIndexQueries.new(@connection, @query_strategy)
+      end
+
+      # @return [PgEventstore::EventSubscriptionPositionQueries]
+      def event_subscription_position_queries
+        EventSubscriptionPositionQueries.new(@connection)
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_feed_strategy.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  module SubscriptionFeedStrategy
+    # @param runners [PgEventstore::SubscriptionRunner, Array<PgEventstore::SubscriptionRunner>]
+    # @return [Array<PgEventstore::SubscriptionRunner>]
+    def add(*runners)
+      raise NotImplementedError
+    end
+
+    # @return [Integer]
+    def size
+      raise NotImplementedError
+    end
+
+    # @return [Boolean]
+    def any?
+      raise NotImplementedError
+    end
+
+    # @return [void]
+    def feed
+      raise NotImplementedError
+    end
+  end
+end
+
+require_relative 'subscription_feed_strategies/collection'
+require_relative 'subscription_feed_strategies/index_read_strategy'
+require_relative 'subscription_feed_strategies/replication_strategy'

data/lib/pg_eventstore/subscriptions/subscription_feeder.rb

@@ -30,6 +30,9 @@ module PgEventstore
       @subscriptions_set_lifecycle = subscriptions_set_lifecycle
       @subscriptions_lifecycle = subscriptions_lifecycle
       @commands_handler = CommandsHandler.new(@config_name, self, @subscriptions_lifecycle.runners)
+      if requires_subscription_position_assignment?
+        @events_subscription_position_worker = EventsSubscriptionPositionWorker.new(@config_name)
+      end
       attach_runner_callbacks
     end
 
@@ -77,6 +80,14 @@ module PgEventstore
         :before_runner_started, :before,
         SubscriptionFeederHandlers.setup_handler(:start_cmds_handler, @commands_handler)
       )
+      if requires_subscription_position_assignment?
+        @basic_runner.define_callback(
+          :before_runner_started, :before,
+          SubscriptionFeederHandlers.setup_handler(
+            :start_events_subscription_position_worker, @events_subscription_position_worker
+          )
+        )
+      end
 
       @basic_runner.define_callback(
         :after_runner_died, :before,
@@ -108,6 +119,14 @@ module PgEventstore
         :after_runner_stopped, :before,
         SubscriptionFeederHandlers.setup_handler(:stop_commands_handler, @commands_handler)
       )
+      if requires_subscription_position_assignment?
+        @basic_runner.define_callback(
+          :after_runner_stopped, :before,
+          SubscriptionFeederHandlers.setup_handler(
+            :stop_events_subscription_position_worker, @events_subscription_position_worker
+          )
+        )
+      end
 
       @basic_runner.define_callback(
         :before_runner_restored, :after,
@@ -126,5 +145,10 @@ module PgEventstore
         ),
       ]
     end
+
+    # @return [Boolean]
+    def requires_subscription_position_assignment?
+      Config::NodeRole.writable.include?(PgEventstore.config(@config_name).eventstore_role)
+    end
   end
 end

data/lib/pg_eventstore/subscriptions/subscription_handler_performance.rb

@@ -8,7 +8,7 @@ module PgEventstore
     extend Forwardable
 
     # @return [Integer] the number of measurements to keep
-    TIMINGS_TO_KEEP = 100
+    TIMINGS_TO_KEEP = 10
 
     def_delegators :@lock, :synchronize
 
@@ -18,10 +18,12 @@ module PgEventstore
     end
 
     # Yields the given block to measure its execution time
+    # @param events_number [Integer] number of events processed by the given block
     # @return [Object] the result of yielded block
-    def track_exec_time
+    def track_exec_time(events_number)
       result = nil
       time = Utils.benchmark { result = yield }
+      time /= events_number
       synchronize do
         @timings.shift if @timings.size == TIMINGS_TO_KEEP
         @timings.push(time)

data/lib/pg_eventstore/subscriptions/subscription_runner.rb

@@ -11,37 +11,43 @@ module PgEventstore
     extend Forwardable
 
     # @return [Integer]
-    MAX_EVENTS_PER_CHUNK = 1_000
+    DEFAULT_MAX_EVENTS_PER_CHUNK = 1_000
     # @return [Integer]
     MIN_EVENTS_PER_CHUNK = 10
     # @return [Integer]
-    INITIAL_EVENTS_PER_CHUNK = 10
+    DEFAULT_INITIAL_EVENTS_PER_CHUNK = 10
 
     # @!attribute subscription
     #   @return [PgEventstore::Subscription]
     attr_reader :subscription
 
     def_delegators :@events_processor, :start, :stop, :stop_async, :feed, :wait_for_finish, :restore, :state, :running?,
-                   :clear_chunk, :within_state
+                   :clear_events_repository, :within_state
     def_delegators :@subscription, :lock!, :id
 
     # @param stats [PgEventstore::SubscriptionHandlerPerformance]
     # @param events_processor [PgEventstore::EventsProcessor]
     # @param subscription [PgEventstore::Subscription]
-    def initialize(stats:, events_processor:, subscription:)
+    # @param max_events_per_chunk [Integer]
+    def initialize(stats:, events_processor:, subscription:,
+                   initial_events_per_chunk: DEFAULT_INITIAL_EVENTS_PER_CHUNK,
+                   max_events_per_chunk: DEFAULT_MAX_EVENTS_PER_CHUNK)
       @stats = stats
       @events_processor = events_processor
       @subscription = subscription
+      @max_events_per_chunk = max_events_per_chunk
+      @initial_events_per_chunk = initial_events_per_chunk
 
       attach_callbacks
     end
 
     # @return [Hash]
     def next_chunk_query_opts
-      @subscription.options.merge(
+      {
         from_position: next_chunk_global_position,
-        max_count: estimate_events_number
-      )
+        max_count: estimate_events_number,
+        resolve_link_tos: @subscription.options[:resolve_link_tos] || false,
+      }
     end
 
     # @return [Boolean]
@@ -61,10 +67,10 @@ module PgEventstore
 
     # @return [Integer]
     def estimate_events_number
-      return INITIAL_EVENTS_PER_CHUNK if @stats.average_event_processing_time == 0
+      return @initial_events_per_chunk if @stats.average_event_processing_time == 0
 
       events_per_chunk = (@subscription.chunk_query_interval / @stats.average_event_processing_time).round
-      events_to_fetch = [events_per_chunk, MAX_EVENTS_PER_CHUNK].min - @events_processor.events_left_in_chunk
+      events_to_fetch = [events_per_chunk, @max_events_per_chunk].min - @events_processor.events_left_in_repo
       return 0 if events_to_fetch < 0 # We still have a lot of events in the chunk - no need to fetch more
 
       [events_to_fetch, MIN_EVENTS_PER_CHUNK].max
@@ -91,6 +97,11 @@ module PgEventstore
         SubscriptionRunnerHandlers.setup_handler(:update_subscription_chunk_stats, @subscription)
       )
 
+      @events_processor.define_callback(
+        :checkpoint, :after,
+        SubscriptionRunnerHandlers.setup_handler(:checkpoint, @subscription)
+      )
+
       @events_processor.define_callback(
         :restart, :after,
         SubscriptionRunnerHandlers.setup_handler(:update_subscription_restarts, @subscription)

data/lib/pg_eventstore/subscriptions/subscription_runner_commands/reset_position.rb

@@ -16,7 +16,7 @@ module PgEventstore
       # @return [void]
       def exec_cmd(subscription_runner)
         subscription_runner.within_state(:stopped) do
-          subscription_runner.clear_chunk
+          subscription_runner.clear_events_repository
           subscription_runner.subscription.update(
             last_chunk_greatest_position: nil, current_position: data['position'], total_processed_events: 0
           )

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -15,15 +15,16 @@ module PgEventstore
       runners = runners.select(&:running?).select(&:time_to_feed?)
       return if runners.empty?
 
-      safe_pos = subscription_service_queries.safe_global_position
-      runners_query_options = runners.to_h do |runner|
-        [runner.id, runner.next_chunk_query_opts.merge(to_position: safe_pos)]
-      end
-      grouped_events = subscription_queries.subscriptions_events(runners_query_options)
-
-      runners.each do |runner|
-        runner.feed(grouped_events[runner.id]) if grouped_events[runner.id]
+      feed_strategies_collection = SubscriptionFeedStrategy::Collection.create(
+        runners,
+        connection,
+        QueryStrategy::Async.new(connection)
+      )
+      query_runner = AsyncRunner.new
+      feed_strategies_collection.each do |strategy|
+        query_runner.async { strategy.feed }
       end
+      query_runner.run
     end
 
     private
@@ -32,15 +33,5 @@ module PgEventstore
     def connection
       PgEventstore.connection(@config_name)
     end
-
-    # @return [PgEventstore::SubscriptionQueries]
-    def subscription_queries
-      SubscriptionQueries.new(connection)
-    end
-
-    # @return [PgEventstore::SubscriptionServiceQueries]
-    def subscription_service_queries
-      SubscriptionServiceQueries.new(connection)
-    end
   end
 end

data/lib/pg_eventstore/subscriptions/subscriptions_lifecycle.rb

@@ -56,18 +56,6 @@ module PgEventstore
       @runners.map(&:subscription)
     end
 
-    # Sets the force_lock flag to true. If set - all related Subscriptions will ignore their lock state and will be
-    # locked by the new SubscriptionsSet.
-    # @return [void]
-    def force_lock!
-      message = <<~TEXT
-        Usage of #force_lock! is deprecated and will be removed in v2. Please pass :force_lock keyword argument when \
-        initializing SubscriptionsManager. Example: PgEventstore.subscriptions_manager(force_lock: true)
-      TEXT
-      Utils.deprecation_warning(message)
-      @force_lock = true
-    end
-
     # @return [Boolean]
     def force_locked?
       @force_lock

data/lib/pg_eventstore/subscriptions/subscriptions_manager.rb

@@ -6,18 +6,22 @@ require_relative 'basic_runner'
 require_relative 'runner_recovery_strategy'
 require_relative 'runner_recovery_strategies'
 require_relative 'subscription'
-require_relative 'synchronized_array'
+require_relative 'events_processor_consumer'
 require_relative 'events_processor'
 require_relative 'subscription_handler_performance'
 require_relative 'subscription_runner'
 require_relative 'subscriptions_set'
+require_relative 'subscription_feed_strategy'
 require_relative 'subscription_runners_feeder'
 require_relative 'subscriptions_set_lifecycle'
 require_relative 'subscriptions_lifecycle'
 require_relative 'callback_handlers/subscription_feeder_handlers'
 require_relative 'callback_handlers/subscription_runner_handlers'
 require_relative 'callback_handlers/events_processor_handlers'
+require_relative 'callback_handlers/events_subscription_position_worker_handlers'
 require_relative 'callback_handlers/commands_handler_handlers'
+require_relative 'commands_handler'
+require_relative 'events_subscription_position_worker'
 require_relative 'subscription_feeder'
 require_relative 'extensions/command_class_lookup_extension'
 require_relative 'extensions/base_command_extension'
@@ -25,10 +29,11 @@ require_relative 'subscription_feeder_commands'
 require_relative 'subscription_runner_commands'
 require_relative 'queries/subscription_command_queries'
 require_relative 'queries/subscription_queries'
-require_relative 'queries/subscription_service_queries'
+require_relative 'queries/event_subscription_position_queries'
 require_relative 'queries/subscriptions_set_command_queries'
 require_relative 'queries/subscriptions_set_queries'
-require_relative 'commands_handler'
+require_relative 'replica_subscription_handler'
+require_relative 'replica_subscription_runner'
 
 module PgEventstore
   # The public Subscriptions API, available to the user.
@@ -41,7 +46,6 @@ module PgEventstore
     private :config
 
     def_delegators :@subscription_feeder, :stop, :running?
-    def_delegators :@subscriptions_lifecycle, :force_lock!
 
     # @param config [PgEventstore::Config]
     # @param set_name [String]
@@ -71,8 +75,8 @@ module PgEventstore
 
     # @param subscription_name [String] subscription's name
     # @param handler [#call] subscription's handler
-    # @param options [Hash] request options
-    # @option options [Integer, Symbol] :from_position a starting subscription position
+    # @param options [Hash] subscription options
+    # @option options [Integer] :from_position a starting subscription position
     # @option options [Boolean] :resolve_link_tos When using projections to create new events you
     #   can set whether the generated events are pointers to existing events. Setting this option to true tells
     #   PgEventstore to return the original event instead a link event.
@@ -91,6 +95,7 @@ module PgEventstore
     #   notification about failed subscription.
     # @param graceful_shutdown_timeout [integer, Float] the number of seconds to wait until force-shutdown the
     #   subscription during the stop process
+    # @param in_batches [Boolean] whether a batch of events should be yielded instead a single event
     # @return [void]
     def subscribe(subscription_name, handler:, options: {}, middlewares: nil,
                   pull_interval: config.subscription_pull_interval,
@@ -98,16 +103,18 @@ module PgEventstore
                   retries_interval: config.subscription_retries_interval,
                   restart_terminator: config.subscription_restart_terminator,
                   failed_subscription_notifier: config.failed_subscription_notifier,
-                  graceful_shutdown_timeout: config.subscription_graceful_shutdown_timeout)
+                  graceful_shutdown_timeout: config.subscription_graceful_shutdown_timeout,
+                  in_batches: false)
       subscription = Subscription.using_connection(config.name).new(
         set: @set_name, name: subscription_name, options:, chunk_query_interval: pull_interval,
         max_restarts_number: max_retries, time_between_restarts: retries_interval
       )
+      consumer_class = EventsProcessorConsumer.consumer_class(in_batches)
       runner = SubscriptionRunner.new(
         stats: SubscriptionHandlerPerformance.new,
         events_processor: EventsProcessor.new(
-          create_raw_event_handler(middlewares, handler),
           graceful_shutdown_timeout:,
+          consumer: consumer_class.create_consumer(handler, deserializer(middlewares)),
           recovery_strategies: recovery_strategies(subscription, restart_terminator, failed_subscription_notifier)
         ),
         subscription:
@@ -117,6 +124,63 @@ module PgEventstore
       true
     end
 
+    # @param subscription_name [String] subscription's name. For example, you can name it after the target database:
+    #   "Target: my_eventstore_replica1"
+    # @param replica_config_name [Symbol] the destination replica database config to replicate events into. You have to
+    #   have it preconfigured with PgEventstore.configure(). Example:
+    #   PgEventstore.configure(name: :my_eventstore_replica1) do |config|
+    #     config.pg_uri = 'postgresql://postgres:postgres@localhost:5432/my_eventstore_replica1'
+    #   end
+    #   Refer to the docs/configuration.md for more info
+    # @param options [Hash] subscription options
+    # @option options [Integer] :from_position a starting subscription position
+    # @option options [Hash] :filter provide it to filter events. It works the same way as a :filter option of
+    #   {PgEventstore::Client#read} method. Use this option when you want to replicate certain set of events.
+    # @param max_events_to_replicate [Integer] set the upper limit of how many events the handler can copy per turn
+    # @param pull_interval [Integer, Float] an interval in seconds to determine how often to query new events of the
+    #   given subscription.
+    # @param max_retries [Integer] max number of retries of failed Subscription
+    # @param retries_interval [Integer, Float] a delay between retries of failed Subscription
+    # @param restart_terminator [#call, nil] a callable object which is invoked with PgEventstore::Subscription instance
+    #   to determine whether restarts should be stopped(true - stops restarts, false - continues restarts)
+    # @param failed_subscription_notifier [#call, nil] a callable object which is invoked with
+    #   PgEventstore::Subscription instance and error instance after the related subscription died due to error and no
+    #   longer can be automatically restarted due to max retries number reached. You can use this hook to send a
+    #   notification about failed subscription.
+    # @param graceful_shutdown_timeout [Integer, Float] the number of seconds to wait until force-shutdown the
+    #   subscription during the stop process
+    # @return [void]
+    def create_replication(subscription_name,
+                           replica_config_name,
+                           options: {},
+                           max_events_to_replicate: config.max_events_to_replicate,
+                           pull_interval: config.subscription_pull_interval,
+                           max_retries: config.subscription_max_retries,
+                           retries_interval: config.subscription_retries_interval,
+                           restart_terminator: config.subscription_restart_terminator,
+                           failed_subscription_notifier: config.failed_subscription_notifier,
+                           graceful_shutdown_timeout: config.subscription_graceful_shutdown_timeout)
+      Utils.assert_node_role!(config, [Config::NodeRole::PRIMARY])
+      subscription = Subscription.using_connection(config.name).new(
+        set: @set_name, name: subscription_name, options:, chunk_query_interval: pull_interval,
+        max_restarts_number: max_retries, time_between_restarts: retries_interval
+      )
+      runner = ReplicaSubscriptionRunner.new(
+        stats: SubscriptionHandlerPerformance.new,
+        events_processor: EventsProcessor.new(
+          graceful_shutdown_timeout:,
+          consumer: EventsProcessorConsumer::Replica.create_consumer(config_name, replica_config_name),
+          recovery_strategies: recovery_strategies(subscription, restart_terminator, failed_subscription_notifier)
+        ),
+        subscription:,
+        max_events_per_chunk: max_events_to_replicate,
+        initial_events_per_chunk: max_events_to_replicate
+      )
+
+      @subscriptions_lifecycle.runners.push(runner)
+      true
+    end
+
     # @return [Array<PgEventstore::Subscription>]
     def subscriptions
       @subscriptions_lifecycle.subscriptions.map(&:dup)
@@ -157,12 +221,8 @@ module PgEventstore
       PgEventstore::CLI.callbacks.run_callbacks(:start_manager, self, &)
     end
 
-    # @param middlewares [Array<Symbol>, nil]
-    # @param handler [#call]
-    # @return [Proc]
-    def create_raw_event_handler(middlewares, handler)
-      deserializer = EventDeserializer.new(select_middlewares(middlewares), config.event_class_resolver)
-      ->(raw_event) { handler.call(deserializer.deserialize(raw_event)) }
+    def deserializer(middlewares)
+      EventDeserializer.new(select_middlewares(middlewares), config.event_class_resolver)
     end
 
     # @param middlewares [Array, nil]

data/lib/pg_eventstore/{subscriptions/synchronized_array.rb → synchronized_array.rb}

@@ -13,6 +13,9 @@ module PgEventstore
     alias old_clear clear
     alias old_size size
     alias old_empty? empty?
+    alias old_slice slice
+    alias old_slice! slice!
+    alias old_at at
 
     def shift(...)
       synchronize { old_shift(...) }
@@ -37,5 +40,17 @@ module PgEventstore
     def empty?(...)
       synchronize { old_empty?(...) }
     end
+
+    def slice(...)
+      synchronize { old_slice(...) }
+    end
+
+    def slice!(...)
+      synchronize { old_slice!(...) }
+    end
+
+    def at(...)
+      synchronize { old_at(...) }
+    end
   end
 end

data/lib/pg_eventstore/tasks/setup.rake

@@ -46,7 +46,7 @@ namespace :pg_eventstore do
   end
 
   task :migrate do
-    migration_files_root = "#{Gem::Specification.find_by_name('pg_eventstore').gem_dir}/db/migrations"
+    migration_files_root = File.join(Gem.loaded_specs.fetch('pg_eventstore').gem_dir, 'db/migrations')
 
     PgEventstore.connection(:_eventstore_db_connection).with do |conn|
       conn.exec('CREATE TABLE IF NOT EXISTS migrations (number int NOT NULL)')
@@ -59,7 +59,7 @@ namespace :pg_eventstore do
           next if latest_migration >= number
 
           if File.extname(f_name) == '.rb'
-            load f_name
+            load(f_name, true)
           else
             conn.exec(File.read(f_name))
           end

data/lib/pg_eventstore/utils.rb

@@ -132,6 +132,37 @@ module PgEventstore
         yield
         Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
       end
+
+      # @param truthy [boolish]
+      # @param message [String, nil]
+      # @raise ArgumentError
+      def assert!(truthy, message = nil)
+        raise ArgumentError, message unless truthy
+      end
+
+      # @param obj
+      # @raise NotImplementedError
+      def missing_implementation!(obj)
+        raise NotImplementedError, obj.inspect
+      end
+
+      # @param config [PgEventstore::Config]
+      # @param expected_roles [Array<Symbol>]
+      # @return [void]
+      def assert_node_role!(config, expected_roles)
+        return if expected_roles.include?(config.eventstore_role)
+
+        raise "You can't perform this operation on #{config.eventstore_role.inspect} node!"
+      end
+
+      # Takes a collection of partition ids, the number of partitions per call and determines the position in the
+      # partition_ids at which we have max_partitions_to_resolve_per_call unique partitions
+      # @param partition_ids [Array<Integer>]
+      # @param max_partitions_to_resolve_per_call [Integer]
+      # @return [Range]
+      def range_to_slice(partition_ids, max_partitions_to_resolve_per_call)
+        # This is stub, used for indexing
+      end
     end
   end
 end

data/lib/pg_eventstore/version.rb

@@ -2,5 +2,5 @@
 
 module PgEventstore
   # @return [String]
-  VERSION = '2.0.0'
+  VERSION = '3.0.0'
 end