pg_eventstore 3.1.1 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9181ca6342afc352a02779065731ea21bb9e8ce6632b924a25342e9726e88f3c
-  data.tar.gz: 6c80c69802d7efcca8be4440d22f2dac1621f379cb710520d755335378a94725
+  metadata.gz: b54fbf71f1dd082e5895ab8b1431f2a92442c0d0865e656b4fe65430503151b5
+  data.tar.gz: 45a0bfd8a606466495643cb41f503643bd6151f915d1e36cdf374eded7dab8b3
 SHA512:
-  metadata.gz: a3d07d058fd23c8b6d19c09b2de3a5c600683205d6bd21df9a304dd0bc4e5428fd34f33dcbeea7cdd95415f8593fb99bac1801e1d08ce506c05b748d9f78ff78
-  data.tar.gz: c09027974c15266d0771c1375fbd7dedcc78841f7d36d9bb0c47f6f619b14922eb9d1561b9e1240a692c32aadef59d55ec3526479befab06e2c6c461e81d5680
+  metadata.gz: 59028bfa802198da04912ba0f339a165eb3d9c35e0cb940b9fb0437ae42909f3570b2f4bbda88d5234fb622420d4ff7425dd1fafa4a8951b4092879ca33fcf77
+  data.tar.gz: 671e391234a4f8b6a008b19bdbc533c94a7e3cc562fab9cdfd4dbef77968fd2ed03814f7ff132a8326aee859cc2e65db04ef63a8667e2e67b18615b102e08102

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [3.1.2]
+
+- Fix subscriptions skipping events when the estimated number of events to fetch drops to 0 during the pull
+
 ## [3.1.1]
 
 - Prevent arbitrary order in replica queries

data/lib/pg_eventstore/chunks/subscription_events_index_chunk.rb

@@ -64,9 +64,8 @@ module PgEventstore
 
       # @return [void]
       def resolve_indexes
-        indexes_to_resolve = @indexes.slice!(
-          Utils.range_to_slice(@indexes.map(&:event_type_partition_id), MAX_PARTITIONS_TO_RESOLVE_PER_CALL)
-        )
+        range = Utils.range_to_slice(@indexes.map(&:event_type_partition_id), MAX_PARTITIONS_TO_RESOLVE_PER_CALL)
+        indexes_to_resolve = @indexes[range]
         global_to_sub_position_map = indexes_to_resolve.to_h { [_1.global_position, _1.subscription_position] }
         raw_events = events_global_index_queries.resolve_indexes(
           indexes_to_resolve,
@@ -80,12 +79,10 @@ module PgEventstore
           )
         end
         raw_events = raw_events.sort_by(&:subscription_position)
+        # Remove indexes only after their events are resolved, so #size keeps counting them while the query runs
+        @indexes.slice!(range)
         @raw_events.push(*raw_events)
         @resolved = @indexes.empty?
-      rescue => exception
-        @indexes.unshift(*indexes_to_resolve)
-        @resolved = false
-        raise exception
       end
 
       # @return [PgEventstore::EventsGlobalIndexQueries]

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/collection.rb

@@ -9,23 +9,25 @@ module PgEventstore
       DEFAULT_SUBSCRIPTIONS_NUM_PER_QUERY = 10
 
       class << self
-        # @param runners [Array<PgEventstore::SubscriptionRunner>]
+        # @param runners_query_options [Hash{PgEventstore::SubscriptionRunner => Hash}]
         # @param connection [PgEventstore::Connection]
         # @param query_strategy [PgEventstore::QueryStrategy]
         # @param subscriptions_per_query [Integer]
         # @return [PgEventstore::SubscriptionFeedStrategy::Collection]
-        def create(runners, connection, query_strategy, subscriptions_per_query: DEFAULT_SUBSCRIPTIONS_NUM_PER_QUERY)
+        def create(runners_query_options, connection, query_strategy,
+                   subscriptions_per_query: DEFAULT_SUBSCRIPTIONS_NUM_PER_QUERY)
           instance = new
-          replica_runners = runners.grep(ReplicaSubscriptionRunner)
-          common_runners = runners - replica_runners
-          common_runners.each_slice(subscriptions_per_query).each do |runners_slice|
+          replica_runners, common_runners = runners_query_options.partition do |runner, _|
+            runner.is_a?(ReplicaSubscriptionRunner)
+          end
+          common_runners.each_slice(subscriptions_per_query) do |runners_slice|
             strategy = IndexReadStrategy.new(connection, query_strategy)
-            strategy.add(*runners_slice)
+            runners_slice.each { |runner, query_options| strategy.add(runner, query_options) }
             instance.push(strategy)
           end
           replica_runners.each_slice(subscriptions_per_query) do |runners_slice|
             strategy = ReplicationStrategy.new(connection, query_strategy)
-            strategy.add(*runners_slice)
+            runners_slice.each { |runner, query_options| strategy.add(runner, query_options) }
             instance.push(strategy)
           end
           instance

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/index_read_strategy.rb

@@ -18,45 +18,45 @@ module PgEventstore
       def initialize(connection, query_strategy)
         @connection = connection
         @query_strategy = query_strategy
-        @runners = []
+        @runners_query_options = {}
       end
 
-      # @param runners [Array<PgEventstore::SubscriptionRunner>]
-      # @return [Array<PgEventstore::SubscriptionRunner>]
-      def add(*runners)
-        @runners.push(*runners)
+      # @param runner [PgEventstore::SubscriptionRunner]
+      # @param query_options [Hash]
+      # @return [void]
+      def add(runner, query_options)
+        @runners_query_options[runner] = query_options
       end
 
       # @return [Integer]
       def size
-        @runners.size
+        @runners_query_options.size
       end
 
       # @return [Boolean]
       def any?
-        @runners.any?
+        @runners_query_options.any?
       end
 
       # @return [void]
       def feed
-        runners_query_options = @runners.to_h do |runner|
-          next_chunk_query_opts = runner.next_chunk_query_opts
-          next_chunk_query_opts[:to_position] =
-            [next_chunk_query_opts[:from_position] + INDEX_LOOK_UP_DISTANCE, safe_position].min
-          [runner.id, next_chunk_query_opts]
+        @runners_query_options.each_value do |query_options|
+          query_options[:to_position] = [query_options[:from_position] + INDEX_LOOK_UP_DISTANCE, safe_position].min
         end
-        grouped_indexes = events_global_index_queries.fetch_indexes_for_subscriptions(runners_query_options)
-        @runners.each do |runner|
+        grouped_indexes = events_global_index_queries.fetch_indexes_for_subscriptions(
+          @runners_query_options.transform_keys(&:id)
+        )
+        @runners_query_options.each do |runner, query_options|
           if grouped_indexes[runner.id]
             chunk = Chunks::SubscriptionEventsIndexChunk.new(
               grouped_indexes[runner.id],
               @connection,
               QueryStrategy::Foreground.new(@connection),
-              runners_query_options[runner.id][:resolve_link_tos]
+              query_options[:resolve_link_tos]
             )
             runner.feed(chunk)
           else
-            runner.feed(Chunks::SubscriptionCheckpointChunk.new(runners_query_options[runner.id][:to_position]))
+            runner.feed(Chunks::SubscriptionCheckpointChunk.new(query_options[:to_position]))
           end
         end
       end

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/replication_strategy.rb

@@ -16,40 +16,40 @@ module PgEventstore
       def initialize(connection, query_strategy)
         @connection = connection
         @query_strategy = query_strategy
-        @runners = []
+        @runners_query_options = {}
       end
 
-      # @param runners [Array<PgEventstore::ReplicaSubscriptionRunner>]
-      # @return [Array<PgEventstore::ReplicaSubscriptionRunner>]
-      def add(*runners)
-        @runners.push(*runners)
+      # @param runner [PgEventstore::ReplicaSubscriptionRunner]
+      # @param query_options [Hash]
+      # @return [void]
+      def add(runner, query_options)
+        @runners_query_options[runner] = query_options
       end
 
       # @return [Integer]
       def size
-        @runners.size
+        @runners_query_options.size
       end
 
       # @return [Boolean]
       def any?
-        @runners.any?
+        @runners_query_options.any?
       end
 
       # @return [void]
       def feed
-        runners_query_options = @runners.to_h do |runner|
-          next_chunk_query_opts = runner.next_chunk_query_opts
-          next_chunk_query_opts[:to_position] =
-            [next_chunk_query_opts[:from_position] + INDEX_LOOK_UP_DISTANCE, safe_position].min
-          [runner.id, next_chunk_query_opts]
+        @runners_query_options.each_value do |query_options|
+          query_options[:to_position] = [query_options[:from_position] + INDEX_LOOK_UP_DISTANCE, safe_position].min
         end
-        grouped_indexes = events_global_index_queries.fetch_indexes_for_subscriptions(runners_query_options)
-        @runners.each do |runner|
+        grouped_indexes = events_global_index_queries.fetch_indexes_for_subscriptions(
+          @runners_query_options.transform_keys(&:id)
+        )
+        @runners_query_options.each do |runner, query_options|
           if grouped_indexes[runner.id]
             chunk = Chunks::ReplicaEventsIndexChunk.new(grouped_indexes[runner.id])
             runner.feed(chunk)
           else
-            runner.feed(Chunks::SubscriptionCheckpointChunk.new(runners_query_options[runner.id][:to_position]))
+            runner.feed(Chunks::SubscriptionCheckpointChunk.new(query_options[:to_position]))
           end
         end
       end

data/lib/pg_eventstore/subscriptions/subscription_feed_strategy.rb

@@ -3,9 +3,10 @@
 module PgEventstore
   # @!visibility private
   module SubscriptionFeedStrategy
-    # @param runners [PgEventstore::SubscriptionRunner, Array<PgEventstore::SubscriptionRunner>]
-    # @return [Array<PgEventstore::SubscriptionRunner>]
-    def add(*runners)
+    # @param runner [PgEventstore::SubscriptionRunner]
+    # @param query_options [Hash]
+    # @return [void]
+    def add(runner, query_options)
       raise NotImplementedError
     end
 

data/lib/pg_eventstore/subscriptions/subscription_runner.rb

@@ -50,11 +50,6 @@ module PgEventstore
       }
     end
 
-    # @return [Boolean]
-    def time_to_feed?
-      estimate_events_number > 0 && @subscription.last_chunk_fed_at + @subscription.chunk_query_interval <= Time.now.utc
-    end
-
     private
 
     # @return [Integer]

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module PgEventstore
-  # This class pulls events from db and feeds given SubscriptionRunners
+  # This class decides which SubscriptionRunners to feed, pulls events from db and feeds them
   # @!visibility private
   class SubscriptionRunnersFeeder
     # @param config_name [Symbol]
@@ -12,11 +12,15 @@ module PgEventstore
     # @param runners [Array<PgEventstore::SubscriptionRunner>]
     # @return [void]
     def feed(runners)
-      runners = runners.select(&:running?).select(&:time_to_feed?)
-      return if runners.empty?
+      runners_query_options = runners.select { |runner| runner.running? && time_to_feed?(runner) }.to_h do |runner|
+        [runner, runner.next_chunk_query_opts]
+      end
+      # A runner which has enough events in its queue estimates the number of events to fetch to 0
+      runners_query_options = runners_query_options.select { |_, query_options| query_options[:max_count] > 0 }
+      return if runners_query_options.empty?
 
       feed_strategies_collection = SubscriptionFeedStrategy::Collection.create(
-        runners,
+        runners_query_options,
         connection,
         QueryStrategy::Async.new(connection)
       )
@@ -29,6 +33,13 @@ module PgEventstore
 
     private
 
+    # @param runner [PgEventstore::SubscriptionRunner]
+    # @return [Boolean]
+    def time_to_feed?(runner)
+      subscription = runner.subscription
+      subscription.last_chunk_fed_at + subscription.chunk_query_interval <= Time.now.utc
+    end
+
     # @return [PgEventstore::Connection]
     def connection
       PgEventstore.connection(@config_name)

data/lib/pg_eventstore/version.rb

@@ -2,5 +2,5 @@
 
 module PgEventstore
   # @return [String]
-  VERSION = '3.1.1'
+  VERSION = '3.1.2'
 end

data/sig/pg_eventstore/subscriptions/subscription_feed_strategy/collection.rbs

@@ -4,7 +4,7 @@ module PgEventstore
       DEFAULT_SUBSCRIPTIONS_NUM_PER_QUERY: Integer
 
       def self.create: (
-          Array[SubscriptionRunner | ReplicaSubscriptionRunner] runners,
+          Hash[SubscriptionRunner | ReplicaSubscriptionRunner, Hash[Symbol, untyped]] runners_query_options,
           Connection connection,
           QueryStrategy query_strategy,
           ?subscriptions_per_query: Integer

data/sig/pg_eventstore/subscriptions/subscription_feed_strategy/index_read_strategy.rbs

@@ -7,7 +7,7 @@ module PgEventstore
 
       @connection: Connection
       @query_strategy: QueryStrategy
-      @runners: Array[SubscriptionRunner]
+      @runners_query_options: Hash[SubscriptionRunner, Hash[Symbol, untyped]]
 
       def initialize: (Connection connection, QueryStrategy query_strategy) -> void
 

data/sig/pg_eventstore/subscriptions/subscription_feed_strategy/replication_strategy.rbs

@@ -7,7 +7,7 @@ module PgEventstore
 
       @connection: Connection
       @query_strategy: QueryStrategy
-      @runners: Array[ReplicaSubscriptionRunner]
+      @runners_query_options: Hash[ReplicaSubscriptionRunner, Hash[Symbol, untyped]]
 
       def initialize: (Connection connection, QueryStrategy query_strategy) -> void
 

data/sig/pg_eventstore/subscriptions/subscription_feed_strategy.rbs

@@ -1,6 +1,6 @@
 module PgEventstore
   module SubscriptionFeedStrategy[unchecked Runner]
-    def add: (*Runner runners) -> Array[Runner]
+    def add: (Runner runner, Hash[Symbol, untyped] query_options) -> void
 
     def any?: -> bool
 

data/sig/pg_eventstore/subscriptions/subscription_runner.rbs

@@ -23,8 +23,6 @@ module PgEventstore
 
     def next_chunk_query_opts: () -> ::Hash[untyped, untyped]
 
-    def time_to_feed?: () -> bool
-
     def next_chunk_global_position: () -> Integer
 
     def estimate_events_number: () -> Integer

data/sig/pg_eventstore/subscriptions/subscription_runners_feeder.rbs

@@ -8,6 +8,8 @@ module PgEventstore
 
     private
 
+    def time_to_feed?: (SubscriptionRunner runner) -> bool
+
     def connection: -> Connection
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pg_eventstore
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.1.2
 platform: ruby
 authors:
 - Ivan Dzyzenko