pg_eventstore 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d164756463155daaa363ac2c51d2b85097559d471912af64298e988d6a1831bb
-  data.tar.gz: a3b727ae69c0cc4d897b9c1bc9ca4d2495677280f64f4af177ae790cefdfec1a
+  metadata.gz: d5a69951bc18bd74869eb843b9e017a1c1f238bfc17750400e30b99f7d7b9f2d
+  data.tar.gz: 56ce94941096227944e8d5f11b9c7f9ca2f1fd816ad477cb02abe58604945c47
 SHA512:
-  metadata.gz: bc96f8a831df8271c74aa4ec9570016c740f454ce3b48cca41e070ec41166d4aea46e68d5630707c30c797aa4ec922f982c53c147c90e2e665eb7f14118ed84b
-  data.tar.gz: c1f8daa5fb4ff0b243d3d34ae8846aef18d20dadf0284f7ac5e9d8c159f2f2fdfe622124c6186a4cc566a445c90228b27814ad93d7b4db3cdf98f793fc564670
+  metadata.gz: f2f682990d1450b874e3d1ad9cf68e3e1cf7dcd546c78bb101982c5a03c5f355f31101442b2dd9a2c59531535e02eff01b12b119c79beca331f7262e927ebe5c
+  data.tar.gz: 2b2624a388eba5772c6a3bdd601ffe6fbcdb3c08c0d3c4585c1dfcf03862cf94f19e39db453950294f1a5a01a35e5ee33c975632c7ba6a2c966fcb85ff6dbb5d

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [1.0.4]
+- Fix bug which caused slow Subscriptions to stop processing new events
+- Optimize Subscriptions update queries
+
 ## [1.0.3]
 - Do no update `Subscription#last_chunk_fed_at` if the chunk is empty
 

data/lib/pg_eventstore/errors.rb

@@ -203,4 +203,7 @@ module PgEventstore
       @event_types = event_types
     end
   end
+
+  class EmptyChunkFedError < Error
+  end
 end

data/lib/pg_eventstore/queries/subscription_queries.rb

@@ -101,6 +101,22 @@ module PgEventstore
       deserialize(updated_attrs)
     end
 
+    # @param subscriptions_set_id [Integer] SubscriptionsSet#id
+    # @param subscriptions_ids [Array<Integer>] Array of Subscription#id
+    # @return [Hash<Integer => Time>]
+    def ping_all(subscriptions_set_id, subscriptions_ids)
+      pg_result = connection.with do |conn|
+        sql = <<~SQL
+          UPDATE subscriptions SET updated_at = $1 WHERE locked_by = $2 AND id = ANY($3::int[])
+            RETURNING id, updated_at
+        SQL
+        conn.exec_params(sql, [Time.now.utc, subscriptions_set_id, subscriptions_ids])
+      end
+      pg_result.to_h do |attrs|
+        [attrs['id'], attrs['updated_at']]
+      end
+    end
+
     # @param query_options [Hash{Integer => Hash}] runner_id/query options association
     # @return [Hash{Integer => Hash}] runner_id/events association
     def subscriptions_events(query_options)

data/lib/pg_eventstore/subscriptions/events_processor.rb

@@ -20,6 +20,8 @@ module PgEventstore
     # @param raw_events [Array<Hash>]
     # @return [void]
     def feed(raw_events)
+      raise EmptyChunkFedError.new("Empty chunk was fed!") if raw_events.empty?
+
       callbacks.run_callbacks(:feed, global_position(raw_events.last))
       @raw_events.push(*raw_events)
     end
@@ -69,11 +71,9 @@ module PgEventstore
       callbacks.run_callbacks(:change_state, ...)
     end
 
-    # @param raw_event [Hash, nil]
-    # @return [Integer, nil]
+    # @param raw_event [Hash]
+    # @return [Integer]
     def global_position(raw_event)
-      return unless raw_event
-
       raw_event['link'] ? raw_event['link']['global_position'] : raw_event['global_position']
     end
   end

data/lib/pg_eventstore/subscriptions/subscription.rb

@@ -73,6 +73,25 @@ module PgEventstore
     #   @return [Time]
     attribute(:updated_at)
 
+    class << self
+      # @param subscriptions_set_id [Integer] SubscriptionsSet#id
+      # @param subscriptions [Array<PgEventstoreSubscription>]
+      # @return [void]
+      def ping_all(subscriptions_set_id, subscriptions)
+        result = subscription_queries.ping_all(subscriptions_set_id, subscriptions.map(&:id))
+        subscriptions.each do |subscription|
+          next unless result[subscription.id]
+
+          subscription.assign_attributes(updated_at: result[subscription.id])
+        end
+      end
+
+      # @return [PgEventstore::SubscriptionQueries]
+      def subscription_queries
+        SubscriptionQueries.new(connection)
+      end
+    end
+
     def options=(val)
       @options = Utils.deep_transform_keys(val, &:to_sym)
     end
@@ -135,6 +154,7 @@ module PgEventstore
 
     private
 
+    # @return [void]
     def reset_runtime_attributes
       update(
         options: options,
@@ -151,8 +171,9 @@ module PgEventstore
       )
     end
 
+    # @return [PgEventstore::SubscriptionQueries]
     def subscription_queries
-      SubscriptionQueries.new(self.class.connection)
+      self.class.subscription_queries
     end
   end
 end

data/lib/pg_eventstore/subscriptions/subscription_feeder.rb

@@ -24,7 +24,7 @@ module PgEventstore
       @commands_handler = CommandsHandler.new(@config_name, self, @runners)
       @basic_runner = BasicRunner.new(0.2, 0)
       @force_lock = false
-      @refreshed_at = Time.at(0)
+      @subscriptions_pinged_at = Time.at(0)
       attach_runner_callbacks
     end
 
@@ -100,6 +100,7 @@ module PgEventstore
       @basic_runner.define_callback(:after_runner_died, :after, method(:restart_runner))
       @basic_runner.define_callback(:process_async, :before, method(:ping_subscriptions_set))
       @basic_runner.define_callback(:process_async, :before, method(:process_async))
+      @basic_runner.define_callback(:process_async, :after, method(:ping_subscriptions))
       @basic_runner.define_callback(:after_runner_stopped, :before, method(:after_runner_stopped))
       @basic_runner.define_callback(:before_runner_restored, :after, method(:update_runner_restarts))
     end
@@ -140,10 +141,25 @@ module PgEventstore
 
     # @return [void]
     def ping_subscriptions_set
-      return unless subscriptions_set.updated_at > Time.now.utc - HEARTBEAT_INTERVAL
+      return if subscriptions_set.updated_at > Time.now.utc - HEARTBEAT_INTERVAL
 
       subscriptions_set.update(updated_at: Time.now.utc)
-      @refreshed_at = Time.now.utc
+    end
+
+    # @return [void]
+    def ping_subscriptions
+      return if @subscriptions_pinged_at > Time.now.utc - HEARTBEAT_INTERVAL
+
+      runners = @runners.select do |runner|
+        next false unless runner.running?
+
+        runner.subscription.updated_at < Time.now.utc - HEARTBEAT_INTERVAL
+      end
+      unless runners.empty?
+        Subscription.using_connection(@config_name).ping_all(subscriptions_set.id, runners.map(&:subscription))
+      end
+
+      @subscriptions_pinged_at = Time.now.utc
     end
 
     # @return [void]

data/lib/pg_eventstore/subscriptions/subscription_runner.rb

@@ -11,6 +11,7 @@ module PgEventstore
     extend Forwardable
 
     MAX_EVENTS_PER_CHUNK = 1_000
+    MIN_EVENTS_PER_CHUNK = 10
     INITIAL_EVENTS_PER_CHUNK = 10
 
     attr_reader :subscription
@@ -56,7 +57,10 @@ module PgEventstore
       return INITIAL_EVENTS_PER_CHUNK if @stats.average_event_processing_time.zero?
 
       events_per_chunk = (@subscription.chunk_query_interval / @stats.average_event_processing_time).round
-      [[events_per_chunk, MAX_EVENTS_PER_CHUNK].min - @events_processor.events_left_in_chunk, 0].max
+      events_to_fetch = [events_per_chunk, MAX_EVENTS_PER_CHUNK].min - @events_processor.events_left_in_chunk
+      return 0 if events_to_fetch < 0 # We still have a lot of events in the chunk - no need to fetch more
+
+      [events_to_fetch, MIN_EVENTS_PER_CHUNK].max
     end
 
     # @return [void]
@@ -103,12 +107,9 @@ module PgEventstore
       @subscription.update(last_error: Utils.error_info(error), last_error_occurred_at: Time.now.utc)
     end
 
-    # @param global_position [Integer, nil]
+    # @param global_position [Integer]
     # @return [void]
     def update_subscription_chunk_stats(global_position)
-      # nil means subscriptions events query were executed, but there were no new events
-      return @subscription.update(updated_at: Time.now.utc) if global_position.nil?
-
       @subscription.update(last_chunk_fed_at: Time.now.utc, last_chunk_greatest_position: global_position)
     end
 

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -17,8 +17,9 @@ module PgEventstore
 
       runners_query_options = runners.to_h { |runner| [runner.id, runner.next_chunk_query_opts] }
       grouped_events = subscription_queries.subscriptions_events(runners_query_options)
+
       runners.each do |runner|
-        runner.feed(grouped_events[runner.id] || [])
+        runner.feed(grouped_events[runner.id]) if grouped_events[runner.id]
       end
     end
 

data/lib/pg_eventstore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEventstore
-  VERSION = "1.0.3"
+  VERSION = "1.0.4"
 end

data/lib/pg_eventstore/web/subscriptions/helpers.rb

@@ -43,7 +43,8 @@ module PgEventstore
         # @return [String] html status
         def colored_state(state, updated_at)
           if state == RunnerState::STATES[:running]
-            if updated_at < Time.now.utc - SubscriptionFeeder::HEARTBEAT_INTERVAL
+            # -1 is added as a margin to prevent false-positive result
+            if updated_at < Time.now.utc - SubscriptionFeeder::HEARTBEAT_INTERVAL - 1
               title = <<~TEXT
                 Something is wrong. Last update was more than #{SubscriptionFeeder::HEARTBEAT_INTERVAL} seconds \
                 ago(#{updated_at}).

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_eventstore
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Ivan Dzyzenko
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-05-17 00:00:00.000000000 Z
+date: 2024-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg