pg_eventstore 0.10.2 → 1.0.0.rc1

data/lib/pg_eventstore/queries/subscription_queries.rb

@@ -22,11 +22,7 @@ module PgEventstore
     # @param attrs [Hash]
     # @return [Hash, nil]
     def find_by(attrs)
-      builder = SQLBuilder.new.select('*').from('subscriptions')
-      attrs.each do |attr, val|
-        builder.where("#{attr} = ?", val)
-      end
-
+      builder = find_by_attrs_builder(attrs).limit(1)
       pg_result = connection.with do |conn|
         conn.exec_params(*builder.to_exec_params)
       end
@@ -35,6 +31,25 @@ module PgEventstore
       deserialize(pg_result.to_a.first)
     end
 
+    # @param attrs [Hash]
+    # @return [Array<Hash>]
+    def find_all(attrs)
+      builder = find_by_attrs_builder(attrs)
+      pg_result = connection.with do |conn|
+        conn.exec_params(*builder.to_exec_params)
+      end
+      return [] if pg_result.ntuples.zero?
+
+      pg_result.map(&method(:deserialize))
+    end
+
+    # @return [Array<String>]
+    def set_collection
+      connection.with do |conn|
+        conn.exec('SELECT set FROM subscriptions GROUP BY set ORDER BY set ASC')
+      end.map { |attrs| attrs['set'] }
+    end
+
     # @param id [Integer]
     # @return [Hash]
     # @raise [PgEventstore::RecordNotFound]
@@ -58,19 +73,32 @@ module PgEventstore
 
     # @param id [Integer]
     # @param attrs [Hash]
-    def update(id, attrs)
+    # @param locked_by [Integer]
+    # @return [Hash]
+    # @raise [PgEventstore::RecordNotFound]
+    # @raise [PgEventstore::WrongLockIdError]
+    def update(id, attrs:, locked_by:)
       attrs = { updated_at: Time.now.utc }.merge(attrs)
       attrs_sql = attrs.keys.map.with_index(1) do |attr, index|
         "#{attr} = $#{index}"
       end.join(', ')
       sql =
         "UPDATE subscriptions SET #{attrs_sql} WHERE id = $#{attrs.keys.size + 1} RETURNING *"
-      pg_result = connection.with do |conn|
-        conn.exec_params(sql, [*attrs.values, id])
+      updated_attrs = transaction_queries.transaction do
+        pg_result = connection.with do |conn|
+          conn.exec_params(sql, [*attrs.values, id])
+        end
+        raise(RecordNotFound.new("subscriptions", id)) if pg_result.ntuples.zero?
+
+        updated_attrs = pg_result.to_a.first
+        unless updated_attrs['locked_by'] == locked_by
+          # Subscription is force-locked by someone else. We have to roll back such transaction
+          raise(WrongLockIdError.new(updated_attrs['set'], updated_attrs['name'], updated_attrs['locked_by']))
+        end
+        updated_attrs
       end
-      raise(RecordNotFound.new("subscriptions", id)) if pg_result.ntuples.zero?
 
-      deserialize(pg_result.to_a.first)
+      deserialize(updated_attrs)
     end
 
     # @param query_options [Hash{Integer => Hash}] runner_id/query options association
@@ -90,13 +118,16 @@ module PgEventstore
     end
 
     # @param id [Integer] subscription's id
-    # @param lock_id [String] UUIDv4 id of the subscriptions set which reserves the subscription
+    # @param lock_id [Integer] id of the subscriptions set which reserves the subscription
     # @param force [Boolean] whether to lock the subscription despite on #locked_by value
-    # @return [String] UUIDv4 lock id
+    # @return [Integer] lock id
     # @raise [SubscriptionAlreadyLockedError] in case the Subscription is already locked
-    def lock!(id, lock_id, force = false)
+    def lock!(id, lock_id, force: false)
       transaction_queries.transaction do
         attrs = find!(id)
+        # We don't care who locked the Subscription - whether it is the same SubscriptionsSet or not - multiple locks
+        # must not happen even with the same SubscriptionsSet. We later assume this to reset Subscription's stats, for
+        # example.
         if attrs[:locked_by] && !force
           raise SubscriptionAlreadyLockedError.new(attrs[:set], attrs[:name], attrs[:locked_by])
         end
@@ -107,22 +138,11 @@ module PgEventstore
       lock_id
     end
 
-    # @param id [Integer] subscription's id
-    # @param lock_id [String] UUIDv4 id of the set which reserved the subscription after itself
+    # @param id [Integer]
     # @return [void]
-    # @raise [SubscriptionUnlockError] in case the Subscription is locked by some SubscriptionsSet, other than the one,
-    #   persisted in memory
-    def unlock!(id, lock_id)
-      transaction_queries.transaction do
-        attrs = find!(id)
-        # Normally this should never happen as locking/unlocking happens within the same process. This is done only for
-        # the matter of consistency.
-        unless attrs[:locked_by] == lock_id
-          raise SubscriptionUnlockError.new(attrs[:set], attrs[:name], lock_id, attrs[:locked_by])
-        end
-        connection.with do |conn|
-          conn.exec_params('UPDATE subscriptions SET locked_by = $1 WHERE id = $2', [nil, id])
-        end
+    def delete(id)
+      connection.with do |conn|
+        conn.exec_params('DELETE FROM subscriptions WHERE id = $1', [id])
       end
     end
 
@@ -159,5 +179,15 @@ module PgEventstore
     def deserialize(hash)
       hash.transform_keys(&:to_sym)
     end
+
+    # @param attrs [Hash]
+    # @return [PgEventstore::SQLBuilder]
+    def find_by_attrs_builder(attrs)
+      builder = SQLBuilder.new.select('*').from('subscriptions').order('id ASC')
+      attrs.each do |attr, val|
+        builder.where("#{attr} = ?", val)
+      end
+      builder
+    end
   end
 end

data/lib/pg_eventstore/queries/subscriptions_set_command_queries.rb

@@ -11,6 +11,13 @@ module PgEventstore
       @connection = connection
     end
 
+    # @return [Hash]
+    def find_or_create_by(...)
+      transaction_queries.transaction do
+        find_by(...) || create(...)
+      end
+    end
+
     # @param subscriptions_set_id [Integer]
     # @param command_name [String]
     # @return [Hash, nil]
@@ -31,7 +38,7 @@ module PgEventstore
     # @param subscriptions_set_id [Integer]
     # @param command_name [String]
     # @return [Hash]
-    def create_by(subscriptions_set_id:, command_name:)
+    def create(subscriptions_set_id:, command_name:)
       sql = <<~SQL
         INSERT INTO subscriptions_set_commands (name, subscriptions_set_id) 
           VALUES ($1, $2)
@@ -72,5 +79,10 @@ module PgEventstore
     def deserialize(hash)
       hash.transform_keys(&:to_sym)
     end
+
+    # @return [PgEventstore::TransactionQueries]
+    def transaction_queries
+      TransactionQueries.new(connection)
+    end
   end
 end

data/lib/pg_eventstore/queries/subscriptions_set_queries.rb

@@ -14,7 +14,7 @@ module PgEventstore
     # @param attrs [Hash]
     # @return [Array<Hash>]
     def find_all(attrs)
-      builder = SQLBuilder.new.select('*').from('subscriptions_set')
+      builder = SQLBuilder.new.select('*').from('subscriptions_set').order('name ASC, id ASC')
       attrs.each do |attr, val|
         builder.where("#{attr} = ?", val)
       end
@@ -25,13 +25,22 @@ module PgEventstore
       pg_result.to_a.map(&method(:deserialize))
     end
 
+    # @return [Array<String>]
+    def set_names
+      builder = SQLBuilder.new.select('name').from('subscriptions_set').group('name').order('name ASC')
+
+      connection.with do |conn|
+        conn.exec_params(*builder.to_exec_params)
+      end.map { |attrs| attrs['name'] }
+    end
+
     # The same as #find_all, but returns first result
     # @return [Hash, nil]
     def find_by(...)
       find_all(...).first
     end
 
-    # @param id [String] UUIDv4
+    # @param id [Integer]
     # @return [Hash]
     # @raise [PgEventstore::RecordNotFound]
     def find!(id)
@@ -52,7 +61,7 @@ module PgEventstore
       deserialize(pg_result.to_a.first)
     end
 
-    # @param id [String] UUIDv4
+    # @param id [Integer]
     # @param attrs [Hash]
     def update(id, attrs)
       attrs = { updated_at: Time.now.utc }.merge(attrs)
@@ -70,7 +79,7 @@ module PgEventstore
       deserialize(pg_result.to_a.first)
     end
 
-    # @return id [Integer]
+    # @param id [Integer]
     # @return [void]
     def delete(id)
       connection.with do |conn|
@@ -85,5 +94,10 @@ module PgEventstore
     def deserialize(hash)
       hash.transform_keys(&:to_sym)
     end
+
+    # @return [PgEventstore::TransactionQueries]
+    def transaction_queries
+      TransactionQueries.new(connection)
+    end
   end
 end

data/lib/pg_eventstore/query_builders/events_filtering_query.rb

@@ -28,8 +28,8 @@ module PgEventstore
         # @return [PgEventstore::QueryBuilders::EventsFiltering]
         def all_stream_filtering(options)
           event_filter = new
-          options in { filter: { event_types: Array => event_type_ids } }
-          event_filter.add_event_types(event_type_ids)
+          options in { filter: { event_types: Array => event_types } }
+          event_filter.add_event_types(event_types)
           event_filter.add_limit(options[:max_count])
           options in { filter: { streams: Array => streams } }
           streams&.each { |attrs| event_filter.add_stream_attrs(**attrs) }
@@ -43,8 +43,8 @@ module PgEventstore
         # @return [PgEventstore::QueryBuilders::EventsFiltering]
         def specific_stream_filtering(stream, options)
           event_filter = new
-          options in { filter: { event_types: Array => event_type_ids } }
-          event_filter.add_event_types(event_type_ids)
+          options in { filter: { event_types: Array => event_types } }
+          event_filter.add_event_types(event_types)
           event_filter.add_limit(options[:max_count])
           event_filter.add_stream_attrs(**stream.to_hash)
           event_filter.add_revision(options[:from_revision], options[:direction])

data/lib/pg_eventstore/subscriptions/command_handlers/subscription_feeder_commands.rb

@@ -3,7 +3,7 @@
 module PgEventstore
   module CommandHandlers
     class SubscriptionFeederCommands
-      AVAILABLE_COMMANDS = %w[StopAll StartAll].freeze
+      AVAILABLE_COMMANDS = %i[StopAll StartAll Restore Stop].to_h { [_1, _1.to_s] }.freeze
 
       # @param config_name [Symbol]
       # @param subscription_feeder [PgEventstore::SubscriptionFeeder]
@@ -16,7 +16,7 @@ module PgEventstore
       # @return [void]
       def process
         queries.find_commands(@subscription_feeder.id).each do |command|
-          unless AVAILABLE_COMMANDS.include?(command[:name])
+          unless AVAILABLE_COMMANDS.values.include?(command[:name])
             PgEventstore.logger&.warn(
               "#{self.class.name}: Don't know how to handle #{command[:name].inspect}. Details: #{command.inspect}."
             )
@@ -47,6 +47,14 @@ module PgEventstore
       def start_all
         @subscription_feeder.start_all
       end
+
+      def restore
+        @subscription_feeder.restore
+      end
+
+      def stop
+        @subscription_feeder.stop
+      end
     end
   end
 end

data/lib/pg_eventstore/subscriptions/command_handlers/subscription_runners_commands.rb

@@ -3,20 +3,22 @@
 module PgEventstore
   module CommandHandlers
     class SubscriptionRunnersCommands
-      AVAILABLE_COMMANDS = %w[StopRunner RestoreRunner StartRunner].freeze
+      AVAILABLE_COMMANDS = %i[Start Stop Restore].to_h { [_1, _1.to_s] }.freeze
 
       # @param config_name [Symbol]
       # @param runners [Array<PgEventstore::SubscriptionRunner>]
-      def initialize(config_name, runners)
+      # @param subscriptions_set_id [Integer]
+      def initialize(config_name, runners, subscriptions_set_id)
         @config_name = config_name
         @runners = runners
+        @subscriptions_set_id = subscriptions_set_id
       end
 
       # Look up commands for all given SubscriptionRunner-s and execute them
       # @return [void]
       def process
-        queries.find_commands(@runners.map(&:id)).each do |command|
-          unless AVAILABLE_COMMANDS.include?(command[:name])
+        queries.find_commands(@runners.map(&:id), subscriptions_set_id: @subscriptions_set_id).each do |command|
+          unless AVAILABLE_COMMANDS.values.include?(command[:name])
             PgEventstore.logger&.warn(
               "#{self.class.name}: Don't know how to handle #{command[:name].inspect}. Details: #{command.inspect}."
             )
@@ -48,19 +50,19 @@ module PgEventstore
 
       # @param subscription_id [Integer]
       # @return [void]
-      def start_runner(subscription_id)
+      def start(subscription_id)
         find_subscription_runner(subscription_id)&.start
       end
 
       # @param subscription_id [Integer]
       # @return [void]
-      def restore_runner(subscription_id)
+      def restore(subscription_id)
         find_subscription_runner(subscription_id)&.restore
       end
 
       # @param subscription_id [Integer]
       # @return [void]
-      def stop_runner(subscription_id)
+      def stop(subscription_id)
         find_subscription_runner(subscription_id)&.stop_async
       end
     end

data/lib/pg_eventstore/subscriptions/commands_handler.rb

@@ -11,6 +11,7 @@ module PgEventstore
     extend Forwardable
 
     RESTART_DELAY = 5 # seconds
+    PULL_INTERVAL = 1
 
     def_delegators :@basic_runner, :start, :stop, :state, :stop_async, :wait_for_finish
 
@@ -21,7 +22,7 @@ module PgEventstore
       @config_name = config_name
       @subscription_feeder = subscription_feeder
       @runners = runners
-      @basic_runner = BasicRunner.new(1, 0)
+      @basic_runner = BasicRunner.new(PULL_INTERVAL, 0)
       attach_runner_callbacks
     end
 
@@ -56,7 +57,7 @@ module PgEventstore
 
     # @return [PgEventstore::CommandHandlers::SubscriptionRunnersCommands]
     def subscription_runners_commands
-      CommandHandlers::SubscriptionRunnersCommands.new(@config_name, @runners)
+      CommandHandlers::SubscriptionRunnersCommands.new(@config_name, @runners, @subscription_feeder.id)
     end
   end
 end

data/lib/pg_eventstore/subscriptions/subscription.rb

@@ -63,8 +63,8 @@ module PgEventstore
     #     processor
     attribute(:last_chunk_greatest_position)
     # @!attribute locked_by
-    #   @return [String, nil] UUIDv4. The id of subscription manager which obtained the lock of the Subscription. _nil_
-    #     value means that the Subscription isn't locked yet by any subscription manager.
+    #   @return [Integer, nil] The id of subscription manager which obtained the lock of the Subscription. _nil_ value
+    #     means that the Subscription isn't locked yet by any subscription manager.
     attribute(:locked_by)
     # @!attribute created_at
     #   @return [Time]
@@ -80,7 +80,7 @@ module PgEventstore
     # @param attrs [Hash]
     # @return [Hash]
     def update(attrs)
-      assign_attributes(subscription_queries.update(id, attrs))
+      assign_attributes(subscription_queries.update(id, attrs: attrs, locked_by: locked_by))
     end
 
     # @param attrs [Hash]
@@ -93,20 +93,13 @@ module PgEventstore
 
     # Locks the Subscription by the given lock id
     # @return [PgEventstore::Subscription]
-    def lock!(lock_id, force = false)
+    def lock!(lock_id, force: false)
       self.id = subscription_queries.find_or_create_by(set: set, name: name)[:id]
-      self.locked_by = subscription_queries.lock!(id, lock_id, force)
+      self.locked_by = subscription_queries.lock!(id, lock_id, force: force)
       reset_runtime_attributes
       self
     end
 
-    # Unlocks the Subscription.
-    # @return [void]
-    def unlock!
-      subscription_queries.unlock!(id, locked_by)
-      self.locked_by = nil
-    end
-
     # Dup the current object without assigned connection
     # @return [PgEventstore::Subscription]
     def dup
@@ -119,6 +112,27 @@ module PgEventstore
       self
     end
 
+    # @return [Integer]
+    def hash
+      id.hash
+    end
+
+    # @param another [Object]
+    # @return [Boolean]
+    def eql?(another)
+      return false unless another.is_a?(Subscription)
+
+      hash == another.hash
+    end
+
+    # @param another [PgEventstore::SubscriptionsSet]
+    # @return [Boolean]
+    def ==(another)
+      return false unless another.is_a?(Subscription)
+
+      id == another.id
+    end
+
     private
 
     def reset_runtime_attributes
@@ -130,6 +144,8 @@ module PgEventstore
         chunk_query_interval: chunk_query_interval,
         last_chunk_fed_at: Time.at(0).utc,
         last_chunk_greatest_position: nil,
+        last_error: nil,
+        last_error_occurred_at: nil,
         time_between_restarts: time_between_restarts,
         state: RunnerState::STATES[:initial]
       )

data/lib/pg_eventstore/subscriptions/subscription_feeder.rb

@@ -3,10 +3,11 @@
 module PgEventstore
   # This class is responsible for starting/stopping all SubscriptionRunners. The background runner of it is responsible
   # for events pulling and feeding those SubscriptionRunners.
-  # @!visibility private
   class SubscriptionFeeder
     extend Forwardable
 
+    HEARTBEAT_INTERVAL = 10 # seconds
+
     def_delegators :subscriptions_set, :id
     def_delegators :@basic_runner, :start, :stop, :restore, :state, :wait_for_finish, :stop_async
 
@@ -17,12 +18,13 @@ module PgEventstore
     def initialize(config_name:, set_name:, max_retries:, retries_interval:)
       @config_name = config_name
       @runners = []
-      @set_name = set_name
-      @max_retries = max_retries
-      @retries_interval = retries_interval
+      @subscriptions_set_attrs = {
+        name: set_name, max_restarts_number: max_retries, time_between_restarts: retries_interval
+      }
       @commands_handler = CommandsHandler.new(@config_name, self, @runners)
       @basic_runner = BasicRunner.new(0.2, 0)
       @force_lock = false
+      @refreshed_at = Time.at(0)
       attach_runner_callbacks
     end
 
@@ -77,18 +79,12 @@ module PgEventstore
     # Locks all Subscriptions behind the current SubscriptionsSet
     # @return [void]
     def lock_all
-      @runners.each { |runner| runner.lock!(subscriptions_set.id, @force_lock) }
-    end
-
-    # @return [void]
-    def unlock_all
-      @runners.each(&:unlock!)
+      @runners.each { |runner| runner.lock!(subscriptions_set.id, force: @force_lock) }
     end
 
     # @return [PgEventstore::SubscriptionsSet]
     def subscriptions_set
-      @subscriptions_set ||= SubscriptionsSet.using_connection(@config_name).
-        create(name: @set_name, max_restarts_number: @max_retries, time_between_restarts: @retries_interval)
+      @subscriptions_set ||= SubscriptionsSet.using_connection(@config_name).create(**@subscriptions_set_attrs)
     end
 
     # @return [PgEventstore::SubscriptionRunnersFeeder]
@@ -102,6 +98,7 @@ module PgEventstore
       @basic_runner.define_callback(:before_runner_started, :before, method(:before_runner_started))
       @basic_runner.define_callback(:after_runner_died, :before, method(:after_runner_died))
       @basic_runner.define_callback(:after_runner_died, :after, method(:restart_runner))
+      @basic_runner.define_callback(:process_async, :before, method(:ping_subscriptions_set))
       @basic_runner.define_callback(:process_async, :before, method(:process_async))
       @basic_runner.define_callback(:after_runner_stopped, :before, method(:after_runner_stopped))
       @basic_runner.define_callback(:before_runner_restored, :after, method(:update_runner_restarts))
@@ -141,13 +138,20 @@ module PgEventstore
       feeder.feed(@runners)
     end
 
+    # @return [void]
+    def ping_subscriptions_set
+      return unless subscriptions_set.updated_at > Time.now.utc - HEARTBEAT_INTERVAL
+
+      subscriptions_set.update(updated_at: Time.now.utc)
+      @refreshed_at = Time.now.utc
+    end
+
     # @return [void]
     def after_runner_stopped
-      @commands_handler.stop
+      @runners.each(&:stop_async).each(&:wait_for_finish)
       @subscriptions_set&.delete
       @subscriptions_set = nil
-      @runners.each(&:stop_async).each(&:wait_for_finish)
-      unlock_all
+      @commands_handler.stop
     end
 
     # @return [void]

data/lib/pg_eventstore/subscriptions/subscription_runner.rb

@@ -16,7 +16,7 @@ module PgEventstore
     attr_reader :subscription
 
     def_delegators :@events_processor, :start, :stop, :stop_async, :feed, :wait_for_finish, :restore, :state, :running?
-    def_delegators :@subscription, :lock!, :unlock!, :id
+    def_delegators :@subscription, :lock!, :id
 
     # @param stats [PgEventstore::SubscriptionHandlerPerformance]
     # @param events_processor [PgEventstore::EventsProcessor]

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -18,7 +18,7 @@ module PgEventstore
       runners_query_options = runners.to_h { |runner| [runner.id, runner.next_chunk_query_opts] }
       grouped_events = subscription_queries.subscriptions_events(runners_query_options)
       runners.each do |runner|
-        runner.feed(grouped_events[runner.id]) if grouped_events[runner.id]
+        runner.feed(grouped_events[runner.id] || [])
       end
     end
 

data/lib/pg_eventstore/subscriptions/subscriptions_set.rb

@@ -22,7 +22,7 @@ module PgEventstore
     end
 
     # @!attribute id
-    #   @return [String] UUIDv4. It is used to lock the Subscription by updating Subscription#locked_by attribute
+    #   @return [Integer] It is used to lock the Subscription by updating Subscription#locked_by attribute
     attribute(:id)
     # @!attribute name
     #   @return [String] name of the set
@@ -87,6 +87,27 @@ module PgEventstore
       self
     end
 
+    # @return [Integer]
+    def hash
+      id.hash
+    end
+
+    # @param another [Object]
+    # @return [Boolean]
+    def eql?(another)
+      return false unless another.is_a?(SubscriptionsSet)
+
+      hash == another.hash
+    end
+
+    # @param another [PgEventstore::SubscriptionsSet]
+    # @return [Boolean]
+    def ==(another)
+      return false unless another.is_a?(SubscriptionsSet)
+
+      id == another.id
+    end
+
     private
 
     # @return [PgEventstore::SubscriptionsSetQueries]

data/lib/pg_eventstore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEventstore
-  VERSION = "0.10.2"
+  VERSION = "1.0.0.rc1"
 end