sequent 7.0.0 → 7.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2271f62c7221e3eb21eb7707ad2f7df52b9adeae3154c0856b860133af872457
-  data.tar.gz: dc3b99bc7c06957f98f5c45be5cff4082ae0d9cca493a9ea3431fc14eea67e08
+  metadata.gz: 31a7eb29122b5706014155cacea33cc9064991eca03762d6c9eb0a300e4d3fe5
+  data.tar.gz: 4724c03d49b69fd01d111a53961936c3b1a6adba91bacb9af0abd97c4337d49a
 SHA512:
-  metadata.gz: 150cff9d3a13c1b230dfb71f6897f740c1c4c5ac3448dbb0fd334858229ae1bf3ff67e1bdad897145e5bc0a2a5cd70a31a430e93939857ba753f2f9895a84fa1
-  data.tar.gz: ae8ae17080b36ed4b60f361aefb3d9b816b12b037f9eb4022c00969c2b9ce2c78cdfe6f4c67358d02c376ec37479d5951ad10e92944ba4b3e6b056b3fd8c2b5e
+  metadata.gz: 6c13866f01dde089ae6cd95ca17fbbfd594209751300f429b0ef4f3680d19f1a468bf2675b24536b6e09150f80471afdab5a663be79fd45692fe758c47dd8f26
+  data.tar.gz: 96d19ae22452eb86ed09fda426e6cb56aad6b47be5589004a8ba0e24623bd8316297d8b29829210fafb57e75b251797007bb9b610d27850809ce4b1c661adcde

data/db/sequent_schema.rb

@@ -8,8 +8,12 @@ ActiveRecord::Schema.define do
     t.text "event_json", :null => false
     t.integer "command_record_id", :null => false
     t.integer "stream_record_id", :null => false
+    t.bigint "xact_id"
   end
 
+  execute %Q{
+ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint
+}
   execute %Q{
 CREATE UNIQUE INDEX unique_event_per_aggregate ON event_records (
   aggregate_id,
@@ -24,6 +28,7 @@ CREATE INDEX snapshot_events ON event_records (aggregate_id, sequence_number DES
   add_index "event_records", ["command_record_id"], :name => "index_event_records_on_command_record_id"
   add_index "event_records", ["event_type"], :name => "index_event_records_on_event_type"
   add_index "event_records", ["created_at"], :name => "index_event_records_on_created_at"
+  add_index "event_records", ["xact_id"], :name => "index_event_records_on_xact_id"
 
   create_table "command_records", :force => true do |t|
     t.string "user_id"

data/lib/sequent/configuration.rb

@@ -10,7 +10,6 @@ require 'logger'
 module Sequent
   class Configuration
     DEFAULT_VERSIONS_TABLE_NAME = 'sequent_versions'
-    DEFAULT_REPLAYED_IDS_TABLE_NAME = 'sequent_replayed_ids'
 
     DEFAULT_MIGRATION_SQL_FILES_DIRECTORY = 'db/tables'
     DEFAULT_DATABASE_CONFIG_DIRECTORY = 'db'
@@ -68,8 +67,7 @@ module Sequent
                   :enable_autoregistration
 
     attr_reader :migrations_class_name,
-                :versions_table_name,
-                :replayed_ids_table_name
+                :versions_table_name
 
     def self.instance
       @instance ||= new
@@ -104,7 +102,6 @@ module Sequent
       self.event_publisher = Sequent::Core::EventPublisher.new
       self.disable_event_handlers = false
       self.versions_table_name = DEFAULT_VERSIONS_TABLE_NAME
-      self.replayed_ids_table_name = DEFAULT_REPLAYED_IDS_TABLE_NAME
       self.migration_sql_files_directory = DEFAULT_MIGRATION_SQL_FILES_DIRECTORY
       self.view_schema_name = DEFAULT_VIEW_SCHEMA_NAME
       self.event_store_schema_name = DEFAULT_EVENT_STORE_SCHEMA_NAME
@@ -135,18 +132,11 @@ module Sequent
       enable_multiple_database_support && ActiveRecord.version > Gem::Version.new('6.1.0')
     end
 
-    def replayed_ids_table_name=(table_name)
-      fail ArgumentError, 'table_name can not be nil' unless table_name
-
-      @replayed_ids_table_name = table_name
-      Sequent::Migrations::ViewSchema::ReplayedIds.table_name = table_name
-    end
-
     def versions_table_name=(table_name)
       fail ArgumentError, 'table_name can not be nil' unless table_name
 
       @versions_table_name = table_name
-      Sequent::Migrations::ViewSchema::Versions.table_name = table_name
+      Sequent::Migrations::Versions.table_name = table_name
     end
 
     def migrations_class_name=(class_name)
@@ -158,13 +148,14 @@ module Sequent
       @migrations_class_name = class_name
     end
 
+    # @!visibility private
     def autoregister!
       return unless enable_autoregistration
 
       # Only autoregister the AggregateSnapshotter if the autoregistration is enabled
       Sequent::Core::AggregateSnapshotter.skip_autoregister = false
 
-      Rails.autoloaders.main.eager_load(force: true) if defined?(Rails)
+      autoload_if_in_rails
 
       self.class.instance.command_handlers ||= []
       for_each_autoregisterable_descenant_of(Sequent::CommandHandler) do |command_handler_class|
@@ -198,6 +189,10 @@ module Sequent
 
     private
 
+    def autoload_if_in_rails
+      Rails.autoloaders.main.eager_load(force: true) if defined?(Rails) && Rails.respond_to?(:autoloaders)
+    end
+
     def for_each_autoregisterable_descenant_of(clazz, &block)
       clazz
         .descendants

data/lib/sequent/core/event_record.rb

@@ -76,6 +76,7 @@ module Sequent
       include SerializesEvent
 
       self.table_name = 'event_records'
+      self.ignored_columns = %w[xact_id]
 
       belongs_to :stream_record
       belongs_to :command_record

data/lib/sequent/core/event_store.rb

@@ -209,6 +209,10 @@ module Sequent
 
       private
 
+      def quote_table_name(table_name)
+        Sequent.configuration.event_record_class.connection.quote_table_name(table_name)
+      end
+
       def event_types
         @event_types = if Sequent.configuration.event_store_cache_event_types
                          ThreadSafe::Cache.new

data/lib/sequent/core/helpers/message_handler.rb

@@ -2,7 +2,6 @@
 
 require_relative 'message_handler_option_registry'
 require_relative 'message_router'
-require_relative 'message_dispatcher'
 
 module Sequent
   module Core
@@ -60,11 +59,7 @@ module Sequent
           end
 
           def message_mapping
-            message_router
-              .routes
-              .select { |matcher, _handlers| matcher.is_a?(MessageMatchers::InstanceOf) }
-              .map { |k, v| [k.expected_class, v] }
-              .to_h
+            message_router.instanceof_routes
           end
 
           def handles_message?(message)
@@ -106,13 +101,17 @@ module Sequent
         end
 
         def handle_message(message)
-          message_dispatcher.dispatch_message(message)
+          handlers = self.class.message_router.match_message(message)
+          dispatch_message(message, handlers) unless handlers.empty?
         end
 
-        private
-
-        def message_dispatcher
-          MessageDispatcher.new(self.class.message_router, self)
+        def dispatch_message(message, handlers)
+          handlers.each do |handler|
+            if Sequent.logger.debug?
+              Sequent.logger.debug("[MessageHandler] Handler #{self.class} handling #{message.class}")
+            end
+            instance_exec(message, &handler)
+          end
         end
       end
     end

data/lib/sequent/core/helpers/message_router.rb

@@ -7,7 +7,7 @@ module Sequent
   module Core
     module Helpers
       class MessageRouter
-        attr_reader :routes
+        attr_reader :routes, :instanceof_routes
 
         def initialize
           clear_routes
@@ -21,7 +21,11 @@ module Sequent
         #
         def register_matchers(*matchers, handler)
           matchers.each do |matcher|
-            @routes[matcher] << handler
+            if matcher.is_a?(MessageMatchers::InstanceOf)
+              @instanceof_routes[matcher.expected_class] << handler
+            else
+              @routes[matcher] << handler
+            end
           end
         end
 
@@ -29,11 +33,12 @@ module Sequent
         # Returns a set of handlers that match the given message, or an empty set when none match.
         #
         def match_message(message)
-          @routes
-            .reduce(Set.new) do |memo, (matcher, handlers)|
-              memo = memo.merge(handlers) if matcher.matches_message?(message)
-              memo
-            end
+          result = Set.new
+          result.merge(@instanceof_routes[message.class])
+          @routes.each do |matcher, handlers|
+            result.merge(handlers) if matcher.matches_message?(message)
+          end
+          result
         end
 
         ##
@@ -47,6 +52,7 @@ module Sequent
         # Removes all routes from the router.
         #
         def clear_routes
+          @instanceof_routes = Hash.new { |h, k| h[k] = Set.new }
           @routes = Hash.new { |h, k| h[k] = Set.new }
         end
       end

data/lib/sequent/core/persistors/active_record_persistor.rb

@@ -117,6 +117,10 @@ module Sequent
           Sequent::ApplicationRecord.connection.execute(statement)
         end
 
+        def prepare
+          # noop
+        end
+
         def commit
           # noop
         end

data/lib/sequent/core/persistors/persistor.rb

@@ -76,6 +76,11 @@ module Sequent
           fail 'Method not supported in this persistor'
         end
 
+        # Hook to implement for instance the persistor batches statements
+        def prepare
+          fail 'Method not supported in this persistor'
+        end
+
         # Hook to implement for instance the persistor batches statements
         def commit
           fail 'Method not supported in this persistor'

data/lib/sequent/core/persistors/replay_optimized_postgres_persistor.rb

@@ -41,13 +41,17 @@ module Sequent
       #   end
       #
       # In this case it is wise to create an index on InvoiceRecord on the aggregate_id and recipient_id
-      # like you would in the database.
+      # attributes like you would in the database. Note that previous versions of this class supported
+      # multi-column indexes. These are now split into multiple single-column indexes and the results of
+      # each index is combined using set-intersection. This reduces the amount of memory used and makes
+      # it possible to use an index in more cases (whenever an indexed attribute is present in the where
+      # clause the index will be used, so not all attributes need to be present).
       #
       # Example:
       #
       #   ReplayOptimizedPostgresPersistor.new(
       #     50,
-      #     {InvoiceRecord => [[:aggregate_id, :recipient_id]]}
+      #     {InvoiceRecord => [:aggregate_id, :recipient_id]}
       #   )
       class ReplayOptimizedPostgresPersistor
         include Persistor
@@ -56,108 +60,106 @@ module Sequent
         attr_reader :record_store
         attr_accessor :insert_with_csv_size
 
-        def self.struct_cache
-          @struct_cache ||= {}
+        # We create a struct on the fly to represent an in-memory record.
+        #
+        # Since the replay happens in memory we implement the ==, eql? and hash methods
+        # to point to the same object. A record is the same if and only if they point to
+        # the same object. These methods are necessary since we use Set instead of [].
+        #
+        # Also basing equality on object identity is more consistent with ActiveRecord,
+        # which is the implementation used during normal (non-optimized) replay.
+        module InMemoryStruct
+          def ==(other)
+            equal?(other)
+          end
+          def eql?(other)
+            equal?(other)
+          end
+          def hash
+            object_id.hash
+          end
         end
 
-        module InitStruct
-          def set_values(values)
-            values.each do |k, v|
-              self[k] = v
+        def struct_cache
+          @struct_cache ||= Hash.new do |hash, record_class|
+            struct_class = Struct.new(*record_class.column_names.map(&:to_sym), keyword_init: true) do
+              include InMemoryStruct
             end
-            self
+            hash[record_class] = struct_class
           end
         end
 
         class Index
+          attr_reader :indexed_columns
+
           def initialize(indexed_columns)
-            @indexed_columns = Hash.new do |hash, record_class|
-              hash[record_class] = if record_class.column_names.include? 'aggregate_id'
-                                     ['aggregate_id']
-                                   else
-                                     []
-                                   end
+            @indexed_columns = indexed_columns.to_set
+            @indexes = @indexed_columns.to_h do |field|
+              [field, {}]
+            end
+            @reverse_indexes = @indexed_columns.to_h do |field|
+              [field, {}.compare_by_identity]
             end
-
-            @indexed_columns = @indexed_columns.merge(
-              indexed_columns.reduce({}) do |memo, (key, ics)|
-                memo.merge({key => ics.map { |c| c.map(&:to_s) }})
-              end,
-            )
-
-            @index = {}
-            @reverse_index = {}
           end
 
-          def add(record_class, record)
-            return unless indexed?(record_class)
-
-            get_keys(record_class, record).each do |key|
-              @index[key.hash] = [] unless @index.key? key.hash
-              @index[key.hash] << record
-
-              @reverse_index[record.object_id.hash] = [] unless @reverse_index.key? record.object_id.hash
-              @reverse_index[record.object_id.hash] << key.hash
+          def add(record)
+            @indexes.map do |field, index|
+              key = Persistors.normalize_symbols(record[field]).freeze
+              records = index[key] || (index[key] = Set.new.compare_by_identity)
+              records << record
+              @reverse_indexes[field][record] = key
             end
           end
 
-          def remove(record_class, record)
-            return unless indexed?(record_class)
-
-            keys = @reverse_index.delete(record.object_id.hash) { [] }
-
-            return unless keys.any?
-
-            keys.each do |key|
-              @index[key].delete(record)
-              @index.delete(key) if @index[key].count == 0
+          def remove(record)
+            @indexes.map do |field, index|
+              key = @reverse_indexes[field].delete(record)
+              remaining = index[key]&.delete(record)
+              index.delete(key) if remaining&.empty?
             end
           end
 
-          def update(record_class, record)
-            remove(record_class, record)
-            add(record_class, record)
+          def update(record)
+            remove(record)
+            add(record)
           end
 
-          def find(record_class, where_clause)
-            key = [record_class.name]
-            get_index(record_class, where_clause).each do |field|
-              key << field
-              key << where_clause.stringify_keys[field]
+          def find(normalized_where_clause)
+            record_sets = normalized_where_clause.map do |(field, expected_value)|
+              if expected_value.is_a?(Array)
+                expected_value.reduce(Set.new.compare_by_identity) do |memo, value|
+                  key = Persistors.normalize_symbols(value)
+                  memo.merge(@indexes[field][key] || [])
+                end
+              else
+                key = Persistors.normalize_symbols(expected_value)
+                @indexes[field][key] || Set.new.compare_by_identity
+              end
+            end
+
+            smallest, *rest = record_sets.sort_by(&:size)
+            return smallest.to_a if smallest.empty? || rest.empty?
+
+            smallest.select do |record|
+              rest.all? { |x| x.include? record }
             end
-            @index[key.hash] || []
           end
 
           def clear
-            @index = {}
-            @reverse_index = {}
+            @indexed_columns.each do |field|
+              @indexes[field].clear
+              @reverse_indexes[field].clear
+            end
           end
 
-          def use_index?(record_class, where_clause)
-            @indexed_columns.key?(record_class) && get_index(record_class, where_clause).present?
+          def use_index?(normalized_where_clause)
+            get_indexes(normalized_where_clause).present?
           end
 
           private
 
-          def indexed?(record_class)
-            @indexed_columns.key?(record_class)
-          end
-
-          def get_keys(record_class, record)
-            @indexed_columns[record_class].map do |index|
-              arr = [record_class.name]
-              index.each do |key|
-                arr << key
-                arr << record[key]
-              end
-              arr
-            end
-          end
-
-          def get_index(record_class, where_clause)
-            @indexed_columns[record_class].find do |indexed_where|
-              where_clause.keys.size == indexed_where.size && (where_clause.keys.map(&:to_s) - indexed_where).empty?
-            end
+          def get_indexes(normalized_where_clause)
+            @indexed_columns & normalized_where_clause.keys
           end
         end
 
@@ -167,17 +169,28 @@ module Sequent
         #   Key corresponds to the name of the 'Record'
         #   Values contains list of lists on which columns to index.
         #   E.g. [[:first_index_column], [:another_index, :with_to_columns]]
-        def initialize(insert_with_csv_size = 50, indices = {})
+        def initialize(insert_with_csv_size = 50, indices = {}, default_indexed_columns = [:aggregate_id])
           @insert_with_csv_size = insert_with_csv_size
-          @record_store = Hash.new { |h, k| h[k] = Set.new }
-          @record_index = Index.new(indices)
+          @record_store = Hash.new { |h, k| h[k] = Set.new.compare_by_identity }
+          @record_index = Hash.new do |h, k|
+            h[k] = Index.new(default_indexed_columns.to_set & k.column_names.map(&:to_sym))
+          end
+
+          indices.each do |record_class, indexed_columns|
+            columns = indexed_columns.flatten(1).map(&:to_sym).to_set + default_indexed_columns
+            @record_index[record_class] = Index.new(columns & record_class.column_names.map(&:to_sym))
+          end
+
+          @record_defaults = Hash.new do |h, record_class|
+            h[record_class] = record_class.column_defaults.symbolize_keys
+          end
         end
 
         def update_record(record_class, event, where_clause = {aggregate_id: event.aggregate_id}, options = {})
           record = get_record!(record_class, where_clause)
-          record.updated_at = event.created_at if record.respond_to?(:updated_at)
+          record.updated_at = event.created_at if record.respond_to?(:updated_at=)
           yield record if block_given?
-          @record_index.update(record_class, record)
+          @record_index[record_class].update(record)
           update_sequence_number = if options.key?(:update_sequence_number)
                                      options[:update_sequence_number]
                                    else
@@ -187,41 +200,16 @@ module Sequent
         end
 
         def create_record(record_class, values)
-          column_names = record_class.column_names
-          values = record_class.column_defaults.with_indifferent_access.merge(values)
-          values.merge!(updated_at: values[:created_at]) if column_names.include?('updated_at')
-          struct_class_name = "#{record_class}Struct"
-          if self.class.struct_cache.key?(struct_class_name)
-            struct_class = self.class.struct_cache[struct_class_name]
-          else
-            # We create a struct on the fly.
-            # Since the replay happens in memory we implement the ==, eql? and hash methods
-            # to point to the same object. A record is the same if and only if they point to
-            # the same object. These methods are necessary since we use Set instead of [].
-            class_def = <<-EOD
-      #{struct_class_name} = Struct.new(*#{column_names.map(&:to_sym)})
-              class #{struct_class_name}
-                include InitStruct
-                def ==(other)
-                  self.equal?(other)
-                end
-                def hash
-                  self.object_id.hash
-                end
-              end
-            EOD
-            # rubocop:disable Security/Eval
-            eval(class_def.to_s)
-            # rubocop:enable Security/Eval
-            struct_class = ReplayOptimizedPostgresPersistor.const_get(struct_class_name)
-            self.class.struct_cache[struct_class_name] = struct_class
+          record = struct_cache[record_class].new(**values)
+          @record_defaults[record_class].each do |column, default|
+            record[column] = default unless values.include? column
           end
-          record = struct_class.new.set_values(values)
+          record.updated_at = values[:created_at] if record.respond_to?(:updated_at)
 
           yield record if block_given?
-          @record_store[record_class] << record
 
-          @record_index.add(record_class, record)
+          @record_store[record_class] << record
+          @record_index[record_class].add(record)
 
           record
         end
@@ -234,7 +222,7 @@ module Sequent
           record = get_record(record_class, values)
           record ||= create_record(record_class, values.merge(created_at: created_at))
           yield record if block_given?
-          @record_index.update(record_class, record)
+          @record_index[record_class].update(record)
           record
         end
 
@@ -260,15 +248,15 @@ module Sequent
 
         def delete_record(record_class, record)
           @record_store[record_class].delete(record)
-          @record_index.remove(record_class, record)
+          @record_index[record_class].remove(record)
         end
 
         def update_all_records(record_class, where_clause, updates)
           find_records(record_class, where_clause).each do |record|
             updates.each_pair do |k, v|
-              record[k.to_sym] = v
+              record[k] = v
             end
-            @record_index.update(record_class, record)
+            @record_index[record_class].update(record)
           end
         end
 
@@ -276,33 +264,41 @@ module Sequent
           records = find_records(record_class, where_clause)
           records.each do |record|
             yield record
-            @record_index.update(record_class, record)
+            @record_index[record_class].update(record)
           end
         end
 
         def do_with_record(record_class, where_clause)
           record = get_record!(record_class, where_clause)
           yield record
-          @record_index.update(record_class, record)
+          @record_index[record_class].update(record)
         end
 
         def find_records(record_class, where_clause)
-          if @record_index.use_index?(record_class, where_clause)
-            @record_index.find(record_class, where_clause)
-          else
-            @record_store[record_class].select do |record|
-              where_clause.all? do |k, v|
-                expected_value = v.is_a?(Symbol) ? v.to_s : v
-                actual_value = record[k.to_sym]
-                actual_value = actual_value.to_s if actual_value.is_a? Symbol
-                if expected_value.is_a?(Array)
-                  expected_value.include?(actual_value)
-                else
-                  actual_value == expected_value
-                end
+          where_clause = where_clause.symbolize_keys
+
+          indexed_columns = @record_index[record_class].indexed_columns
+          indexed_fields, non_indexed_fields = where_clause.partition { |field, _| indexed_columns.include? field }
+
+          candidate_records = if indexed_fields.present?
+                                @record_index[record_class].find(indexed_fields)
+                              else
+                                @record_store[record_class]
+                              end
+
+          return candidate_records.to_a if non_indexed_fields.empty?
+
+          candidate_records.select do |record|
+            non_indexed_fields.all? do |k, v|
+              expected_value = Persistors.normalize_symbols(v)
+              actual_value = Persistors.normalize_symbols(record[k])
+              if expected_value.is_a?(Array)
+                expected_value.include?(actual_value)
+              else
+                actual_value == expected_value
               end
             end
-          end.dup
+          end
         end
 
         def last_record(record_class, where_clause)
@@ -310,6 +306,10 @@ module Sequent
           results.empty? ? nil : results.last
         end
 
+        def prepare
+          # noop
+        end
+
         def commit
           @record_store.each do |clazz, records|
             @column_cache ||= {}
@@ -358,7 +358,7 @@ module Sequent
 
         def clear
           @record_store.clear
-          @record_index.clear
+          @record_index.values.each(&:clear)
         end
 
         private
@@ -366,12 +366,19 @@ module Sequent
         def cast_value_to_column_type(clazz, column_name, record)
           uncasted_value = ActiveModel::Attribute.from_database(
             column_name,
-            record[column_name.to_sym],
+            record[column_name],
             Sequent::ApplicationRecord.connection.lookup_cast_type_from_column(@column_cache[clazz.name][column_name]),
           ).value_for_database
           Sequent::ApplicationRecord.connection.type_cast(uncasted_value)
         end
       end
+
+      # Normalizes symbol values to strings (by using its name) while
+      # preserving all other values. This allows symbol/string
+      # indifferent comparisons.
+      def self.normalize_symbols(value)
+        value.is_a?(Symbol) ? value.name : value
+      end
     end
   end
 end

data/lib/sequent/dry_run/dry_run.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative 'read_only_replay_optimized_postgres_persistor'
+require_relative 'view_schema'