sequent 6.0.1 → 7.1.0

data/lib/sequent/core/persistors/replay_optimized_postgres_persistor.rb

@@ -41,13 +41,17 @@ module Sequent
       #   end
       #
       # In this case it is wise to create an index on InvoiceRecord on the aggregate_id and recipient_id
-      # like you would in the database.
+      # attributes like you would in the database. Note that previous versions of this class supported
+      # multi-column indexes. These are now split into multiple single-column indexes and the results of
+      # each index is combined using set-intersection. This reduces the amount of memory used and makes
+      # it possible to use an index in more cases (whenever an indexed attribute is present in the where
+      # clause the index will be used, so not all attributes need to be present).
       #
       # Example:
       #
       #   ReplayOptimizedPostgresPersistor.new(
       #     50,
-      #     {InvoiceRecord => [[:aggregate_id, :recipient_id]]}
+      #     {InvoiceRecord => [:aggregate_id, :recipient_id]}
       #   )
       class ReplayOptimizedPostgresPersistor
         include Persistor
@@ -56,108 +60,106 @@ module Sequent
         attr_reader :record_store
         attr_accessor :insert_with_csv_size
 
-        def self.struct_cache
-          @struct_cache ||= {}
+        # We create a struct on the fly to represent an in-memory record.
+        #
+        # Since the replay happens in memory we implement the ==, eql? and hash methods
+        # to point to the same object. A record is the same if and only if they point to
+        # the same object. These methods are necessary since we use Set instead of [].
+        #
+        # Also basing equality on object identity is more consistent with ActiveRecord,
+        # which is the implementation used during normal (non-optimized) replay.
+        module InMemoryStruct
+          def ==(other)
+            equal?(other)
+          end
+          def eql?(other)
+            equal?(other)
+          end
+          def hash
+            object_id.hash
+          end
         end
 
-        module InitStruct
-          def set_values(values)
-            values.each do |k, v|
-              self[k] = v
+        def struct_cache
+          @struct_cache ||= Hash.new do |hash, record_class|
+            struct_class = Struct.new(*record_class.column_names.map(&:to_sym), keyword_init: true) do
+              include InMemoryStruct
             end
-            self
+            hash[record_class] = struct_class
           end
         end
 
         class Index
+          attr_reader :indexed_columns
+
           def initialize(indexed_columns)
-            @indexed_columns = Hash.new do |hash, record_class|
-              hash[record_class] = if record_class.column_names.include? 'aggregate_id'
-                                     ['aggregate_id']
-                                   else
-                                     []
-                                   end
+            @indexed_columns = indexed_columns.to_set
+            @indexes = @indexed_columns.to_h do |field|
+              [field, {}]
+            end
+            @reverse_indexes = @indexed_columns.to_h do |field|
+              [field, {}.compare_by_identity]
             end
-
-            @indexed_columns = @indexed_columns.merge(
-              indexed_columns.reduce({}) do |memo, (key, ics)|
-                memo.merge({key => ics.map { |c| c.map(&:to_s) }})
-              end,
-            )
-
-            @index = {}
-            @reverse_index = {}
           end
 
-          def add(record_class, record)
-            return unless indexed?(record_class)
-
-            get_keys(record_class, record).each do |key|
-              @index[key.hash] = [] unless @index.key? key.hash
-              @index[key.hash] << record
-
-              @reverse_index[record.object_id.hash] = [] unless @reverse_index.key? record.object_id.hash
-              @reverse_index[record.object_id.hash] << key.hash
+          def add(record)
+            @indexes.map do |field, index|
+              key = Persistors.normalize_symbols(record[field]).freeze
+              records = index[key] || (index[key] = Set.new.compare_by_identity)
+              records << record
+              @reverse_indexes[field][record] = key
             end
           end
 
-          def remove(record_class, record)
-            return unless indexed?(record_class)
-
-            keys = @reverse_index.delete(record.object_id.hash) { [] }
-
-            return unless keys.any?
-
-            keys.each do |key|
-              @index[key].delete(record)
-              @index.delete(key) if @index[key].count == 0
+          def remove(record)
+            @indexes.map do |field, index|
+              key = @reverse_indexes[field].delete(record)
+              remaining = index[key]&.delete(record)
+              index.delete(key) if remaining&.empty?
             end
           end
 
-          def update(record_class, record)
-            remove(record_class, record)
-            add(record_class, record)
+          def update(record)
+            remove(record)
+            add(record)
           end
 
-          def find(record_class, where_clause)
-            key = [record_class.name]
-            get_index(record_class, where_clause).each do |field|
-              key << field
-              key << where_clause.stringify_keys[field]
+          def find(normalized_where_clause)
+            record_sets = normalized_where_clause.map do |(field, expected_value)|
+              if expected_value.is_a?(Array)
+                expected_value.reduce(Set.new.compare_by_identity) do |memo, value|
+                  key = Persistors.normalize_symbols(value)
+                  memo.merge(@indexes[field][key] || [])
+                end
+              else
+                key = Persistors.normalize_symbols(expected_value)
+                @indexes[field][key] || Set.new.compare_by_identity
+              end
+            end
+
+            smallest, *rest = record_sets.sort_by(&:size)
+            return smallest.to_a if smallest.empty? || rest.empty?
+
+            smallest.select do |record|
+              rest.all? { |x| x.include? record }
             end
-            @index[key.hash] || []
           end
 
           def clear
-            @index = {}
-            @reverse_index = {}
+            @indexed_columns.each do |field|
+              @indexes[field].clear
+              @reverse_indexes[field].clear
+            end
           end
 
-          def use_index?(record_class, where_clause)
-            @indexed_columns.key?(record_class) && get_index(record_class, where_clause).present?
+          def use_index?(normalized_where_clause)
+            get_indexes(normalized_where_clause).present?
           end
 
           private
 
-          def indexed?(record_class)
-            @indexed_columns.key?(record_class)
-          end
-
-          def get_keys(record_class, record)
-            @indexed_columns[record_class].map do |index|
-              arr = [record_class.name]
-              index.each do |key|
-                arr << key
-                arr << record[key]
-              end
-              arr
-            end
-          end
-
-          def get_index(record_class, where_clause)
-            @indexed_columns[record_class].find do |indexed_where|
-              where_clause.keys.size == indexed_where.size && (where_clause.keys.map(&:to_s) - indexed_where).empty?
-            end
+          def get_indexes(normalized_where_clause)
+            @indexed_columns & normalized_where_clause.keys
           end
         end
 
@@ -167,17 +169,28 @@ module Sequent
         #   Key corresponds to the name of the 'Record'
         #   Values contains list of lists on which columns to index.
         #   E.g. [[:first_index_column], [:another_index, :with_to_columns]]
-        def initialize(insert_with_csv_size = 50, indices = {})
+        def initialize(insert_with_csv_size = 50, indices = {}, default_indexed_columns = [:aggregate_id])
           @insert_with_csv_size = insert_with_csv_size
-          @record_store = Hash.new { |h, k| h[k] = Set.new }
-          @record_index = Index.new(indices)
+          @record_store = Hash.new { |h, k| h[k] = Set.new.compare_by_identity }
+          @record_index = Hash.new do |h, k|
+            h[k] = Index.new(default_indexed_columns.to_set & k.column_names.map(&:to_sym))
+          end
+
+          indices.each do |record_class, indexed_columns|
+            columns = indexed_columns.flatten(1).map(&:to_sym).to_set + default_indexed_columns
+            @record_index[record_class] = Index.new(columns & record_class.column_names.map(&:to_sym))
+          end
+
+          @record_defaults = Hash.new do |h, record_class|
+            h[record_class] = record_class.column_defaults.symbolize_keys
+          end
         end
 
         def update_record(record_class, event, where_clause = {aggregate_id: event.aggregate_id}, options = {})
           record = get_record!(record_class, where_clause)
-          record.updated_at = event.created_at if record.respond_to?(:updated_at)
+          record.updated_at = event.created_at if record.respond_to?(:updated_at=)
           yield record if block_given?
-          @record_index.update(record_class, record)
+          @record_index[record_class].update(record)
           update_sequence_number = if options.key?(:update_sequence_number)
                                      options[:update_sequence_number]
                                    else
@@ -187,41 +200,16 @@ module Sequent
         end
 
         def create_record(record_class, values)
-          column_names = record_class.column_names
-          values = record_class.column_defaults.with_indifferent_access.merge(values)
-          values.merge!(updated_at: values[:created_at]) if column_names.include?('updated_at')
-          struct_class_name = "#{record_class}Struct"
-          if self.class.struct_cache.key?(struct_class_name)
-            struct_class = self.class.struct_cache[struct_class_name]
-          else
-            # We create a struct on the fly.
-            # Since the replay happens in memory we implement the ==, eql? and hash methods
-            # to point to the same object. A record is the same if and only if they point to
-            # the same object. These methods are necessary since we use Set instead of [].
-            class_def = <<-EOD
-      #{struct_class_name} = Struct.new(*#{column_names.map(&:to_sym)})
-              class #{struct_class_name}
-                include InitStruct
-                def ==(other)
-                  self.equal?(other)
-                end
-                def hash
-                  self.object_id.hash
-                end
-              end
-            EOD
-            # rubocop:disable Security/Eval
-            eval(class_def.to_s)
-            # rubocop:enable Security/Eval
-            struct_class = ReplayOptimizedPostgresPersistor.const_get(struct_class_name)
-            self.class.struct_cache[struct_class_name] = struct_class
+          record = struct_cache[record_class].new(**values)
+          @record_defaults[record_class].each do |column, default|
+            record[column] = default unless values.include? column
           end
-          record = struct_class.new.set_values(values)
+          record.updated_at = values[:created_at] if record.respond_to?(:updated_at)
 
           yield record if block_given?
-          @record_store[record_class] << record
 
-          @record_index.add(record_class, record)
+          @record_store[record_class] << record
+          @record_index[record_class].add(record)
 
           record
         end
@@ -234,7 +222,7 @@ module Sequent
           record = get_record(record_class, values)
           record ||= create_record(record_class, values.merge(created_at: created_at))
           yield record if block_given?
-          @record_index.update(record_class, record)
+          @record_index[record_class].update(record)
           record
         end
 
@@ -260,15 +248,15 @@ module Sequent
 
         def delete_record(record_class, record)
           @record_store[record_class].delete(record)
-          @record_index.remove(record_class, record)
+          @record_index[record_class].remove(record)
         end
 
         def update_all_records(record_class, where_clause, updates)
           find_records(record_class, where_clause).each do |record|
             updates.each_pair do |k, v|
-              record[k.to_sym] = v
+              record[k] = v
             end
-            @record_index.update(record_class, record)
+            @record_index[record_class].update(record)
           end
         end
 
@@ -276,33 +264,41 @@ module Sequent
           records = find_records(record_class, where_clause)
           records.each do |record|
             yield record
-            @record_index.update(record_class, record)
+            @record_index[record_class].update(record)
           end
         end
 
         def do_with_record(record_class, where_clause)
           record = get_record!(record_class, where_clause)
           yield record
-          @record_index.update(record_class, record)
+          @record_index[record_class].update(record)
         end
 
         def find_records(record_class, where_clause)
-          if @record_index.use_index?(record_class, where_clause)
-            @record_index.find(record_class, where_clause)
-          else
-            @record_store[record_class].select do |record|
-              where_clause.all? do |k, v|
-                expected_value = v.is_a?(Symbol) ? v.to_s : v
-                actual_value = record[k.to_sym]
-                actual_value = actual_value.to_s if actual_value.is_a? Symbol
-                if expected_value.is_a?(Array)
-                  expected_value.include?(actual_value)
-                else
-                  actual_value == expected_value
-                end
+          where_clause = where_clause.symbolize_keys
+
+          indexed_columns = @record_index[record_class].indexed_columns
+          indexed_fields, non_indexed_fields = where_clause.partition { |field, _| indexed_columns.include? field }
+
+          candidate_records = if indexed_fields.present?
+                                @record_index[record_class].find(indexed_fields)
+                              else
+                                @record_store[record_class]
+                              end
+
+          return candidate_records.to_a if non_indexed_fields.empty?
+
+          candidate_records.select do |record|
+            non_indexed_fields.all? do |k, v|
+              expected_value = Persistors.normalize_symbols(v)
+              actual_value = Persistors.normalize_symbols(record[k])
+              if expected_value.is_a?(Array)
+                expected_value.include?(actual_value)
+              else
+                actual_value == expected_value
               end
             end
-          end.dup
+          end
         end
 
         def last_record(record_class, where_clause)
@@ -310,6 +306,10 @@ module Sequent
           results.empty? ? nil : results.last
         end
 
+        def prepare
+          # noop
+        end
+
         def commit
           @record_store.each do |clazz, records|
             @column_cache ||= {}
@@ -358,7 +358,7 @@ module Sequent
 
         def clear
           @record_store.clear
-          @record_index.clear
+          @record_index.values.each(&:clear)
         end
 
         private
@@ -366,12 +366,19 @@ module Sequent
         def cast_value_to_column_type(clazz, column_name, record)
           uncasted_value = ActiveModel::Attribute.from_database(
             column_name,
-            record[column_name.to_sym],
+            record[column_name],
             Sequent::ApplicationRecord.connection.lookup_cast_type_from_column(@column_cache[clazz.name][column_name]),
           ).value_for_database
           Sequent::ApplicationRecord.connection.type_cast(uncasted_value)
         end
       end
+
+      # Normalizes symbol values to strings (by using its name) while
+      # preserving all other values. This allows symbol/string
+      # indifferent comparisons.
+      def self.normalize_symbols(value)
+        value.is_a?(Symbol) ? value.name : value
+      end
     end
   end
 end

data/lib/sequent/core/projector.rb

@@ -88,6 +88,10 @@ module Sequent
       include Migratable
       extend ActiveSupport::DescendantsTracker
 
+      class << self
+        attr_accessor :abstract_class, :skip_autoregister
+      end
+
       def initialize(persistor = Sequent::Core::Persistors::ActiveRecordPersistor.new)
         ensure_valid!
         @persistor = persistor

data/lib/sequent/core/transactions/active_record_transaction_provider.rb

@@ -30,8 +30,9 @@ module Sequent
       #
       class ActiveRecordTransactionProvider
         def transactional(&block)
-          Sequent::ApplicationRecord.transaction(requires_new: true, &block)
+          result = Sequent::ApplicationRecord.transaction(requires_new: true, &block)
           after_commit_queue.pop.call until after_commit_queue.empty?
+          result
         ensure
           clear_after_commit_queue
         end

data/lib/sequent/core/workflow.rb

@@ -9,6 +9,10 @@ module Sequent
       include Helpers::MessageHandler
       extend ActiveSupport::DescendantsTracker
 
+      class << self
+        attr_accessor :abstract_class, :skip_autoregister
+      end
+
       def self.on(*args, **opts, &block)
         decorated_block = ->(event) do
           begin

data/lib/sequent/dry_run/dry_run.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative 'read_only_replay_optimized_postgres_persistor'
+require_relative 'view_schema'

data/lib/sequent/dry_run/read_only_replay_optimized_postgres_persistor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative '../core/persistors/replay_optimized_postgres_persistor'
+module Sequent
+  module DryRun
+    # Subclass of ReplayOptimizedPostgresPersistor
+    # This persistor does not persist anything. Mainly usefull for
+    # performance testing migrations.
+    class ReadOnlyReplayOptimizedPostgresPersistor < Core::Persistors::ReplayOptimizedPostgresPersistor
+      def prepare
+        @starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def commit
+        # Running in dryrun mode, not committing anything.
+        ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        elapsed = ending - @starting
+        count = @record_store.values.map(&:size).sum
+        Sequent.logger.info(
+          "dryrun: processed #{count} records in #{elapsed.round(2)} s (#{(count / elapsed).round(2)} records/s)",
+        )
+        clear
+      end
+    end
+  end
+end

data/lib/sequent/dry_run/view_schema.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative '../migrations/view_schema'
+require_relative 'read_only_replay_optimized_postgres_persistor'
+
+module Sequent
+  module DryRun
+    # Subclass of Migrations::ViewSchema to dry run a migration.
+    # This migration does not insert anything into the database, mainly usefull
+    # for performance testing migrations.
+    class ViewSchema < Migrations::ViewSchema
+      def migrate_dryrun(regex:, group_exponent: 3, limit: nil, offset: nil)
+        persistor = DryRun::ReadOnlyReplayOptimizedPostgresPersistor.new
+
+        projectors = Sequent::Core::Migratable.all.select { |p| p.replay_persistor.nil? && p.name.match(regex || /.*/) }
+        if projectors.present?
+          Sequent.logger.info "Dry run using the following projectors: #{projectors.map(&:name).join(', ')}"
+
+          starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          groups = groups(group_exponent: group_exponent, limit: limit, offset: offset)
+          replay!(persistor, projectors: projectors, groups: groups)
+          ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          Sequent.logger.info("Done migrate_dryrun for version #{Sequent.new_version} in #{ending - starting} s")
+        end
+      end
+
+      private
+
+      # override so no ids are inserted.
+      def insert_ids
+        ->(progress, done, ids) {}
+      end
+    end
+  end
+end

data/lib/sequent/generator/template_project/db/sequent_schema.rb

@@ -8,6 +8,7 @@ ActiveRecord::Schema.define do
     t.text "event_json", :null => false
     t.integer "command_record_id", :null => false
     t.integer "stream_record_id", :null => false
+    t.bigint "xact_id"
   end
 
   execute %Q{

data/lib/sequent/migrations/errors.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Sequent
+  module Migrations
+    class MigrationError < RuntimeError; end
+    class MigrationNotStarted < MigrationError; end
+    class MigrationDone < MigrationError; end
+    class ConcurrentMigration < MigrationError; end
+
+    class InvalidMigrationDefinition < MigrationError; end
+  end
+end

data/lib/sequent/migrations/migrations.rb

@@ -5,7 +5,6 @@ module Sequent
   end
 end
 
-require_relative 'migrate_events'
 require_relative 'projectors'
 require_relative 'view_schema'
 require_relative 'functions'

data/lib/sequent/migrations/planner.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'errors'
+
 module Sequent
   module Migrations
     class Planner
@@ -69,10 +71,10 @@ module Sequent
           .each_with_index
           .select { |migration, _index| migration.instance_of?(AlterTable) }
           .select do |migration, index|
-                       migrations
-                         .slice((index + 1)..-1)
-                         .find { |m| m.instance_of?(ReplayTable) && m.record_class == migration.record_class }
-                     end.map(&:first)
+            migrations
+              .slice((index + 1)..-1)
+              .find { |m| m.instance_of?(ReplayTable) && m.record_class == migration.record_class }
+          end.map(&:first)
       end
 
       def remove_redundancy(grouped_migrations)
@@ -100,7 +102,8 @@ module Sequent
       def map_to_migrations(migrations)
         migrations.reduce({}) do |memo, (version, ms)|
           unless ms.is_a?(Array)
-            fail "Declared migrations for version #{version} must be an Array. For example: {'3' => [FooProjector]}"
+            fail InvalidMigrationDefinition,
+                 "Declared migrations for version #{version} must be an Array. For example: {'3' => [FooProjector]}"
           end
 
           memo[version] = ms.flat_map do |migration|
@@ -109,14 +112,15 @@ module Sequent
                 #{Sequent.configuration.migration_sql_files_directory}/#{migration.table_name}_#{version}.sql
               EOS
               unless File.exist?(alter_table_sql_file_name)
-                fail "Missing file #{alter_table_sql_file_name} to apply for version #{version}"
+                fail InvalidMigrationDefinition,
+                     "Missing file #{alter_table_sql_file_name} to apply for version #{version}"
               end
 
               migration.copy(version)
             elsif migration < Sequent::Projector
               migration.managed_tables.map { |table| ReplayTable.create(table, version) }
             else
-              fail "Unknown Migration #{migration}"
+              fail InvalidMigrationDefinition, "Unknown Migration #{migration}"
             end
           end