sequent 7.1.1 → 8.0.0

data/lib/sequent/generator/template_project/spec/spec_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/setup'
 Bundler.setup
 
@@ -8,12 +10,7 @@ require 'database_cleaner'
 
 require_relative '../my_app'
 
-db_config = Sequent::Support::Database.read_config('test')
-Sequent::Support::Database.establish_connection(db_config)
-
-Sequent::Support::Database.drop_schema!(Sequent.configuration.view_schema_name)
-
-Sequent::Migrations::ViewSchema.new(db_config: db_config).create_view_tables
+Sequent::Test::DatabaseHelpers.maintain_test_database_schema(env: 'test')
 
 module DomainTests
   def self.included(base)
@@ -23,7 +20,7 @@ end
 
 RSpec.configure do |config|
   config.include Sequent::Test::CommandHandlerHelpers
-  config.include DomainTests, file_path: /spec\/lib/
+  config.include DomainTests, file_path: %r{/spec\/lib/}
 
   # Domain tests run with a clean sequent configuration and the in memory FakeEventStore
   config.around :each, :domain_tests do |example|

data/lib/sequent/generator.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-require_relative './generator/generator'
+require_relative 'generator/generator'

data/lib/sequent/internal/aggregate_type.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require_relative '../application_record'
+
+module Sequent
+  module Internal
+    class AggregateType < Sequent::ApplicationRecord
+      self.inheritance_column = nil
+    end
+  end
+end

data/lib/sequent/internal/command_type.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require_relative '../application_record'
+
+module Sequent
+  module Internal
+    class CommandType < Sequent::ApplicationRecord
+      self.inheritance_column = nil
+    end
+  end
+end

data/lib/sequent/internal/event_type.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require_relative '../application_record'
+
+module Sequent
+  module Internal
+    class EventType < Sequent::ApplicationRecord
+      self.inheritance_column = nil
+    end
+  end
+end

data/lib/sequent/internal/internal.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative 'aggregate_type'
+require_relative 'command_type'
+require_relative 'event_type'
+require_relative 'partitioned_aggregate'
+require_relative 'partitioned_command'
+require_relative 'partitioned_event'
+
+module Sequent
+  module Internal
+  end
+  private_constant :Internal
+end

data/lib/sequent/internal/partitioned_aggregate.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require_relative '../application_record'
+
+module Sequent
+  module Internal
+    class PartitionedAggregate < Sequent::ApplicationRecord
+      self.table_name = :aggregates
+      self.primary_key = %i[aggregate_id]
+
+      belongs_to :aggregate_type
+      if Gem.loaded_specs['activerecord'].version < Gem::Version.create('7.2')
+        has_many :partitioned_events,
+                 inverse_of: :partitioned_aggregate,
+                 primary_key: %w[events_partition_key aggregate_id],
+                 query_constraints: %w[partition_key aggregate_id]
+      else
+        has_many :partitioned_events,
+                 inverse_of: :partitioned_aggregate,
+                 primary_key: %i[events_partition_key aggregate_id],
+                 foreign_key: %i[partition_key aggregate_id]
+      end
+    end
+  end
+end

data/lib/sequent/internal/partitioned_command.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require_relative '../application_record'
+
+module Sequent
+  module Internal
+    class PartitionedCommand < Sequent::ApplicationRecord
+      self.table_name = :commands
+
+      belongs_to :command_type
+      has_many :partitioned_events,
+               inverse_of: :partitioned_command
+    end
+  end
+end

data/lib/sequent/internal/partitioned_event.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require_relative '../application_record'
+
+module Sequent
+  module Internal
+    class PartitionedEvent < Sequent::ApplicationRecord
+      self.table_name = :events
+      self.primary_key = %i[partition_key aggregate_id sequence_number]
+
+      belongs_to :event_type
+      belongs_to :partitioned_command,
+                 inverse_of: :partitioned_events,
+                 foreign_key: :command_id
+      if Gem.loaded_specs['activerecord'].version < Gem::Version.create('7.2')
+        belongs_to :partitioned_aggregate,
+                   inverse_of: :partitioned_events,
+                   primary_key: %w[partition_key aggregate_id],
+                   query_constraints: %w[events_partition_key aggregate_id]
+      else
+        belongs_to :partitioned_aggregate,
+                   inverse_of: :partitioned_events,
+                   primary_key: %w[partition_key aggregate_id],
+                   foreign_key: %w[events_partition_key aggregate_id]
+      end
+    end
+  end
+end

data/lib/sequent/migrations/grouper.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Sequent
+  module Migrations
+    module Grouper
+      GroupEndpoint = Data.define(:partition_key, :aggregate_id) do
+        def <=>(other)
+          return unless other.is_a?(self.class)
+
+          [partition_key, aggregate_id] <=> [other.partition_key, other.aggregate_id]
+        end
+
+        include Comparable
+
+        def to_s
+          "(#{partition_key}, #{aggregate_id})"
+        end
+      end
+
+      # Generate approximately equally sized groups based on the
+      # events partition keys and the number of events per partition
+      # key.  Each group is defined by a lower bound (partition-key,
+      # aggregate-id) and upper bound (partition-key, aggregate-id)
+      # (inclusive).
+      #
+      # For splitting a partition into equal sized groups the
+      # assumption is made that aggregate-ids and their events are
+      # equally distributed.
+      def self.group_partitions(partitions, target_group_size)
+        return [] unless partitions.present?
+
+        partitions = partitions.sort.map do |key, count|
+          PartitionData.new(key:, original_size: count, remaining_size: count, lower_bound: 0)
+        end
+
+        partition = partitions.shift
+        current_start = GroupEndpoint.new(partition.key, LOWEST_UUID)
+        current_size = 0
+
+        result = []
+        while partition.present?
+          if current_size + partition.remaining_size < target_group_size
+            current_size += partition.remaining_size
+            if partitions.empty?
+              result << (current_start..GroupEndpoint.new(partition.key, HIGHEST_UUID))
+              break
+            end
+            partition = partitions.shift
+          elsif current_size + partition.remaining_size == target_group_size
+            result << (current_start..GroupEndpoint.new(partition.key, HIGHEST_UUID))
+
+            partition = partitions.shift
+            break unless partition
+
+            current_start = GroupEndpoint.new(partition.key, LOWEST_UUID)
+            current_size = 0
+          else
+            taken = target_group_size - current_size
+            upper_bound = partition.lower_bound + (UUID_COUNT * taken / partition.original_size)
+
+            result << (current_start..GroupEndpoint.new(partition.key, number_to_uuid(upper_bound - 1)))
+
+            remaining_size = partition.remaining_size - taken
+            partition = partition.with(remaining_size:, lower_bound: upper_bound)
+            current_start = GroupEndpoint.new(partition.key, number_to_uuid(upper_bound))
+            current_size = 0
+          end
+        end
+        result
+      end
+
+      PartitionData = Data.define(:key, :original_size, :remaining_size, :lower_bound)
+
+      def self.number_to_uuid(number)
+        fail ArgumentError, number unless (0..UUID_COUNT - 1).include? number
+
+        s = format('%032x', number)
+        "#{s[0..7]}-#{s[8..11]}-#{s[12..15]}-#{s[16..19]}-#{s[20..]}"
+      end
+
+      def self.uuid_to_number(uuid)
+        Integer(uuid.gsub('-', ''), 16)
+      end
+
+      UUID_COUNT = 2**128
+      LOWEST_UUID = number_to_uuid(0)
+      HIGHEST_UUID = number_to_uuid(UUID_COUNT - 1)
+    end
+  end
+end

data/lib/sequent/migrations/sequent_schema.rb

@@ -19,7 +19,8 @@ module Sequent
           Sequent::Support::Database.establish_connection(db_config)
 
           event_store_schema = Sequent.configuration.event_store_schema_name
-          schema_exists = Sequent::Support::Database.schema_exists?(event_store_schema)
+          event_records_table = Sequent.configuration.event_record_class.table_name
+          schema_exists = Sequent::Support::Database.schema_exists?(event_store_schema, event_records_table)
 
           FAIL_IF_EXISTS.call(event_store_schema) if schema_exists && fail_if_exists
           return if schema_exists

data/lib/sequent/migrations/view_schema.rb

@@ -8,11 +8,12 @@ require_relative '../support/database'
 require_relative '../sequent'
 require_relative '../util/timer'
 require_relative '../util/printer'
-require_relative './projectors'
+require_relative 'projectors'
 require_relative 'planner'
 require_relative 'executor'
 require_relative 'sql'
 require_relative 'versions'
+require_relative 'grouper'
 
 module Sequent
   module Migrations
@@ -60,16 +61,6 @@ module Sequent
     #
     # end
     class ViewSchema
-      # Corresponds with the index on aggregate_id column in the event_records table
-      #
-      # Since we replay in batches of the first 3 chars of the uuid we created an index on
-      # these 3 characters. Hence the name ;-)
-      #
-      # This also means that the online replay is divided up into 16**3 groups
-      # This might seem a lot for starting event store, but when you will get more
-      # events, you will see that this is pretty good partitioned.
-      LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE = 3
-
       include Sequent::Util::Timer
       include Sequent::Util::Printer
       include Sql
@@ -143,11 +134,10 @@ module Sequent
       # Utility method that replays events for all managed_tables from all Sequent::Core::Projector's
       #
       # This method is mainly useful in test scenario's or development tasks
-      def replay_all!(group_exponent: 1)
+      def replay_all!
         replay!(
           Sequent.configuration.online_replay_persistor_class.new,
           projectors: Core::Migratable.projectors,
-          groups: groups(group_exponent: group_exponent),
         )
       end
 
@@ -205,7 +195,6 @@ module Sequent
         if plan.projectors.any?
           replay!(
             Sequent.configuration.online_replay_persistor_class.new,
-            groups: groups,
             maximum_xact_id_exclusive: Versions.running.first.xmin_xact_id,
           )
         end
@@ -216,17 +205,15 @@ module Sequent
           Versions.end_online!(Sequent.new_version)
         end
         Sequent.logger.info("Done migrate_online for version #{Sequent.new_version}")
-      rescue ConcurrentMigration
-        # Do not rollback the migration when this is a concurrent migration as the other one is running
-        raise
-      rescue InvalidMigrationDefinition
-        # Do not rollback the migration when since there is nothing to rollback
+      rescue ConcurrentMigration, InvalidMigrationDefinition
+        # ConcurrentMigration: Do not rollback the migration when this is a concurrent migration
+        #                      as the other one is running
+        # InvalidMigrationDefinition: Do not rollback the migration when since there is nothing to rollback
         raise
       rescue Exception => e # rubocop:disable Lint/RescueException
         rollback_migration
         raise e
       end
-
       ##
       # Last part of a view schema migration
       #
@@ -259,7 +246,6 @@ module Sequent
         if plan.projectors.any?
           replay!(
             Sequent.configuration.offline_replay_persistor_class.new,
-            groups: groups(group_exponent: 1),
             minimum_xact_id_inclusive: Versions.running.first.xmin_xact_id,
           )
         end
@@ -309,18 +295,33 @@ module Sequent
 
       def replay!(
         replay_persistor,
-        groups:,
         projectors: plan.projectors,
         minimum_xact_id_inclusive: nil,
         maximum_xact_id_exclusive: nil
       )
-        logger.info "groups: #{groups.size}"
+        event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
+        group_target_size = Sequent.configuration.replay_group_target_size
+        event_type_ids = Internal::EventType.where(type: event_types).pluck(:id)
+
+        partitions_query = Internal::PartitionedEvent.where(event_type_id: event_type_ids)
+        partitions_query = xact_id_filter(partitions_query, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
+
+        partitions = partitions_query.group(:partition_key).order(:partition_key).count
+        event_count = partitions.values.sum
+
+        groups = Sequent::Migrations::Grouper.group_partitions(partitions, group_target_size)
+
+        if groups.empty?
+          groups = [nil..nil]
+        else
+          groups.prepend(nil..groups.first.begin)
+          groups.append(groups.last.end..nil)
+        end
 
         with_sequent_config(replay_persistor, projectors) do
-          logger.info 'Start replaying events'
+          logger.info "Start replaying #{event_count} events in #{groups.size} groups"
 
-          time("#{groups.size} groups replayed") do
-            event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
+          time("#{event_count} events in #{groups.size} groups replayed") do
             disconnect!
 
             @connected = false
@@ -328,24 +329,23 @@ module Sequent
             result = Parallel.map_with_index(
               groups,
               in_processes: Sequent.configuration.number_of_replay_processes,
-            ) do |aggregate_prefixes, index|
+            ) do |group, index|
               @connected ||= establish_connection
               msg = <<~EOS.chomp
-                Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{groups.size} replayed
+                Group #{group} (#{index + 1}/#{groups.size}) replayed
               EOS
               time(msg) do
                 replay_events(
-                  aggregate_prefixes,
-                  event_types,
-                  minimum_xact_id_inclusive,
-                  maximum_xact_id_exclusive,
+                  -> {
+                    event_stream(group, event_type_ids, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
+                  },
                   replay_persistor,
                   &on_progress
                 )
               end
               nil
             rescue StandardError => e
-              logger.error "Replaying failed for ids: ^#{aggregate_prefixes.first} - #{aggregate_prefixes.last}"
+              logger.error "Replaying failed for group: #{group}"
               logger.error '+++++++++++++++ ERROR +++++++++++++++'
               recursively_print(e)
               raise Parallel::Kill # immediately kill all sub-processes
@@ -357,19 +357,14 @@ module Sequent
       end
 
       def replay_events(
-        aggregate_prefixes,
-        event_types,
-        minimum_xact_id_inclusive,
-        maximum_xact_id_exclusive,
+        get_events,
         replay_persistor,
         &on_progress
       )
         Sequent.configuration.event_store.replay_events_from_cursor(
           block_size: 1000,
-          get_events: -> {
-            event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
-          },
-          on_progress: on_progress,
+          get_events:,
+          on_progress:,
         )
 
         replay_persistor.commit
@@ -387,30 +382,6 @@ module Sequent
         Versions.rollback!(Sequent.new_version)
       end
 
-      def groups(group_exponent: 3, limit: nil, offset: nil)
-        number_of_groups = 16**group_exponent
-        groups = groups_of_aggregate_id_prefixes(number_of_groups)
-        groups = groups.drop(offset) unless offset.nil?
-        groups = groups.take(limit) unless limit.nil?
-        groups
-      end
-
-      def groups_of_aggregate_id_prefixes(number_of_groups)
-        all_prefixes = (0...16**LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE).to_a.map do |i|
-          i.to_s(16)
-        end
-        all_prefixes = all_prefixes.map { |s| s.length == 3 ? s : "#{'0' * (3 - s.length)}#{s}" }
-
-        logger.info "Number of groups #{number_of_groups}"
-
-        logger.debug "Prefixes: #{all_prefixes.length}"
-        if number_of_groups > all_prefixes.length
-          fail "Can not have more groups #{number_of_groups} than number of prefixes #{all_prefixes.length}"
-        end
-
-        all_prefixes.each_slice(all_prefixes.length / number_of_groups).to_a
-      end
-
       def in_view_schema(&block)
         Sequent::Support::Database.with_schema_search_path(view_schema, db_config, &block)
       end
@@ -451,27 +422,55 @@ module Sequent
         Sequent::Configuration.restore(old_config)
       end
 
-      def event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
-        fail ArgumentError, 'aggregate_prefixes is mandatory' unless aggregate_prefixes.present?
+      def event_stream(group, event_type_ids, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
+        fail ArgumentError, 'group is mandatory' if group.nil?
 
-        event_stream = Sequent.configuration.event_record_class.where(event_type: event_types)
-        event_stream = event_stream.where(<<~SQL, aggregate_prefixes)
-          substring(aggregate_id::text from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?)
-        SQL
-        if minimum_xact_id_inclusive && maximum_xact_id_exclusive
+        event_stream = Internal::PartitionedEvent
+          .joins('JOIN event_types ON events.event_type_id = event_types.id')
+          .where(
+            event_type_id: event_type_ids,
+          )
+        if group.begin && group.end
+          event_stream = event_stream.where(
+            '(events.partition_key, events.aggregate_id) BETWEEN (?, ?) AND (?, ?)',
+            group.begin.partition_key,
+            group.begin.aggregate_id,
+            group.end.partition_key,
+            group.end.aggregate_id,
+          )
+        elsif group.end
+          event_stream = event_stream.where(
+            '(events.partition_key, events.aggregate_id) < (?, ?)',
+            group.end.partition_key,
+            group.end.aggregate_id,
+          )
+        elsif group.begin
           event_stream = event_stream.where(
+            '(events.partition_key, events.aggregate_id) > (?, ?)',
+            group.begin.partition_key,
+            group.begin.aggregate_id,
+          )
+        end
+        event_stream = xact_id_filter(event_stream, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
+        event_stream
+          .order('events.partition_key', 'events.aggregate_id', 'events.sequence_number')
+          .select('event_types.type AS event_type, enrich_event_json(events) AS event_json')
+      end
+
+      def xact_id_filter(events_query, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
+        if minimum_xact_id_inclusive && maximum_xact_id_exclusive
+          events_query.where(
             'xact_id >= ? AND xact_id < ?',
             minimum_xact_id_inclusive,
             maximum_xact_id_exclusive,
           )
         elsif minimum_xact_id_inclusive
-          event_stream = event_stream.where('xact_id >= ?', minimum_xact_id_inclusive)
+          events_query.where('xact_id >= ?', minimum_xact_id_inclusive)
         elsif maximum_xact_id_exclusive
-          event_stream = event_stream.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive)
+          events_query.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive)
+        else
+          events_query
         end
-        event_stream
-          .order('aggregate_id ASC, sequence_number ASC')
-          .select('id, event_type, event_json, sequence_number')
       end
 
       ## shortcut methods

data/lib/sequent/rake/migration_tasks.rb

@@ -125,6 +125,11 @@ module Sequent
               end
             end
 
+            desc 'Aborts if a migration is pending'
+            task abort_if_pending_migrations: [:create_and_migrate_sequent_view_schema] do
+              abort if Sequent.new_version != Sequent::Migrations::Versions.current_version
+            end
+
             desc <<-EOS
               Shows the current status of the migrations
             EOS
@@ -192,18 +197,15 @@ module Sequent
 
               Pass a regular expression as parameter to select the projectors to run, otherwise all projectors are selected.
             EOS
-            task :dryrun, %i[regex group_exponent limit offset] => ['sequent:init', :init] do |_task, args|
+            task :dryrun, %i[regex group_target_size] => ['sequent:init', :init] do |_task, args|
               ensure_sequent_env_set!
 
               db_config = Sequent::Support::Database.read_config(@env)
               view_schema = Sequent::DryRun::ViewSchema.new(db_config: db_config)
 
-              view_schema.migrate_dryrun(
-                regex: args[:regex],
-                group_exponent: (args[:group_exponent] || 3).to_i,
-                limit: args[:limit]&.to_i,
-                offset: args[:offset]&.to_i,
-              )
+              Sequent.configuration.replay_group_target_size = group_target_size
+
+              view_schema.migrate_dryrun(regex: args[:regex])
             end
           end
 
@@ -213,31 +215,54 @@ module Sequent
             EOS
             task :init
 
-            task :set_snapshot_threshold, %i[aggregate_type threshold] => ['sequent:init', :init] do |_t, args|
-              aggregate_type = args['aggregate_type']
-              threshold = args['threshold']
+            task :connect, ['sequent:init', :init, :set_env_var] do
+              ensure_sequent_env_set!
+
+              Sequent.configuration.command_handlers << Sequent::Core::AggregateSnapshotter.new
+
+              db_config = Sequent::Support::Database.read_config(@env)
+              Sequent::Support::Database.establish_connection(db_config)
+            end
+
+            desc <<~EOS
+              Takes up-to `limit` snapshots, starting with the highest priority aggregates (based on snapshot outdated time and number of events)
+            EOS
+            task :take_snapshots, %i[limit] => :connect do |_t, args|
+              limit = args['limit']&.to_i
 
-              unless aggregate_type
+              unless limit
                 fail ArgumentError,
-                     'usage rake sequent:snapshots:set_snapshot_threshold[AggregegateType,threshold]'
+                     'usage rake sequent:snapshots:take_snapshots[limit]'
               end
-              unless threshold
+
+              aggregate_ids = Sequent.configuration.event_store.select_aggregates_for_snapshotting(limit:)
+
+              Sequent.logger.info "Taking #{aggregate_ids.size} snapshots"
+              aggregate_ids.each do |aggregate_id|
+                Sequent.command_service.execute_commands(Sequent::Core::TakeSnapshot.new(aggregate_id:))
+              end
+            end
+
+            desc <<~EOS
+              Takes a new snapshot for the aggregate specified by `aggregate_id`
+            EOS
+            task :take_snapshot, %i[aggregate_id] => :connect do |_t, args|
+              aggregate_id = args['aggregate_id']
+
+              unless aggregate_id
                 fail ArgumentError,
-                     'usage rake sequent:snapshots:set_snapshot_threshold[AggregegateType,threshold]'
+                     'usage rake sequent:snapshots:take_snapshot[aggregate_id]'
               end
 
-              execute <<~EOS
-                UPDATE #{Sequent.configuration.stream_record_class} SET snapshot_threshold = #{threshold.to_i} WHERE aggregate_type = '#{aggregate_type}'
-              EOS
+              Sequent.command_service.execute_commands(Sequent::Core::TakeSnapshot.new(aggregate_id:))
             end
 
-            task delete_all: ['sequent:init', :init] do
-              result = Sequent::ApplicationRecord
-                .connection
-                .execute(<<~EOS)
-                  DELETE FROM #{Sequent.configuration.event_record_class.table_name} WHERE event_type = 'Sequent::Core::SnapshotEvent'
-                EOS
-              Sequent.logger.info "Deleted #{result.cmd_tuples} aggregate snapshots from the event store"
+            desc <<~EOS
+              Delete all aggregate snapshots, which can negatively impact performance of a running system.
+            EOS
+            task delete_all: :connect do
+              Sequent.configuration.event_store.delete_all_snapshots
+              Sequent.logger.info 'Deleted all aggregate snapshots from the event store'
             end
           end
         end

data/lib/sequent/sequent.rb

@@ -8,6 +8,7 @@ require_relative 'core/aggregate_root'
 require_relative 'core/projector'
 require_relative 'core/workflow'
 require_relative 'core/value_object'
+require_relative 'internal/internal'
 require_relative 'migrations/migrations'
 
 module Sequent