sequent 6.0.1 → 7.1.0

data/lib/sequent/migrations/versions.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Sequent
+  module Migrations
+    class Versions < Sequent::ApplicationRecord
+      MIGRATE_ONLINE_RUNNING = 1
+      MIGRATE_ONLINE_FINISHED = 2
+      MIGRATE_OFFLINE_RUNNING = 3
+      DONE = nil
+
+      def self.migration_sql
+        <<~SQL.chomp
+          CREATE TABLE IF NOT EXISTS #{table_name} (version integer NOT NULL, CONSTRAINT version_pk PRIMARY KEY(version));
+          ALTER TABLE #{table_name} drop constraint if exists only_one_running;
+          ALTER TABLE #{table_name} ADD COLUMN IF NOT EXISTS status INTEGER DEFAULT NULL CONSTRAINT only_one_running CHECK (status in (1,2,3));
+          ALTER TABLE #{table_name} ADD COLUMN IF NOT EXISTS xmin_xact_id BIGINT;
+          DROP INDEX IF EXISTS single_migration_running;
+          CREATE UNIQUE INDEX single_migration_running ON #{table_name} ((status * 0)) where status is not null;
+        SQL
+      end
+
+      scope :done, -> { where(status: DONE) }
+      scope :running,
+            -> {
+              where(status: [MIGRATE_ONLINE_RUNNING, MIGRATE_ONLINE_FINISHED, MIGRATE_OFFLINE_RUNNING])
+            }
+
+      def self.current_version
+        done.latest_version || 0
+      end
+
+      def self.version_currently_migrating
+        running.latest_version
+      end
+
+      def self.latest_version
+        latest&.version
+      end
+
+      def self.latest
+        order('version desc').limit(1).first
+      end
+
+      def self.start_online!(new_version)
+        create!(version: new_version, status: MIGRATE_ONLINE_RUNNING, xmin_xact_id: current_snapshot_xmin_xact_id)
+      rescue ActiveRecord::RecordNotUnique
+        raise ConcurrentMigration, "Migration for version #{new_version} is already running"
+      end
+
+      def self.end_online!(new_version)
+        find_by!(version: new_version, status: MIGRATE_ONLINE_RUNNING).update(status: MIGRATE_ONLINE_FINISHED)
+      end
+
+      def self.rollback!(new_version)
+        running.where(version: new_version).delete_all
+      end
+
+      def self.start_offline!(new_version)
+        current_migration = find_by(version: new_version)
+        fail MigrationNotStarted if current_migration.blank?
+
+        current_migration.with_lock('FOR UPDATE NOWAIT') do
+          current_migration.reload
+          fail MigrationDone if current_migration.status.nil?
+          fail ConcurrentMigration if current_migration.status != MIGRATE_ONLINE_FINISHED
+
+          current_migration.update(status: MIGRATE_OFFLINE_RUNNING)
+        end
+      rescue ActiveRecord::LockWaitTimeout
+        raise ConcurrentMigration
+      end
+
+      def self.end_offline!(new_version)
+        find_by!(version: new_version, status: MIGRATE_OFFLINE_RUNNING).update(status: DONE)
+      end
+
+      def self.current_snapshot_xmin_xact_id
+        connection.execute('SELECT pg_snapshot_xmin(pg_current_snapshot())::text::bigint AS xmin').first['xmin']
+      end
+    end
+  end
+end

data/lib/sequent/migrations/view_schema.rb

@@ -3,6 +3,7 @@
 require 'parallel'
 require 'postgresql_cursor'
 
+require_relative 'errors'
 require_relative '../support/database'
 require_relative '../sequent'
 require_relative '../util/timer'
@@ -11,11 +12,10 @@ require_relative './projectors'
 require_relative 'planner'
 require_relative 'executor'
 require_relative 'sql'
+require_relative 'versions'
 
 module Sequent
   module Migrations
-    class MigrationError < RuntimeError; end
-
     ##
     # ViewSchema is used for migration of you view_schema. For instance
     # when you create new Projectors or change existing Projectors.
@@ -74,9 +74,6 @@ module Sequent
       include Sequent::Util::Printer
       include Sql
 
-      class Versions < Sequent::ApplicationRecord; end
-      class ReplayedIds < Sequent::ApplicationRecord; end
-
       attr_reader :view_schema, :db_config, :logger
 
       class << self
@@ -111,7 +108,7 @@ module Sequent
       ##
       # Returns the current version from the database
       def current_version
-        Versions.order('version desc').limit(1).first&.version || 0
+        Versions.current_version
       end
 
       ##
@@ -150,7 +147,7 @@ module Sequent
         replay!(
           Sequent.configuration.online_replay_persistor_class.new,
           projectors: Core::Migratable.projectors,
-          group_exponent: group_exponent,
+          groups: groups(group_exponent: group_exponent),
         )
       end
 
@@ -160,14 +157,7 @@ module Sequent
       # This method is mainly useful during an initial setup of the view schema
       def create_view_schema_if_not_exists
         exec_sql(%(CREATE SCHEMA IF NOT EXISTS #{view_schema}))
-        in_view_schema do
-          exec_sql(<<~SQL.chomp)
-            CREATE TABLE IF NOT EXISTS #{Versions.table_name} (version integer NOT NULL, CONSTRAINT version_pk PRIMARY KEY(version))
-          SQL
-          exec_sql(<<~SQL.chomp)
-            CREATE TABLE IF NOT EXISTS #{ReplayedIds.table_name} (event_id bigint NOT NULL, CONSTRAINT event_id_pk PRIMARY KEY(event_id))
-          SQL
-        end
+        migrate_metadata_tables
       end
 
       def plan
@@ -194,27 +184,45 @@ module Sequent
       #
       # If anything fails an exception is raised and everything is rolled back
       #
+      # @raise ConcurrentMigrationError if migration is already running
       def migrate_online
+        ensure_valid_plan!
+        migrate_metadata_tables
+
         return if Sequent.new_version == current_version
 
         ensure_version_correct!
 
+        Sequent.logger.info("Start migrate_online for version #{Sequent.new_version}")
+
         in_view_schema do
-          truncate_replay_ids_table!
+          Versions.start_online!(Sequent.new_version)
 
           drop_old_tables(Sequent.new_version)
           executor.execute_online(plan)
         end
 
-        replay!(Sequent.configuration.online_replay_persistor_class.new) if plan.projectors.any?
+        if plan.projectors.any?
+          replay!(
+            Sequent.configuration.online_replay_persistor_class.new,
+            groups: groups,
+            maximum_xact_id_exclusive: Versions.running.first.xmin_xact_id,
+          )
+        end
 
         in_view_schema do
           executor.create_indexes_after_execute_online(plan)
+          executor.reset_table_names(plan)
+          Versions.end_online!(Sequent.new_version)
         end
-        executor.reset_table_names(plan)
-        # rubocop:disable Lint/RescueException
-      rescue Exception => e
-        # rubocop:enable Lint/RescueException
+        Sequent.logger.info("Done migrate_online for version #{Sequent.new_version}")
+      rescue ConcurrentMigration
+        # Do not rollback the migration when this is a concurrent migration as the other one is running
+        raise
+      rescue InvalidMigrationDefinition
+        # Do not rollback the migration when since there is nothing to rollback
+        raise
+      rescue Exception => e # rubocop:disable Lint/RescueException
         rollback_migration
         raise e
       end
@@ -232,7 +240,7 @@ module Sequent
       # 2.1 Rename current tables with the +current version+ as SUFFIX
       # 2.2 Rename the new tables and remove the +new version+ suffix
       # 2.3 Add the new version in the +Versions+ table
-      # 3. Performs cleanup of replayed event ids
+      # 3. Update the versions table to complete the migration
       #
       # If anything fails an exception is raised and everything is rolled back
       #
@@ -242,6 +250,8 @@ module Sequent
         return if Sequent.new_version == current_version
 
         ensure_version_correct!
+        in_view_schema { Versions.start_offline!(Sequent.new_version) }
+        Sequent.logger.info("Start migrate_offline for version #{Sequent.new_version}")
 
         executor.set_table_names_to_new_version(plan)
 
@@ -249,8 +259,8 @@ module Sequent
         if plan.projectors.any?
           replay!(
             Sequent.configuration.offline_replay_persistor_class.new,
-            exclude_ids: true,
-            group_exponent: 1,
+            groups: groups(group_exponent: 1),
+            minimum_xact_id_inclusive: Versions.running.first.xmin_xact_id,
           )
         end
 
@@ -259,22 +269,33 @@ module Sequent
             # 2.1, 2.2
             executor.execute_offline(plan, current_version)
             # 2.3 Create migration record
-            Versions.create!(version: Sequent.new_version)
+            Versions.end_offline!(Sequent.new_version)
           end
-
-          # 3. Truncate replayed ids
-          truncate_replay_ids_table!
         end
         logger.info "Migrated to version #{Sequent.new_version}"
-        # rubocop:disable Lint/RescueException
-      rescue Exception => e
-        # rubocop:enable Lint/RescueException
+      rescue ConcurrentMigration
+        raise
+      rescue MigrationDone
+        # no-op same as Sequent.new_version == current_version
+      rescue Exception => e # rubocop:disable Lint/RescueException
         rollback_migration
         raise e
       end
 
       private
 
+      def ensure_valid_plan!
+        plan
+      end
+
+      def migrate_metadata_tables
+        Sequent::ApplicationRecord.transaction do
+          in_view_schema do
+            exec_sql([Versions.migration_sql].join("\n"))
+          end
+        end
+      end
+
       def ensure_version_correct!
         create_view_schema_if_not_exists
         new_version = Sequent.new_version
@@ -285,19 +306,22 @@ module Sequent
         end
       end
 
-      def replay!(replay_persistor, projectors: plan.projectors, exclude_ids: false, group_exponent: 3)
-        logger.info "group_exponent: #{group_exponent.inspect}"
+      def replay!(
+        replay_persistor,
+        groups:,
+        projectors: plan.projectors,
+        minimum_xact_id_inclusive: nil,
+        maximum_xact_id_exclusive: nil
+      )
+        logger.info "groups: #{groups.size}"
 
         with_sequent_config(replay_persistor, projectors) do
           logger.info 'Start replaying events'
 
-          time("#{16**group_exponent} groups replayed") do
+          time("#{groups.size} groups replayed") do
             event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
             disconnect!
 
-            number_of_groups = 16**group_exponent
-            groups = groups_of_aggregate_id_prefixes(number_of_groups)
-
             @connected = false
             # using `map_with_index` because https://github.com/grosser/parallel/issues/175
             result = Parallel.map_with_index(
@@ -306,10 +330,17 @@ module Sequent
             ) do |aggregate_prefixes, index|
               @connected ||= establish_connection
               msg = <<~EOS.chomp
-                Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{number_of_groups} replayed
+                Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{groups.size} replayed
               EOS
               time(msg) do
-                replay_events(aggregate_prefixes, event_types, exclude_ids, replay_persistor, &insert_ids)
+                replay_events(
+                  aggregate_prefixes,
+                  event_types,
+                  minimum_xact_id_inclusive,
+                  maximum_xact_id_exclusive,
+                  replay_persistor,
+                  &on_progress
+                )
               end
               nil
             rescue StandardError => e
@@ -324,10 +355,19 @@ module Sequent
         end
       end
 
-      def replay_events(aggregate_prefixes, event_types, exclude_already_replayed, replay_persistor, &on_progress)
+      def replay_events(
+        aggregate_prefixes,
+        event_types,
+        minimum_xact_id_inclusive,
+        maximum_xact_id_exclusive,
+        replay_persistor,
+        &on_progress
+      )
         Sequent.configuration.event_store.replay_events_from_cursor(
           block_size: 1000,
-          get_events: -> { event_stream(aggregate_prefixes, event_types, exclude_already_replayed) },
+          get_events: -> {
+            event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
+          },
           on_progress: on_progress,
         )
 
@@ -342,12 +382,16 @@ module Sequent
         establish_connection
         drop_old_tables(Sequent.new_version)
 
-        truncate_replay_ids_table!
         executor.reset_table_names(plan)
+        Versions.rollback!(Sequent.new_version)
       end
 
-      def truncate_replay_ids_table!
-        exec_sql("truncate table #{ReplayedIds.table_name}")
+      def groups(group_exponent: 3, limit: nil, offset: nil)
+        number_of_groups = 16**group_exponent
+        groups = groups_of_aggregate_id_prefixes(number_of_groups)
+        groups = groups.drop(offset) unless offset.nil?
+        groups = groups.take(limit) unless limit.nil?
+        groups
       end
 
       def groups_of_aggregate_id_prefixes(number_of_groups)
@@ -382,15 +426,8 @@ module Sequent
         end
       end
 
-      def insert_ids
+      def on_progress
         ->(progress, done, ids) do
-          unless ids.empty?
-            exec_sql(
-              "insert into #{ReplayedIds.table_name} (event_id) values #{ids.map do |id|
-                                                                           "(#{id})"
-                                                                         end.join(',')}",
-            )
-          end
           Sequent::Core::EventStore::PRINT_PROGRESS[progress, done, ids] if progress > 0
         end
       end
@@ -413,18 +450,24 @@ module Sequent
         Sequent::Configuration.restore(old_config)
       end
 
-      def event_stream(aggregate_prefixes, event_types, exclude_already_replayed)
+      def event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
         fail ArgumentError, 'aggregate_prefixes is mandatory' unless aggregate_prefixes.present?
 
         event_stream = Sequent.configuration.event_record_class.where(event_type: event_types)
         event_stream = event_stream.where(<<~SQL, aggregate_prefixes)
-          substring(aggregate_id::varchar from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?)
+          substring(aggregate_id::text from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?)
         SQL
-        if exclude_already_replayed
-          event_stream = event_stream
-            .where("NOT EXISTS (SELECT 1 FROM #{ReplayedIds.table_name} WHERE event_id = event_records.id)")
+        if minimum_xact_id_inclusive && maximum_xact_id_exclusive
+          event_stream = event_stream.where(
+            'xact_id >= ? AND xact_id < ?',
+            minimum_xact_id_inclusive,
+            maximum_xact_id_exclusive,
+          )
+        elsif minimum_xact_id_inclusive
+          event_stream = event_stream.where('xact_id >= ?', minimum_xact_id_inclusive)
+        elsif maximum_xact_id_exclusive
+          event_stream = event_stream.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive)
         end
-        event_stream = event_stream.where('event_records.created_at > ?', 1.day.ago) if exclude_already_replayed
         event_stream
           .order('aggregate_id ASC, sequence_number ASC')
           .select('id, event_type, event_json, sequence_number')

data/lib/sequent/rake/migration_tasks.rb

@@ -31,6 +31,12 @@ module Sequent
           EOS
           task init: :set_env_var
 
+          desc 'Creates sequent view schema if not exists and runs internal migrations'
+          task create_and_migrate_sequent_view_schema: ['sequent:init', :init] do
+            ensure_sequent_env_set!
+            Sequent::Migrations::ViewSchema.create_view_schema_if_not_exists(env: @env)
+          end
+
           namespace :db do
             desc 'Creates the database and initializes the event_store schema for the current env'
             task create: ['sequent:init'] do
@@ -47,9 +53,9 @@ module Sequent
               ensure_sequent_env_set!
 
               if @env == 'production' && args[:production] != 'yes_drop_production'
-                fail <<~OES
+                fail <<~EOS
                   Wont drop db in production unless you whitelist the environment as follows: rake sequent:db:drop[yes_drop_production]
-                OES
+                EOS
               end
 
               db_config = Sequent::Support::Database.read_config(@env)
@@ -91,12 +97,70 @@ module Sequent
             task :init
 
             desc 'Prints the current version in the database'
-            task current_version: ['sequent:init', :init] do
-              ensure_sequent_env_set!
+            task current_version: [:create_and_migrate_sequent_view_schema] do
+              puts "Current version in the database is: #{Sequent::Migrations::Versions.current_version}"
+            end
 
-              Sequent::Support::Database.connect!(@env)
+            desc 'Returns whether a migration is currently running'
+            task check_running_migrations: [:create_and_migrate_sequent_view_schema] do
+              if Sequent::Migrations::Versions.running.any?
+                puts <<~EOS
+                  Migration is running, current version: #{Sequent::Migrations::Versions.current_version},
+                  target version #{Sequent::Migrations::Versions.version_currently_migrating}
+                EOS
+              else
+                puts 'No running migrations'
+              end
+            end
 
-              puts "Current version in the database is: #{Sequent::Migrations::ViewSchema::Versions.maximum(:version)}"
+            desc 'Returns whether a migration is pending'
+            task check_pending_migrations: [:create_and_migrate_sequent_view_schema] do
+              if Sequent.new_version != Sequent::Migrations::Versions.current_version
+                puts <<~EOS
+                  Migration is pending, current version: #{Sequent::Migrations::Versions.current_version},
+                  pending version: #{Sequent.new_version}
+                EOS
+              else
+                puts 'No pending migrations'
+              end
+            end
+
+            desc <<-EOS
+              Shows the current status of the migrations
+            EOS
+            task status: ['sequent:init', :init] do
+              ensure_sequent_env_set!
+              db_config = Sequent::Support::Database.read_config(@env)
+              view_schema = Sequent::Migrations::ViewSchema.new(db_config: db_config)
+
+              latest_done_version = Sequent::Migrations::Versions.done.latest
+              latest_version = Sequent::Migrations::Versions.latest
+              pending_version = Sequent.new_version
+              case latest_version.status
+              when Sequent::Migrations::Versions::DONE
+                if pending_version == latest_version.version
+                  puts "Current version #{latest_version.version}, no pending changes"
+                else
+                  puts "Current version #{latest_version.version}, pending version #{pending_version}"
+                end
+              when Sequent::Migrations::Versions::MIGRATE_ONLINE_RUNNING
+                puts "Online migration from #{latest_done_version.version} to #{latest_version.version} is running"
+              when Sequent::Migrations::Versions::MIGRATE_ONLINE_FINISHED
+                projectors = view_schema.plan.projectors
+                event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
+
+                current_snapshot_xmin_xact_id = Sequent::Migrations::Versions.current_snapshot_xmin_xact_id
+                pending_events = Sequent.configuration.event_record_class
+                  .where(event_type: event_types)
+                  .where('xact_id >= ?', current_snapshot_xmin_xact_id)
+                  .count
+                print <<~EOS
+                  Online migration from #{latest_done_version.version} to #{latest_version.version} is finished.
+                  #{current_snapshot_xmin_xact_id - latest_version.xmin_xact_id} transactions behind current state (#{pending_events} pending events).
+                EOS
+              when Sequent::Migrations::Versions::MIGRATE_OFFLINE_RUNNING
+                puts "Offline migration from #{latest_done_version.version} to #{latest_version.version} is running"
+              end
             end
 
             desc <<~EOS
@@ -122,6 +186,25 @@ module Sequent
 
               view_schema.migrate_offline
             end
+
+            desc <<~EOS
+              Runs the projectors in replay mode without making any changes to the database, useful for (performance) testing against real data.
+
+              Pass a regular expression as parameter to select the projectors to run, otherwise all projectors are selected.
+            EOS
+            task :dryrun, %i[regex group_exponent limit offset] => ['sequent:init', :init] do |_task, args|
+              ensure_sequent_env_set!
+
+              db_config = Sequent::Support::Database.read_config(@env)
+              view_schema = Sequent::DryRun::ViewSchema.new(db_config: db_config)
+
+              view_schema.migrate_dryrun(
+                regex: args[:regex],
+                group_exponent: (args[:group_exponent] || 3).to_i,
+                limit: args[:limit]&.to_i,
+                offset: args[:offset]&.to_i,
+              )
+            end
           end
 
           namespace :snapshots do

data/lib/sequent/sequent.rb

@@ -22,21 +22,14 @@ module Sequent
   # A minimal setup could look like this:
   #
   #   Sequent.configure do |config|
-  #     config.event_handlers = [
-  #       MyProjector.new,
-  #       AnotherProjector.new,
-  #       MyWorkflow.new,
-  #     ]
-  #
-  #     config.command_handlers = [
-  #       MyCommandHandler.new,
-  #     ]
-  #
+  #     config.enable_autoregistration = true
   #   end
   #
-  #
+  # Calling configure a second time will create a new configuration
   def self.configure
+    Configuration.reset
     yield Configuration.instance
+    Configuration.instance.autoregister!
   end
 
   def self.configuration

data/lib/sequent/support/database.rb

@@ -53,7 +53,9 @@ module Sequent
           db_config = db_config.deep_merge(
             Sequent.configuration.primary_database_key => db_config_overrides,
           ).stringify_keys
-          ActiveRecord.legacy_connection_handling = false
+          if ActiveRecord::VERSION::MAJOR >= 7 && ActiveRecord::VERSION::MINOR < 1
+            ActiveRecord.legacy_connection_handling = false
+          end
           ActiveRecord::Base.configurations = db_config.stringify_keys
           ActiveRecord::Base.connects_to database: {
             Sequent.configuration.primary_database_role => Sequent.configuration.primary_database_key,
@@ -130,16 +132,6 @@ module Sequent
         self.class.execute_sql(sql)
       end
 
-      def migrate(migrations_path, schema_migration: ActiveRecord::SchemaMigration, verbose: true)
-        ActiveRecord::Migration.verbose = verbose
-        if ActiveRecord::VERSION::MAJOR >= 6
-          ActiveRecord::MigrationContext.new([migrations_path], schema_migration).up
-        elsif ActiveRecord::VERSION::MAJOR >= 5 && ActiveRecord::VERSION::MINOR >= 2
-          ActiveRecord::MigrationContext.new([migrations_path]).up
-        else
-          ActiveRecord::Migrator.migrate(migrations_path)
-        end
-      end
     end
   end
 end

data/lib/sequent/support.rb

@@ -1,5 +1,3 @@
 # frozen_string_literal: true
 
 require_relative 'support/database'
-require_relative 'support/view_projection'
-require_relative 'support/view_schema'

data/lib/sequent/util/util.rb

@@ -4,3 +4,4 @@ require_relative 'skip_if_already_processing'
 require_relative 'timer'
 require_relative 'printer'
 require_relative 'dry_run'
+require_relative 'web/clear_cache'

data/lib/sequent/util/web/clear_cache.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Sequent
+  module Util
+    module Web
+      class ClearCache
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          @app.call(env)
+        ensure
+          Sequent.aggregate_repository.clear!
+        end
+      end
+    end
+  end
+end

data/lib/sequent.rb

@@ -5,5 +5,6 @@ require_relative 'sequent/sequent'
 require_relative 'sequent/core/core'
 require_relative 'sequent/util/util'
 require_relative 'sequent/migrations/migrations'
+require_relative 'sequent/dry_run/dry_run'
 
 require_relative 'notices'

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sequent
-  VERSION = '6.0.1'
+  VERSION = '7.1.0'
 end