sequent 3.2.2 → 4.0.0

data/lib/sequent/core/stream_record.rb

@@ -13,7 +13,7 @@ module Sequent
       end
     end
 
-    class StreamRecord < ActiveRecord::Base
+    class StreamRecord < Sequent::ApplicationRecord
 
       self.table_name = "stream_records"
 

data/lib/sequent/core/transactions/active_record_transaction_provider.rb

@@ -4,10 +4,12 @@ module Sequent
 
       class ActiveRecordTransactionProvider
         def transactional
-          ActiveRecord::Base.transaction(requires_new: true) do
+          Sequent::ApplicationRecord.transaction(requires_new: true) do
             yield
           end
-          after_commit_queue.each &:call
+          while(!after_commit_queue.empty?) do
+            after_commit_queue.pop.call
+          end
         ensure
           clear_after_commit_queue
         end
@@ -19,11 +21,11 @@ module Sequent
         private
 
         def after_commit_queue
-          Thread.current[:after_commit_queue] ||= []
+          Thread.current[:after_commit_queue] ||= Queue.new
         end
 
         def clear_after_commit_queue
-          Thread.current[:after_commit_queue] = []
+          after_commit_queue.clear
         end
       end
 

data/lib/sequent/generator.rb

@@ -1,4 +1 @@
-require_relative './generator/project'
-require_relative './generator/aggregate'
-require_relative './generator/command'
-require_relative './generator/event'
+require_relative './generator/generator'

data/lib/sequent/generator/generator.rb

@@ -0,0 +1,4 @@
+require_relative 'aggregate'
+require_relative 'command'
+require_relative 'project'
+require_relative 'event'

data/lib/sequent/generator/project.rb

@@ -54,7 +54,7 @@ module Sequent
       end
 
       def name_camelized
-        @name_camelized ||= name.camelize
+        @name_camelized ||= name.gsub(/\W/, '_').squeeze('_').camelize
       end
     end
   end

data/lib/sequent/generator/template_project/Gemfile

@@ -1,5 +1,5 @@
 source "https://rubygems.org"
-ruby "2.5.1"
+ruby "2.6.5"
 
 gem 'rake'
 # let's use the latest and greatest

data/lib/sequent/generator/template_project/app/records/post_record.rb

@@ -1,2 +1,2 @@
-class PostRecord < ActiveRecord::Base
+class PostRecord < Sequent::ApplicationRecord
 end

data/lib/sequent/generator/template_project/spec/app/projectors/post_projector_spec.rb

@@ -1,4 +1,4 @@
-require_relative '../../spec_helper'
+require 'spec_helper'
 require_relative '../../../app/projectors/post_projector'
 
 describe PostProjector do

data/lib/sequent/generator/template_project/spec/lib/post/post_command_handler_spec.rb

@@ -1,4 +1,4 @@
-require_relative '../../spec_helper'
+require 'spec_helper'
 require_relative '../../../lib/post'
 
 describe PostCommandHandler do

data/lib/sequent/migrations/executor.rb

@@ -0,0 +1,78 @@
+require_relative 'sql'
+
+module Sequent
+  module Migrations
+
+    ##
+    # The executor is the implementation of the 3-phase deploy in Sequent.
+    # is responsible for executing the `Planner::Plan`.
+    #
+    class Executor
+      include Sql
+
+      def execute_online(plan)
+        plan.replay_tables.each do |migration|
+          table = migration.record_class
+          sql_file = "#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}.sql"
+          statements = sql_file_to_statements(sql_file) { |raw_sql| raw_sql.gsub('%SUFFIX%', "_#{migration.version}") }
+          statements.each(&method(:exec_sql))
+          table.table_name = "#{table.table_name}_#{migration.version}"
+          table.reset_column_information
+        end
+      end
+
+      def create_indexes_after_execute_online(plan)
+        plan.replay_tables.each do |migration|
+          table = migration.record_class
+          original_table_name = table.table_name.gsub("_#{migration.version}", '')
+          indexes_file_name = "#{Sequent.configuration.migration_sql_files_directory}/#{original_table_name}.indexes.sql"
+          if File.exist?(indexes_file_name)
+            statements = sql_file_to_statements(indexes_file_name) { |raw_sql| raw_sql.gsub('%SUFFIX%', "_#{migration.version}") }
+            statements.each(&method(:exec_sql))
+          end
+        end
+      end
+
+      def execute_offline(plan, current_version)
+        plan.replay_tables.each do |migration|
+          table = migration.record_class
+          current_table_name = table.table_name.gsub("_#{migration.version}", "")
+          # 2 Rename old table
+          exec_sql("ALTER TABLE IF EXISTS #{current_table_name} RENAME TO #{current_table_name}_#{current_version}")
+          # 3 Rename new table
+          exec_sql("ALTER TABLE #{table.table_name} RENAME TO #{current_table_name}")
+          # Use new table from now on
+          table.table_name = current_table_name
+          table.reset_column_information
+        end
+
+        plan.alter_tables.each do |migration|
+          table = migration.record_class
+          sql_file = "#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}_#{migration.version}.sql"
+          statements = sql_file_to_statements(sql_file)
+          statements.each(&method(:exec_sql))
+        end
+      end
+
+      def reset_table_names(plan)
+        plan.replay_tables.each do |migration|
+          table = migration.record_class
+          table.table_name = table.table_name.gsub("_#{migration.version}", "")
+          table.reset_column_information
+        end
+      end
+
+      def set_table_names_to_new_version(plan)
+        plan.replay_tables.each do |migration|
+          table = migration.record_class
+          unless table.table_name.end_with?("_#{migration.version}")
+            table.table_name = "#{table.table_name}_#{migration.version}"
+            table.reset_column_information
+            fail MigrationError.new("Table #{table.table_name} does not exist. Did you run ViewSchema.migrate_online first?") unless table.table_exists?
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/sequent/migrations/functions.rb

@@ -0,0 +1,76 @@
+module Sequent
+  module Migrations
+    class Migration
+
+      module ClassMethods
+        def create(record_class, version)
+          migration = new(record_class)
+          migration.version = version
+          migration
+        end
+      end
+
+      def self.inherited(child_class)
+        class << child_class
+          include ClassMethods
+        end
+      end
+
+      attr_reader :record_class
+      attr_accessor :version
+
+      def initialize(record_class)
+        @record_class = record_class
+        @version = nil
+      end
+
+      def table_name
+        @record_class.table_name
+      end
+
+      def copy(with_version)
+        self.class.create(record_class, with_version)
+      end
+
+      def ==(other)
+        return false unless other.class == self.class
+
+        self.table_name == other.table_name && version == other.version
+      end
+
+      def hash
+        self.table_name.hash + (version&.hash || 0)
+      end
+    end
+
+    class AlterTable < Migration; end
+
+    class ReplayTable < Migration; end
+
+    module Functions
+
+      module ClassMethods
+        def alter_table(name)
+          AlterTable.new(name)
+        end
+
+        def replay_table(name)
+          ReplayTable.new(name)
+        end
+
+        # Short hand for Sequent::Core::Migratable.all
+        def all_projectors
+          Sequent::Core::Migratable.all
+        end
+
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+    end
+
+    include Functions
+  end
+end

data/lib/sequent/migrations/migrations.rb

@@ -7,3 +7,4 @@ end
 require_relative 'migrate_events'
 require_relative 'projectors'
 require_relative 'view_schema'
+require_relative 'functions'

data/lib/sequent/migrations/planner.rb

@@ -0,0 +1,118 @@
+module Sequent
+  module Migrations
+    class Planner
+      Plan = Struct.new(:projectors, :migrations) do
+        def replay_tables
+          migrations.select { |m| m.class == ReplayTable }
+        end
+
+        def alter_tables
+          migrations.select { |m| m.class == AlterTable }
+        end
+
+        def empty?
+          migrations.empty?
+        end
+      end
+
+      attr_reader :versions
+
+      def initialize(versions)
+        @versions = versions
+      end
+
+      def plan(old, new)
+        migrations = versions.slice(*Range.new(old + 1, new).to_a.map(&:to_s))
+
+        Plan.new(
+          migrations.yield_self(&method(:select_projectors)),
+          migrations
+            .yield_self(&method(:create_migrations))
+            .yield_self(&method(:remove_redundant_migrations))
+        )
+      end
+
+      private
+
+      def select_projectors(migrations)
+        migrations
+          .values
+          .flatten
+          .select { |v| v.is_a?(Class) && v < Sequent::Projector }.uniq
+      end
+
+      def remove_redundant_migrations(migrations)
+        redundant_migrations = migrations
+                                 .yield_self(&method(:group_identical_migrations))
+                                 .yield_self(&method(:select_redundant_migrations))
+                                 .yield_self(&method(:remove_redundancy))
+                                 .values
+                                 .flatten
+
+        (migrations - redundant_migrations)
+          .yield_self(&method(:remove_alter_tables_before_replay_table))
+      end
+
+      def group_identical_migrations(migrations)
+        migrations
+          .group_by { |migration| {migration_type: migration.class, record_class: migration.record_class} }
+      end
+
+      def select_redundant_migrations(grouped_migrations)
+        grouped_migrations.select { |type, ms| type[:migration_type] == ReplayTable && ms.length > 1 }
+      end
+
+      def remove_alter_tables_before_replay_table(migrations)
+        migrations - migrations
+                       .each_with_index
+                       .select { |migration, _index| migration.class == AlterTable }
+                       .select { |migration, index| migrations
+                                                      .slice((index + 1)..-1)
+                                                      .find { |m| m.class == ReplayTable && m.record_class == migration.record_class }
+                       }.map(&:first)
+      end
+
+      def remove_redundancy(grouped_migrations)
+        grouped_migrations.reduce({}) { |memo, (key, ms)|
+          memo[key] = ms
+                        .yield_self(&method(:order_by_version_desc))
+                        .slice(1..-1)
+          memo
+        }
+      end
+
+      def order_by_version_desc(migrations)
+        migrations.sort_by { |m| m.version.to_i }
+          .reverse
+      end
+
+      def create_migrations(migrations)
+        migrations
+          .yield_self(&method(:map_to_migrations))
+          .values
+          .compact
+          .flatten
+      end
+
+      def map_to_migrations(migrations)
+        migrations.reduce({}) do |memo, (version, _migrations)|
+          fail "Declared migrations for version #{version} must be an Array. For example: {'3' => [FooProjector]}" unless _migrations.is_a?(Array)
+
+          memo[version] = _migrations.flat_map do |migration|
+            if migration.is_a?(AlterTable)
+              alter_table_sql_file_name = "#{Sequent.configuration.migration_sql_files_directory}/#{migration.table_name}_#{version}.sql"
+              fail "Missing file #{alter_table_sql_file_name} to apply for version #{version}" unless File.exist?(alter_table_sql_file_name)
+              migration.copy(version)
+            elsif migration < Sequent::Projector
+              migration.managed_tables.map { |table| ReplayTable.create(table, version) }
+            else
+              fail "Unknown Migration #{migration}"
+            end
+          end
+
+          memo
+        end
+      end
+    end
+  end
+end

data/lib/sequent/migrations/projectors.rb

@@ -1,18 +1,19 @@
+require_relative 'planner'
 module Sequent
   module Migrations
     class Projectors
       def self.versions
-        fail "Define your own Sequent::Migrations::Projectors class that extends this class and implements this method"
+        fail "Define your own Sequent::Migrations::List class that extends this class and implements this method"
       end
 
       def self.version
-        fail "Define your own Sequent::Migrations::Projectors class that extends this class and implements this method"
+        fail "Define your own Sequent::Migrations::List class that extends this class and implements this method"
       end
 
-      def self.projectors_between(old, new)
-        versions.values_at(*Range.new(old + 1, new).to_a.map(&:to_s)).compact.flatten.uniq
+      def self.migrations_between(old, new)
+        Planner.new(versions).plan(old, new)
       end
     end
+
   end
 end
-

data/lib/sequent/migrations/sql.rb

@@ -0,0 +1,17 @@
+require_relative '../application_record'
+
+module Sequent
+  module Migrations
+    module Sql
+      def sql_file_to_statements(file_location)
+        sql_string = File.read(file_location, encoding: 'bom|utf-8')
+        sql_string = yield(sql_string) if block_given?
+        sql_string.split(/;$/).reject { |statement| statement.remove("\n").blank? }
+      end
+
+      def exec_sql(sql)
+        Sequent::ApplicationRecord.connection.execute(sql)
+      end
+    end
+  end
+end

data/lib/sequent/migrations/view_schema.rb

@@ -6,30 +6,56 @@ require_relative '../sequent'
 require_relative '../util/timer'
 require_relative '../util/printer'
 require_relative './projectors'
+require_relative 'planner'
+require_relative 'executor'
+require_relative 'sql'
 
 module Sequent
   module Migrations
     class MigrationError < RuntimeError; end
 
-
     ##
-    # Responsible for migration of Projectors between view schema versions.
+    # ViewSchema is used for migration of you view_schema. For instance
+    # when you create new Projectors or change existing Projectors.
     #
-    # A Projector needs migration when for instance:
+    # The following migrations are supported:
     #
-    # - New columns are added
-    # - Structure is changed
+    # - ReplayTable (Projector migrations)
+    # - AlterTable (For instance if you introduce a new column)
     #
     # To maintain your migrations you need to:
     # 1. Create a class that extends `Sequent::Migrations::Projectors` and specify in `Sequent.configuration.migrations_class_name`
-    # 2. Define per version which Projectors you want to migrate
+    # 2. Define per version which migrations you want to execute
     #    See the definition of `Sequent::Migrations::Projectors.versions` and `Sequent::Migrations::Projectors.version`
     # 3. Specify in Sequent where your sql files reside (Sequent.configuration.migration_sql_files_directory)
     # 4. Ensure that you add %SUFFIX% to each name that needs to be unique in postgres (like TABLE names, INDEX names, PRIMARY KEYS)
     #    E.g. `create table foo%SUFFIX% (id serial NOT NULL, CONSTRAINT foo_pkey%SUFFIX% PRIMARY KEY (id))`
+    # 5. If you want to run an `alter_table` migration ensure that
+    #   a sql file named `table_name_VERSION.sql` exists.
+    #
+    # Example:
+    #
+    # class AppMigrations < Sequent::Migrations::Projectors
+    #   def self.version
+    #     '3'
+    #   end
+    #
+    #   def self.versions
+    #     {
+    #       '1' => [Sequent.all_projectors],
+    #       '2' => [
+    #         UserProjector,
+    #         InvoiceProjector,
+    #       ],
+    #       '3' => [
+    #         Sequent::Migrations.alter_table(UserRecord)
+    #       ]
+    #
+    #     }
+    #   end
     #
+    # end
     class ViewSchema
-
       # Corresponds with the index on aggregate_id column in the event_records table
       #
       # Since we replay in batches of the first 3 chars of the uuid we created an index on
@@ -42,9 +68,10 @@ module Sequent
 
       include Sequent::Util::Timer
       include Sequent::Util::Printer
+      include Sql
 
-      class Versions < ActiveRecord::Base; end
-      class ReplayedIds < ActiveRecord::Base; end
+      class Versions < Sequent::ApplicationRecord; end
+      class ReplayedIds < Sequent::ApplicationRecord; end
 
       attr_reader :view_schema, :db_config, :logger
 
@@ -71,6 +98,12 @@ module Sequent
           Sequent::Core::Migratable.all.flat_map(&:managed_tables).each do |table|
             statements = sql_file_to_statements("#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}.sql") { |raw_sql| raw_sql.remove('%SUFFIX%') }
             statements.each { |statement| exec_sql(statement) }
+
+            indexes_file_name = "#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}.indexes.sql"
+            if File.exist?(indexes_file_name)
+              statements = sql_file_to_statements(indexes_file_name) { |raw_sql| raw_sql.remove('%SUFFIX%') }
+              statements.each(&method(:exec_sql))
+            end
           end
         end
       end
@@ -80,7 +113,7 @@ module Sequent
       #
       # This method is mainly useful in test scenario's or development tasks
       def replay_all!
-        replay!(Sequent::Core::Migratable.all, Sequent.configuration.online_replay_persistor_class.new)
+        replay!(Sequent.configuration.online_replay_persistor_class.new)
       end
 
       ##
@@ -95,6 +128,14 @@ module Sequent
         end
       end
 
+      def plan
+        @plan ||= Planner.new(Sequent.migration_class.versions).plan(current_version, Sequent.new_version)
+      end
+
+      def executor
+        @executor ||= Executor.new
+      end
+
       ##
       # First part of a view schema migration
       #
@@ -118,14 +159,16 @@ module Sequent
           truncate_replay_ids_table!
 
           drop_old_tables(Sequent.new_version)
-          for_each_table_to_migrate do |table|
-            statements = sql_file_to_statements("#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}.sql") { |raw_sql| raw_sql.gsub('%SUFFIX%', "_#{Sequent.new_version}") }
-            statements.each { |statement| exec_sql(statement) }
-            table.table_name = "#{table.table_name}_#{Sequent.new_version}"
-            table.reset_column_information
-          end
+          executor.execute_online(plan)
+        end
+
+        if plan.projectors.any?
+          replay!(Sequent.configuration.online_replay_persistor_class.new)
+        end
+
+        in_view_schema do
+          executor.create_indexes_after_execute_online(plan)
         end
-        replay!(projectors_to_migrate, Sequent.configuration.online_replay_persistor_class.new)
       rescue Exception => e
         rollback_migration
         raise e
@@ -155,28 +198,20 @@ module Sequent
 
         ensure_version_correct!
 
-        set_table_names_to_new_version
+        executor.set_table_names_to_new_version(plan)
 
         # 1 replay events not yet replayed
-        replay!(projectors_to_migrate, Sequent.configuration.offline_replay_persistor_class.new, exclude_ids: true, group_exponent: 1)
+        replay!(Sequent.configuration.offline_replay_persistor_class.new, exclude_ids: true, group_exponent: 1) if plan.projectors.any?
 
         in_view_schema do
-          ActiveRecord::Base.transaction do
-            for_each_table_to_migrate do |table|
-              current_table_name = table.table_name.gsub("_#{Sequent.new_version}", "")
-              # 2 Rename old table
-              exec_sql("ALTER TABLE IF EXISTS #{current_table_name} RENAME TO #{current_table_name}_#{current_version}")
-              # 3 Rename new table
-              exec_sql("ALTER TABLE #{table.table_name} RENAME TO #{current_table_name}")
-              # Use new table from now on
-              table.table_name = current_table_name
-              table.reset_column_information
-            end
-            # 4. Create migration record
+          Sequent::ApplicationRecord.transaction do
+            # 2.1, 2.2
+            executor.execute_offline(plan, current_version)
+            # 2.3 Create migration record
             Versions.create!(version: Sequent.new_version)
           end
 
-          # 5. Truncate replayed ids
+          # 3. Truncate replayed ids
           truncate_replay_ids_table!
         end
         logger.info "Migrated to version #{Sequent.new_version}"
@@ -186,41 +221,27 @@ module Sequent
       end
 
       private
-      def set_table_names_to_new_version
-        for_each_table_to_migrate do |table|
-          unless table.table_name.end_with?("_#{Sequent.new_version}")
-            table.table_name = "#{table.table_name}_#{Sequent.new_version}"
-            table.reset_column_information
-            fail MigrationError.new("Table #{table.table_name} does not exist. Did you run migrate_online first?") unless table.table_exists?
-          end
-        end
-      end
 
-      def reset_table_names
-        for_each_table_to_migrate do |table|
-          table.table_name = table.table_name.gsub("_#{Sequent.new_version}", "")
-          table.reset_column_information
-        end
-      end
 
       def ensure_version_correct!
         create_view_schema_if_not_exists
         new_version = Sequent.new_version
 
         fail ArgumentError.new("new_version [#{new_version}] must be greater or equal to current_version [#{current_version}]") if new_version < current_version
+
       end
 
-      def replay!(projectors, replay_persistor, exclude_ids: false, group_exponent: 3)
+      def replay!(replay_persistor, projectors: plan.projectors, exclude_ids: false, group_exponent: 3)
         logger.info "group_exponent: #{group_exponent.inspect}"
 
         with_sequent_config(replay_persistor, projectors) do
           logger.info "Start replaying events"
 
-          time("#{16**group_exponent} groups replayed") do
+          time("#{16 ** group_exponent} groups replayed") do
             event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
             disconnect!
 
-            number_of_groups = 16**group_exponent
+            number_of_groups = 16 ** group_exponent
             groups = groups_of_aggregate_id_prefixes(number_of_groups)
 
             @connected = false
@@ -264,7 +285,7 @@ module Sequent
         drop_old_tables(Sequent.new_version)
 
         truncate_replay_ids_table!
-        reset_table_names
+        executor.reset_table_names(plan)
       end
 
       def truncate_replay_ids_table!
@@ -272,7 +293,7 @@ module Sequent
       end
 
       def groups_of_aggregate_id_prefixes(number_of_groups)
-        all_prefixes = (0...16**LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE).to_a.map { |i| i.to_s(16) } # first x digits of hex
+        all_prefixes = (0...16 ** LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE).to_a.map { |i| i.to_s(16) } # first x digits of hex
         all_prefixes = all_prefixes.map { |s| s.length == 3 ? s : "#{"0" * (3 - s.length)}#{s}" }
 
         logger.info "Number of groups #{number_of_groups}"
@@ -280,17 +301,7 @@ module Sequent
         logger.debug "Prefixes: #{all_prefixes.length}"
         fail "Can not have more groups #{number_of_groups} than number of prefixes #{all_prefixes.length}" if number_of_groups > all_prefixes.length
 
-        all_prefixes.each_slice(all_prefixes.length/number_of_groups).to_a
-      end
-
-      def for_each_table_to_migrate
-        projectors_to_migrate.flat_map(&:managed_tables).each do |managed_table|
-          yield(managed_table)
-        end
-      end
-
-      def projectors_to_migrate
-        Sequent.migration_class.projectors_between(current_version, Sequent.new_version)
+        all_prefixes.each_slice(all_prefixes.length / number_of_groups).to_a
       end
 
       def in_view_schema
@@ -299,12 +310,6 @@ module Sequent
         end
       end
 
-      def sql_file_to_statements(file_location)
-        raw_sql_string = File.read(file_location, encoding: 'bom|utf-8')
-        sql_string = yield(raw_sql_string)
-        sql_string.split(/;$/).reject { |statement| statement.remove("\n").blank? }
-      end
-
       def drop_old_tables(new_version)
         versions_to_check = (current_version - 10)..new_version
         old_tables = versions_to_check.flat_map do |old_version|
@@ -358,10 +363,6 @@ module Sequent
       def establish_connection
         Sequent::Support::Database.establish_connection(db_config)
       end
-
-      def exec_sql(sql)
-        ActiveRecord::Base.connection.execute(sql)
-      end
     end
   end
 end