sequent 2.1.0 → 3.0.0

data/lib/sequent/generator/template_project/db/sequent_schema.rb

@@ -0,0 +1,51 @@
+ActiveRecord::Schema.define do
+
+  create_table "event_records", :force => true do |t|
+    t.string "aggregate_id", :null => false
+    t.integer "sequence_number", :null => false
+    t.datetime "created_at", :null => false
+    t.string "event_type", :null => false
+    t.text "event_json", :null => false
+    t.integer "command_record_id", :null => false
+    t.integer "stream_record_id", :null => false
+  end
+
+  execute %Q{
+CREATE UNIQUE INDEX unique_event_per_aggregate ON event_records (
+  aggregate_id,
+  sequence_number,
+  (CASE event_type WHEN 'Sequent::Core::SnapshotEvent' THEN 0 ELSE 1 END)
+)
+}
+  execute %Q{
+CREATE INDEX snapshot_events ON event_records (aggregate_id, sequence_number DESC) WHERE event_type = 'Sequent::Core::SnapshotEvent'
+}
+
+  add_index "event_records", ["command_record_id"], :name => "index_event_records_on_command_record_id"
+  add_index "event_records", ["event_type"], :name => "index_event_records_on_event_type"
+  add_index "event_records", ["created_at"], :name => "index_event_records_on_created_at"
+
+  create_table "command_records", :force => true do |t|
+    t.string "user_id"
+    t.string "aggregate_id"
+    t.string "command_type", :null => false
+    t.text "command_json", :null => false
+    t.datetime "created_at", :null => false
+  end
+
+  create_table "stream_records", :force => true do |t|
+    t.datetime "created_at", :null => false
+    t.string "aggregate_type", :null => false
+    t.string "aggregate_id", :null => false
+    t.integer "snapshot_threshold"
+  end
+
+  add_index "stream_records", ["aggregate_id"], :name => "index_stream_records_on_aggregate_id", :unique => true
+  execute %q{
+ALTER TABLE event_records ADD CONSTRAINT command_fkey FOREIGN KEY (command_record_id) REFERENCES command_records (id)
+}
+  execute %q{
+ALTER TABLE event_records ADD CONSTRAINT stream_fkey FOREIGN KEY (stream_record_id) REFERENCES stream_records (id)
+}
+
+end

data/lib/sequent/generator/template_project/db/tables/post_records.sql

@@ -0,0 +1,10 @@
+CREATE TABLE post_records%SUFFIX% (
+    id serial NOT NULL,
+    aggregate_id uuid NOT NULL,
+    author character varying,
+    title character varying,
+    content character varying,
+    CONSTRAINT post_records_pkey%SUFFIX% PRIMARY KEY (id)
+);
+
+CREATE UNIQUE INDEX post_records_keys%SUFFIX% ON post_records%SUFFIX% USING btree (aggregate_id);

data/lib/sequent/generator/template_project/lib/post.rb

@@ -0,0 +1,4 @@
+require_relative 'post/commands'
+require_relative 'post/events'
+require_relative 'post/post'
+require_relative 'post/post_command_handler'

data/lib/sequent/generator/template_project/lib/post/commands.rb

@@ -0,0 +1,4 @@
+class AddPost < Sequent::Command
+  attrs author: String, title: String, content: String
+  validates_presence_of :author, :title, :content
+end

data/lib/sequent/generator/template_project/lib/post/events.rb

@@ -0,0 +1,14 @@
+class PostAdded < Sequent::Event
+end
+
+class PostAuthorChanged < Sequent::Event
+  attrs author: String
+end
+
+class PostTitleChanged < Sequent::Event
+  attrs title: String
+end
+
+class PostContentChanged < Sequent::Event
+  attrs content: String
+end

data/lib/sequent/generator/template_project/lib/post/post.rb

@@ -0,0 +1,24 @@
+class Post < Sequent::AggregateRoot
+  def initialize(command)
+    super(command.aggregate_id)
+    apply PostAdded
+    apply PostAuthorChanged, author: command.author
+    apply PostTitleChanged, title: command.title
+    apply PostContentChanged, content: command.content
+  end
+
+  on PostAdded do
+  end
+
+  on PostAuthorChanged do |event|
+    @author = event.author
+  end
+
+  on PostTitleChanged do |event|
+    @title = event.title
+  end
+
+  on PostContentChanged do |event|
+    @content = event.content
+  end
+end

data/lib/sequent/generator/template_project/lib/post/post_command_handler.rb

@@ -0,0 +1,5 @@
+class PostCommandHandler < Sequent::CommandHandler
+  on AddPost do |command|
+    repository.add_aggregate Post.new(command)
+  end
+end

data/lib/sequent/generator/template_project/my_app.rb

@@ -0,0 +1,11 @@
+require 'sequent'
+require 'sequent/support'
+require 'erb'
+require_relative 'lib/post'
+require_relative 'app/projectors/post_projector'
+
+require_relative 'config/initializers/sequent'
+
+module MyApp
+
+end

data/lib/sequent/generator/template_project/spec/app/projectors/post_projector_spec.rb

@@ -0,0 +1,32 @@
+require_relative '../../spec_helper'
+require_relative '../../../app/projectors/post_projector'
+
+describe PostProjector do
+  let(:aggregate_id) { Sequent.new_uuid }
+  let(:post_projector) { PostProjector.new }
+  let(:post_added) { PostAdded.new(aggregate_id: aggregate_id, sequence_number: 1) }
+
+  context PostAdded do
+    it 'creates a projection' do
+      post_projector.handle_message(post_added)
+      expect(PostRecord.count).to eq(1)
+      record = PostRecord.first
+      expect(record.aggregate_id).to eq(aggregate_id)
+    end
+  end
+
+  context PostTitleChanged do
+    let(:post_title_changed) do
+      PostTitleChanged.new(aggregate_id: aggregate_id, title: 'ben en kim', sequence_number: 2)
+    end
+
+    before { post_projector.handle_message(post_added) }
+
+    it 'updates a projection' do
+      post_projector.handle_message(post_title_changed)
+      expect(PostRecord.count).to eq(1)
+      record = PostRecord.first
+      expect(record.title).to eq('ben en kim')
+    end
+  end
+end

data/lib/sequent/generator/template_project/spec/lib/post/post_command_handler_spec.rb

@@ -0,0 +1,20 @@
+require_relative '../../spec_helper'
+require_relative '../../../lib/post'
+
+describe PostCommandHandler do
+  let(:aggregate_id) { Sequent.new_uuid }
+
+  before :each do
+    Sequent.configuration.command_handlers = [PostCommandHandler.new]
+  end
+
+  it 'creates a post' do
+    when_command AddPost.new(aggregate_id: aggregate_id, author: 'ben', title: 'My first blogpost', content: 'Hello World!')
+    then_events(
+      PostAdded.new(aggregate_id: aggregate_id, sequence_number: 1),
+      PostAuthorChanged.new(aggregate_id: aggregate_id, sequence_number: 2, author: 'ben'),
+      PostTitleChanged,
+      PostContentChanged
+    )
+  end
+end

data/lib/sequent/generator/template_project/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+require 'bundler/setup'
+Bundler.setup
+
+ENV['RACK_ENV'] ||= 'test'
+
+require 'sequent/test'
+require 'database_cleaner'
+
+require_relative '../my_app'
+
+db_config = Sequent::Support::Database.read_config('test')
+Sequent::Support::Database.establish_connection(db_config)
+
+Sequent::Support::Database.drop_schema!(Sequent.configuration.view_schema_name)
+
+Sequent::Migrations::ViewSchema.new(db_config: db_config).create_view_tables
+Sequent.configuration.event_store = Sequent::Test::CommandHandlerHelpers::FakeEventStore.new
+
+RSpec.configure do |config|
+  config.include Sequent::Test::CommandHandlerHelpers
+
+  config.around do |example|
+    Sequent.configuration.aggregate_repository.clear
+    DatabaseCleaner.strategy = :truncation
+    DatabaseCleaner.cleaning do
+      example.run
+    end
+  end
+end

data/lib/sequent/migrations/migrate_events.rb

@@ -24,6 +24,7 @@ module Sequent
       ##
       # @param env The string representing the current environment. E.g. "development", "production"
       def initialize(env)
+        warn '[DEPRECATED] Use of MigrateEvents is deprecated and will be removed from future version. Please use Sequent::Migrations::ViewSchema instead. See the changelog on how to update.'
         @env = env
       end
 

data/lib/sequent/migrations/migrations.rb

@@ -5,3 +5,5 @@ module Sequent
 end
 
 require_relative 'migrate_events'
+require_relative 'projectors'
+require_relative 'view_schema'

data/lib/sequent/migrations/projectors.rb

@@ -0,0 +1,18 @@
+module Sequent
+  module Migrations
+    class Projectors
+      def self.versions
+        fail "Define your own Sequent::Migrations::Projectors class that extends this class and implements this method"
+      end
+
+      def self.version
+        fail "Define your own Sequent::Migrations::Projectors class that extends this class and implements this method"
+      end
+
+      def self.projectors_between(old, new)
+        versions.values_at(*Range.new(old + 1, new).to_a.map(&:to_s)).compact.flatten.uniq
+      end
+    end
+  end
+end
+

data/lib/sequent/migrations/view_schema.rb

@@ -0,0 +1,364 @@
+require 'parallel'
+require 'postgresql_cursor'
+
+require_relative '../support/database'
+require_relative '../sequent'
+require_relative '../util/timer'
+require_relative '../util/printer'
+require_relative './projectors'
+
+module Sequent
+  module Migrations
+    class MigrationError < RuntimeError; end
+
+
+    ##
+    # Responsible for migration of Projectors between view schema versions.
+    #
+    # A Projector needs migration when for instance:
+    #
+    # - New columns are added
+    # - Structure is changed
+    #
+    # To maintain your migrations you need to:
+    # 1. Create a class that extends `Sequent::Migrations::Projectors` and specify in `Sequent.configuration.migrations_class_name`
+    # 2. Define per version which Projectors you want to migrate
+    #    See the definition of `Sequent::Migrations::Projectors.versions` and `Sequent::Migrations::Projectors.version`
+    # 3. Specify in Sequent where your sql files reside (Sequent.configuration.migration_sql_files_directory)
+    # 4. Ensure that you add %SUFFIX% to each name that needs to be unique in postgres (like TABLE names, INDEX names, PRIMARY KEYS)
+    #    E.g. `create table foo%SUFFIX% (id serial NOT NULL, CONSTRAINT foo_pkey%SUFFIX% PRIMARY KEY (id))`
+    #
+    class ViewSchema
+
+      # Corresponds with the index on aggregate_id column in the event_records table
+      #
+      # Since we replay in batches of the first 3 chars of the uuid we created an index on
+      # these 3 characters. Hence the name ;-)
+      #
+      # This also means that the online replay is divided up into 16**3 groups
+      # This might seem a lot for starting event store, but when you will get more
+      # events, you will see that this is pretty good partitioned.
+      LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE = 3
+
+      include Sequent::Util::Timer
+      include Sequent::Util::Printer
+
+      class Versions < ActiveRecord::Base; end
+      class ReplayedIds < ActiveRecord::Base; end
+
+      attr_reader :view_schema, :db_config, :logger
+
+      def initialize(db_config:)
+        @db_config = db_config
+        @view_schema = Sequent.configuration.view_schema_name
+        @logger = Sequent.logger
+      end
+
+      ##
+      # Returns the current version from the database
+      def current_version
+        Versions.order('version desc').limit(1).first&.version || 0
+      end
+
+      ##
+      # Utility method that creates all tables in the view schema
+      #
+      # This method is mainly useful in test scenario to just create
+      # the entire view schema without replaying the events
+      def create_view_tables
+        create_view_schema_if_not_exists
+        in_view_schema do
+          Sequent::Core::Migratable.all.flat_map(&:managed_tables).each do |table|
+            statements = sql_file_to_statements("#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}.sql") { |raw_sql| raw_sql.remove('%SUFFIX%') }
+            statements.each { |statement| exec_sql(statement) }
+          end
+        end
+      end
+
+      ##
+      # Utility method that replays events for all managed_tables from all Sequent::Core::Projector's
+      #
+      # This method is mainly useful in test scenario's or development tasks
+      def replay_all!
+        replay!(Sequent::Core::Migratable.all, Sequent.configuration.online_replay_persistor_class.new)
+      end
+
+      ##
+      # Utility method that creates the view_schema and the meta data tables
+      #
+      # This method is mainly useful during an initial setup of the view schema
+      def create_view_schema_if_not_exists
+        exec_sql(%Q{CREATE SCHEMA IF NOT EXISTS #{view_schema}})
+        in_view_schema do
+          exec_sql(%Q{CREATE TABLE IF NOT EXISTS #{Versions.table_name} (version integer NOT NULL, CONSTRAINT version_pk PRIMARY KEY(version))})
+          exec_sql(%Q{CREATE TABLE IF NOT EXISTS #{ReplayedIds.table_name} (event_id bigint NOT NULL, CONSTRAINT event_id_pk PRIMARY KEY(event_id))})
+        end
+      end
+
+      ##
+      # First part of a view schema migration
+      #
+      # Call this method while your application is running.
+      # The online part consists of:
+      #
+      # 1. Ensure any previous migrations are cleaned up
+      # 2. Create new tables for the Projectors which need to be migrated to the new version
+      #   These tables will be called `table_name_VERSION`.
+      # 3. Replay all events to populate the tables
+      #   It keeps track of all events that are already replayed.
+      #
+      # If anything fails an exception is raised and everything is rolled back
+      #
+      def migrate_online
+        ensure_version_correct!
+
+        in_view_schema do
+          truncate_replay_ids_table!
+
+          drop_old_tables(Sequent.new_version)
+          for_each_table_to_migrate do |table|
+            statements = sql_file_to_statements("#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}.sql") { |raw_sql| raw_sql.gsub('%SUFFIX%', "_#{Sequent.new_version}") }
+            statements.each { |statement| exec_sql(statement) }
+            table.table_name = "#{table.table_name}_#{Sequent.new_version}"
+            table.reset_column_information
+          end
+        end
+        replay!(projectors_to_migrate, Sequent.configuration.online_replay_persistor_class.new)
+      rescue Exception => e
+        rollback_migration
+        raise e
+      end
+
+      ##
+      # Last part of a view schema migration
+      #
+      # +You have to ensure no events are being added to the event store while this method is running.+
+      # For instance put your application in maintenance mode.
+      #
+      # The offline part consists of:
+      #
+      # 1. Replay all events not yet replayed since #migration_online
+      # 2. Within a single transaction do:
+      # 2.1 Rename current tables with the +current version+ as SUFFIX
+      # 2.2 Rename the new tables and remove the +new version+ suffix
+      # 2.3 Add the new version in the +Versions+ table
+      # 3. Performs cleanup of replayed event ids
+      #
+      # If anything fails an exception is raised and everything is rolled back
+      #
+      # When this method succeeds you can safely start the application from Sequent's point of view.
+      #
+      def migrate_offline
+        return if Sequent.new_version == current_version
+
+        ensure_version_correct!
+
+        set_table_names_to_new_version
+
+        # 1 replay events not yet replayed
+        replay!(projectors_to_migrate, Sequent.configuration.offline_replay_persistor_class.new, exclude_ids: true, group_exponent: 1)
+
+        in_view_schema do
+          ActiveRecord::Base.transaction do
+            for_each_table_to_migrate do |table|
+              current_table_name = table.table_name.gsub("_#{Sequent.new_version}", "")
+              # 2 Rename old table
+              exec_sql("ALTER TABLE IF EXISTS #{current_table_name} RENAME TO #{current_table_name}_#{current_version}")
+              # 3 Rename new table
+              exec_sql("ALTER TABLE #{table.table_name} RENAME TO #{current_table_name}")
+              # Use new table from now on
+              table.table_name = current_table_name
+              table.reset_column_information
+            end
+            # 4. Create migration record
+            Versions.create!(version: Sequent.new_version)
+          end
+
+          # 5. Truncate replayed ids
+          truncate_replay_ids_table!
+        end
+        logger.info "Migrated to version #{Sequent.new_version}"
+      rescue Exception => e
+        rollback_migration
+        raise e
+      end
+
+      private
+      def set_table_names_to_new_version
+        for_each_table_to_migrate do |table|
+          unless table.table_name.end_with?("_#{Sequent.new_version}")
+            table.table_name = "#{table.table_name}_#{Sequent.new_version}"
+            table.reset_column_information
+            fail MigrationError.new("Table #{table.table_name} does not exist. Did you run migrate_online first?") unless table.table_exists?
+          end
+        end
+      end
+
+      def reset_table_names
+        for_each_table_to_migrate do |table|
+          table.table_name = table.table_name.gsub("_#{Sequent.new_version}", "")
+          table.reset_column_information
+        end
+      end
+
+      def ensure_version_correct!
+        create_view_schema_if_not_exists
+        new_version = Sequent.new_version
+
+        fail ArgumentError.new("new_version [#{new_version}] must be greater or equal to current_version [#{current_version}]") if new_version < current_version
+      end
+
+      def replay!(projectors, replay_persistor, exclude_ids: false, group_exponent: 3)
+        logger.info "group_exponent: #{group_exponent.inspect}"
+
+        with_sequent_config(replay_persistor, projectors) do
+          logger.info "Start replaying events"
+
+          time("#{16**group_exponent} groups replayed") do
+            event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
+            disconnect!
+
+            number_of_groups = 16**group_exponent
+            groups = groups_of_aggregate_id_prefixes(number_of_groups)
+
+            @connected = false
+            # using `map_with_index` because https://github.com/grosser/parallel/issues/175
+            result = Parallel.map_with_index(groups, in_processes: Sequent.configuration.number_of_replay_processes) do |aggregate_prefixes, index|
+              begin
+                @connected ||= establish_connection
+                time("Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{number_of_groups} replayed") do
+                  replay_events(aggregate_prefixes, event_types, exclude_ids, replay_persistor, &insert_ids)
+                end
+                nil
+              rescue => e
+                logger.error "Replaying failed for ids: ^#{aggregate_prefixes.first} - #{aggregate_prefixes.last}"
+                logger.error "+++++++++++++++ ERROR +++++++++++++++"
+                recursively_print(e)
+                raise Parallel::Kill # immediately kill all sub-processes
+              end
+            end
+            establish_connection
+            fail if result.nil?
+          end
+        end
+      end
+
+      def replay_events(aggregate_prefixes, event_types, exclude_already_replayed, replay_persistor, &on_progress)
+        Sequent.configuration.event_store.replay_events_from_cursor(
+          block_size: 1000,
+          get_events: -> { event_stream(aggregate_prefixes, event_types, exclude_already_replayed) },
+          on_progress: on_progress
+        )
+
+        replay_persistor.commit
+
+        # Also commit all specific declared replay persistors on projectors.
+        Sequent.configuration.event_handlers.select { |e| e.class.replay_persistor }.each(&:commit)
+      end
+
+      def rollback_migration
+        disconnect!
+        establish_connection
+        drop_old_tables(Sequent.new_version)
+
+        truncate_replay_ids_table!
+        reset_table_names
+      end
+
+      def truncate_replay_ids_table!
+        exec_sql("truncate table #{ReplayedIds.table_name}")
+      end
+
+      def groups_of_aggregate_id_prefixes(number_of_groups)
+        all_prefixes = (0...16**LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE).to_a.map { |i| i.to_s(16) } # first x digits of hex
+        all_prefixes = all_prefixes.map { |s| s.length == 3 ? s : "#{"0" * (3 - s.length)}#{s}" }
+
+        logger.info "Number of groups #{number_of_groups}"
+
+        logger.debug "Prefixes: #{all_prefixes.length}"
+        fail "Can not have more groups #{number_of_groups} than number of prefixes #{all_prefixes.length}" if number_of_groups > all_prefixes.length
+
+        all_prefixes.each_slice(all_prefixes.length/number_of_groups).to_a
+      end
+
+      def for_each_table_to_migrate
+        projectors_to_migrate.flat_map(&:managed_tables).each do |managed_table|
+          yield(managed_table)
+        end
+      end
+
+      def projectors_to_migrate
+        Sequent.migration_class.projectors_between(current_version, Sequent.new_version)
+      end
+
+      def in_view_schema
+        Sequent::Support::Database.with_schema_search_path(view_schema, db_config) do
+          yield
+        end
+      end
+
+      def sql_file_to_statements(file_location)
+        raw_sql_string = File.read(file_location, encoding: 'bom|utf-8')
+        sql_string = yield(raw_sql_string)
+        sql_string.split(/;$/).reject { |statement| statement.remove("\n").blank? }
+      end
+
+      def drop_old_tables(new_version)
+        versions_to_check = (current_version - 10)..new_version
+        old_tables = versions_to_check.flat_map do |old_version|
+          exec_sql(
+            "select table_name from information_schema.tables where table_schema = '#{Sequent.configuration.view_schema_name}' and table_name LIKE '%_#{old_version}'"
+          ).flat_map { |row| row.values }
+        end
+        old_tables.each do |old_table|
+          exec_sql("DROP TABLE #{Sequent.configuration.view_schema_name}.#{old_table} CASCADE")
+        end
+      end
+
+      def insert_ids
+        ->(progress, done, ids) do
+          exec_sql("insert into #{ReplayedIds.table_name} (event_id) values #{ids.map { |id| "(#{id})" }.join(',')}") unless ids.empty?
+          Sequent::Core::EventStore::PRINT_PROGRESS[progress, done, ids] if progress > 0
+        end
+      end
+
+      def with_sequent_config(replay_persistor, projectors, &block)
+        old_config = Sequent.configuration
+
+        config = Sequent.configuration.dup
+
+        replay_projectors = projectors.map { |projector_class| projector_class.new(projector_class.replay_persistor || replay_persistor) }
+        config.transaction_provider = Sequent::Core::Transactions::NoTransactions.new
+        config.event_handlers = replay_projectors
+
+        Sequent::Configuration.restore(config)
+
+        block.call
+      ensure
+        Sequent::Configuration.restore(old_config)
+      end
+
+      def event_stream(aggregate_prefixes, event_types, exclude_already_replayed)
+        fail ArgumentError.new("aggregate_prefixes is mandatory") unless aggregate_prefixes.present?
+
+        event_stream = Sequent.configuration.event_record_class.where(event_type: event_types)
+        event_stream = event_stream.where("substring(aggregate_id::varchar from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?)", aggregate_prefixes)
+        event_stream = event_stream.where("NOT EXISTS (SELECT 1 FROM #{ReplayedIds.table_name} WHERE event_id = event_records.id)") if exclude_already_replayed
+        event_stream.order('sequence_number ASC').select('id, event_type, event_json, sequence_number')
+      end
+
+      ## shortcut methods
+      def disconnect!
+        Sequent::Support::Database.disconnect!
+      end
+
+      def establish_connection
+        Sequent::Support::Database.establish_connection(db_config)
+      end
+
+      def exec_sql(sql)
+        ActiveRecord::Base.connection.execute(sql)
+      end
+    end
+  end
+end