sequent 7.0.0 → 7.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../core/persistors/replay_optimized_postgres_persistor'
4
+ module Sequent
5
+ module DryRun
6
+ # Subclass of ReplayOptimizedPostgresPersistor
7
+ # This persistor does not persist anything. Mainly usefull for
8
+ # performance testing migrations.
9
+ class ReadOnlyReplayOptimizedPostgresPersistor < Core::Persistors::ReplayOptimizedPostgresPersistor
10
+ def prepare
11
+ @starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
12
+ end
13
+
14
+ def commit
15
+ # Running in dryrun mode, not committing anything.
16
+ ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
17
+ elapsed = ending - @starting
18
+ count = @record_store.values.map(&:size).sum
19
+ Sequent.logger.info(
20
+ "dryrun: processed #{count} records in #{elapsed.round(2)} s (#{(count / elapsed).round(2)} records/s)",
21
+ )
22
+ clear
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../migrations/view_schema'
4
+ require_relative 'read_only_replay_optimized_postgres_persistor'
5
+
6
+ module Sequent
7
+ module DryRun
8
+ # Subclass of Migrations::ViewSchema to dry run a migration.
9
+ # This migration does not insert anything into the database, mainly usefull
10
+ # for performance testing migrations.
11
+ class ViewSchema < Migrations::ViewSchema
12
+ def migrate_dryrun(regex:, group_exponent: 3, limit: nil, offset: nil)
13
+ persistor = DryRun::ReadOnlyReplayOptimizedPostgresPersistor.new
14
+
15
+ projectors = Sequent::Core::Migratable.all.select { |p| p.replay_persistor.nil? && p.name.match(regex || /.*/) }
16
+ if projectors.present?
17
+ Sequent.logger.info "Dry run using the following projectors: #{projectors.map(&:name).join(', ')}"
18
+
19
+ starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
20
+ groups = groups(group_exponent: group_exponent, limit: limit, offset: offset)
21
+ replay!(persistor, projectors: projectors, groups: groups)
22
+ ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
23
+
24
+ Sequent.logger.info("Done migrate_dryrun for version #{Sequent.new_version} in #{ending - starting} s")
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ # override so no ids are inserted.
31
+ def insert_ids
32
+ ->(progress, done, ids) {}
33
+ end
34
+ end
35
+ end
36
+ end
@@ -8,6 +8,7 @@ ActiveRecord::Schema.define do
8
8
  t.text "event_json", :null => false
9
9
  t.integer "command_record_id", :null => false
10
10
  t.integer "stream_record_id", :null => false
11
+ t.bigint "xact_id"
11
12
  end
12
13
 
13
14
  execute %Q{
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sequent
4
+ module Migrations
5
+ class MigrationError < RuntimeError; end
6
+ class MigrationNotStarted < MigrationError; end
7
+ class MigrationDone < MigrationError; end
8
+ class ConcurrentMigration < MigrationError; end
9
+
10
+ class InvalidMigrationDefinition < MigrationError; end
11
+ end
12
+ end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative 'errors'
4
+
3
5
  module Sequent
4
6
  module Migrations
5
7
  class Planner
@@ -100,7 +102,8 @@ module Sequent
100
102
  def map_to_migrations(migrations)
101
103
  migrations.reduce({}) do |memo, (version, ms)|
102
104
  unless ms.is_a?(Array)
103
- fail "Declared migrations for version #{version} must be an Array. For example: {'3' => [FooProjector]}"
105
+ fail InvalidMigrationDefinition,
106
+ "Declared migrations for version #{version} must be an Array. For example: {'3' => [FooProjector]}"
104
107
  end
105
108
 
106
109
  memo[version] = ms.flat_map do |migration|
@@ -109,14 +112,15 @@ module Sequent
109
112
  #{Sequent.configuration.migration_sql_files_directory}/#{migration.table_name}_#{version}.sql
110
113
  EOS
111
114
  unless File.exist?(alter_table_sql_file_name)
112
- fail "Missing file #{alter_table_sql_file_name} to apply for version #{version}"
115
+ fail InvalidMigrationDefinition,
116
+ "Missing file #{alter_table_sql_file_name} to apply for version #{version}"
113
117
  end
114
118
 
115
119
  migration.copy(version)
116
120
  elsif migration < Sequent::Projector
117
121
  migration.managed_tables.map { |table| ReplayTable.create(table, version) }
118
122
  else
119
- fail "Unknown Migration #{migration}"
123
+ fail InvalidMigrationDefinition, "Unknown Migration #{migration}"
120
124
  end
121
125
  end
122
126
 
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sequent
4
+ module Migrations
5
+ class Versions < Sequent::ApplicationRecord
6
+ MIGRATE_ONLINE_RUNNING = 1
7
+ MIGRATE_ONLINE_FINISHED = 2
8
+ MIGRATE_OFFLINE_RUNNING = 3
9
+ DONE = nil
10
+
11
+ def self.migration_sql
12
+ <<~SQL.chomp
13
+ CREATE TABLE IF NOT EXISTS #{table_name} (version integer NOT NULL, CONSTRAINT version_pk PRIMARY KEY(version));
14
+ ALTER TABLE #{table_name} drop constraint if exists only_one_running;
15
+ ALTER TABLE #{table_name} ADD COLUMN IF NOT EXISTS status INTEGER DEFAULT NULL CONSTRAINT only_one_running CHECK (status in (1,2,3));
16
+ ALTER TABLE #{table_name} ADD COLUMN IF NOT EXISTS xmin_xact_id BIGINT;
17
+ DROP INDEX IF EXISTS single_migration_running;
18
+ CREATE UNIQUE INDEX single_migration_running ON #{table_name} ((status * 0)) where status is not null;
19
+ SQL
20
+ end
21
+
22
+ scope :done, -> { where(status: DONE) }
23
+ scope :running,
24
+ -> {
25
+ where(status: [MIGRATE_ONLINE_RUNNING, MIGRATE_ONLINE_FINISHED, MIGRATE_OFFLINE_RUNNING])
26
+ }
27
+
28
+ def self.current_version
29
+ done.latest_version || 0
30
+ end
31
+
32
+ def self.version_currently_migrating
33
+ running.latest_version
34
+ end
35
+
36
+ def self.latest_version
37
+ latest&.version
38
+ end
39
+
40
+ def self.latest
41
+ order('version desc').limit(1).first
42
+ end
43
+
44
+ def self.start_online!(new_version)
45
+ create!(version: new_version, status: MIGRATE_ONLINE_RUNNING, xmin_xact_id: current_snapshot_xmin_xact_id)
46
+ rescue ActiveRecord::RecordNotUnique
47
+ raise ConcurrentMigration, "Migration for version #{new_version} is already running"
48
+ end
49
+
50
+ def self.end_online!(new_version)
51
+ find_by!(version: new_version, status: MIGRATE_ONLINE_RUNNING).update(status: MIGRATE_ONLINE_FINISHED)
52
+ end
53
+
54
+ def self.rollback!(new_version)
55
+ running.where(version: new_version).delete_all
56
+ end
57
+
58
+ def self.start_offline!(new_version)
59
+ current_migration = find_by(version: new_version)
60
+ fail MigrationNotStarted if current_migration.blank?
61
+
62
+ current_migration.with_lock('FOR UPDATE NOWAIT') do
63
+ current_migration.reload
64
+ fail MigrationDone if current_migration.status.nil?
65
+ fail ConcurrentMigration if current_migration.status != MIGRATE_ONLINE_FINISHED
66
+
67
+ current_migration.update(status: MIGRATE_OFFLINE_RUNNING)
68
+ end
69
+ rescue ActiveRecord::LockWaitTimeout
70
+ raise ConcurrentMigration
71
+ end
72
+
73
+ def self.end_offline!(new_version)
74
+ find_by!(version: new_version, status: MIGRATE_OFFLINE_RUNNING).update(status: DONE)
75
+ end
76
+
77
+ def self.current_snapshot_xmin_xact_id
78
+ connection.execute('SELECT pg_snapshot_xmin(pg_current_snapshot())::text::bigint AS xmin').first['xmin']
79
+ end
80
+ end
81
+ end
82
+ end
@@ -3,6 +3,7 @@
3
3
  require 'parallel'
4
4
  require 'postgresql_cursor'
5
5
 
6
+ require_relative 'errors'
6
7
  require_relative '../support/database'
7
8
  require_relative '../sequent'
8
9
  require_relative '../util/timer'
@@ -11,11 +12,10 @@ require_relative './projectors'
11
12
  require_relative 'planner'
12
13
  require_relative 'executor'
13
14
  require_relative 'sql'
15
+ require_relative 'versions'
14
16
 
15
17
  module Sequent
16
18
  module Migrations
17
- class MigrationError < RuntimeError; end
18
-
19
19
  ##
20
20
  # ViewSchema is used for migration of you view_schema. For instance
21
21
  # when you create new Projectors or change existing Projectors.
@@ -74,9 +74,6 @@ module Sequent
74
74
  include Sequent::Util::Printer
75
75
  include Sql
76
76
 
77
- class Versions < Sequent::ApplicationRecord; end
78
- class ReplayedIds < Sequent::ApplicationRecord; end
79
-
80
77
  attr_reader :view_schema, :db_config, :logger
81
78
 
82
79
  class << self
@@ -111,7 +108,7 @@ module Sequent
111
108
  ##
112
109
  # Returns the current version from the database
113
110
  def current_version
114
- Versions.order('version desc').limit(1).first&.version || 0
111
+ Versions.current_version
115
112
  end
116
113
 
117
114
  ##
@@ -150,7 +147,7 @@ module Sequent
150
147
  replay!(
151
148
  Sequent.configuration.online_replay_persistor_class.new,
152
149
  projectors: Core::Migratable.projectors,
153
- group_exponent: group_exponent,
150
+ groups: groups(group_exponent: group_exponent),
154
151
  )
155
152
  end
156
153
 
@@ -160,14 +157,7 @@ module Sequent
160
157
  # This method is mainly useful during an initial setup of the view schema
161
158
  def create_view_schema_if_not_exists
162
159
  exec_sql(%(CREATE SCHEMA IF NOT EXISTS #{view_schema}))
163
- in_view_schema do
164
- exec_sql(<<~SQL.chomp)
165
- CREATE TABLE IF NOT EXISTS #{Versions.table_name} (version integer NOT NULL, CONSTRAINT version_pk PRIMARY KEY(version))
166
- SQL
167
- exec_sql(<<~SQL.chomp)
168
- CREATE TABLE IF NOT EXISTS #{ReplayedIds.table_name} (event_id bigint NOT NULL, CONSTRAINT event_id_pk PRIMARY KEY(event_id))
169
- SQL
170
- end
160
+ migrate_metadata_tables
171
161
  end
172
162
 
173
163
  def plan
@@ -194,27 +184,45 @@ module Sequent
194
184
  #
195
185
  # If anything fails an exception is raised and everything is rolled back
196
186
  #
187
+ # @raise ConcurrentMigrationError if migration is already running
197
188
  def migrate_online
189
+ ensure_valid_plan!
190
+ migrate_metadata_tables
191
+
198
192
  return if Sequent.new_version == current_version
199
193
 
200
194
  ensure_version_correct!
201
195
 
196
+ Sequent.logger.info("Start migrate_online for version #{Sequent.new_version}")
197
+
202
198
  in_view_schema do
203
- truncate_replay_ids_table!
199
+ Versions.start_online!(Sequent.new_version)
204
200
 
205
201
  drop_old_tables(Sequent.new_version)
206
202
  executor.execute_online(plan)
207
203
  end
208
204
 
209
- replay!(Sequent.configuration.online_replay_persistor_class.new) if plan.projectors.any?
205
+ if plan.projectors.any?
206
+ replay!(
207
+ Sequent.configuration.online_replay_persistor_class.new,
208
+ groups: groups,
209
+ maximum_xact_id_exclusive: Versions.running.first.xmin_xact_id,
210
+ )
211
+ end
210
212
 
211
213
  in_view_schema do
212
214
  executor.create_indexes_after_execute_online(plan)
215
+ executor.reset_table_names(plan)
216
+ Versions.end_online!(Sequent.new_version)
213
217
  end
214
- executor.reset_table_names(plan)
215
- # rubocop:disable Lint/RescueException
216
- rescue Exception => e
217
- # rubocop:enable Lint/RescueException
218
+ Sequent.logger.info("Done migrate_online for version #{Sequent.new_version}")
219
+ rescue ConcurrentMigration
220
+ # Do not rollback the migration when this is a concurrent migration as the other one is running
221
+ raise
222
+ rescue InvalidMigrationDefinition
223
+ # Do not rollback the migration when since there is nothing to rollback
224
+ raise
225
+ rescue Exception => e # rubocop:disable Lint/RescueException
218
226
  rollback_migration
219
227
  raise e
220
228
  end
@@ -232,7 +240,7 @@ module Sequent
232
240
  # 2.1 Rename current tables with the +current version+ as SUFFIX
233
241
  # 2.2 Rename the new tables and remove the +new version+ suffix
234
242
  # 2.3 Add the new version in the +Versions+ table
235
- # 3. Performs cleanup of replayed event ids
243
+ # 3. Update the versions table to complete the migration
236
244
  #
237
245
  # If anything fails an exception is raised and everything is rolled back
238
246
  #
@@ -242,6 +250,8 @@ module Sequent
242
250
  return if Sequent.new_version == current_version
243
251
 
244
252
  ensure_version_correct!
253
+ in_view_schema { Versions.start_offline!(Sequent.new_version) }
254
+ Sequent.logger.info("Start migrate_offline for version #{Sequent.new_version}")
245
255
 
246
256
  executor.set_table_names_to_new_version(plan)
247
257
 
@@ -249,8 +259,8 @@ module Sequent
249
259
  if plan.projectors.any?
250
260
  replay!(
251
261
  Sequent.configuration.offline_replay_persistor_class.new,
252
- exclude_ids: true,
253
- group_exponent: 1,
262
+ groups: groups(group_exponent: 1),
263
+ minimum_xact_id_inclusive: Versions.running.first.xmin_xact_id,
254
264
  )
255
265
  end
256
266
 
@@ -259,22 +269,33 @@ module Sequent
259
269
  # 2.1, 2.2
260
270
  executor.execute_offline(plan, current_version)
261
271
  # 2.3 Create migration record
262
- Versions.create!(version: Sequent.new_version)
272
+ Versions.end_offline!(Sequent.new_version)
263
273
  end
264
-
265
- # 3. Truncate replayed ids
266
- truncate_replay_ids_table!
267
274
  end
268
275
  logger.info "Migrated to version #{Sequent.new_version}"
269
- # rubocop:disable Lint/RescueException
270
- rescue Exception => e
271
- # rubocop:enable Lint/RescueException
276
+ rescue ConcurrentMigration
277
+ raise
278
+ rescue MigrationDone
279
+ # no-op same as Sequent.new_version == current_version
280
+ rescue Exception => e # rubocop:disable Lint/RescueException
272
281
  rollback_migration
273
282
  raise e
274
283
  end
275
284
 
276
285
  private
277
286
 
287
+ def ensure_valid_plan!
288
+ plan
289
+ end
290
+
291
+ def migrate_metadata_tables
292
+ Sequent::ApplicationRecord.transaction do
293
+ in_view_schema do
294
+ exec_sql([Versions.migration_sql].join("\n"))
295
+ end
296
+ end
297
+ end
298
+
278
299
  def ensure_version_correct!
279
300
  create_view_schema_if_not_exists
280
301
  new_version = Sequent.new_version
@@ -285,19 +306,22 @@ module Sequent
285
306
  end
286
307
  end
287
308
 
288
- def replay!(replay_persistor, projectors: plan.projectors, exclude_ids: false, group_exponent: 3)
289
- logger.info "group_exponent: #{group_exponent.inspect}"
309
+ def replay!(
310
+ replay_persistor,
311
+ groups:,
312
+ projectors: plan.projectors,
313
+ minimum_xact_id_inclusive: nil,
314
+ maximum_xact_id_exclusive: nil
315
+ )
316
+ logger.info "groups: #{groups.size}"
290
317
 
291
318
  with_sequent_config(replay_persistor, projectors) do
292
319
  logger.info 'Start replaying events'
293
320
 
294
- time("#{16**group_exponent} groups replayed") do
321
+ time("#{groups.size} groups replayed") do
295
322
  event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
296
323
  disconnect!
297
324
 
298
- number_of_groups = 16**group_exponent
299
- groups = groups_of_aggregate_id_prefixes(number_of_groups)
300
-
301
325
  @connected = false
302
326
  # using `map_with_index` because https://github.com/grosser/parallel/issues/175
303
327
  result = Parallel.map_with_index(
@@ -306,10 +330,17 @@ module Sequent
306
330
  ) do |aggregate_prefixes, index|
307
331
  @connected ||= establish_connection
308
332
  msg = <<~EOS.chomp
309
- Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{number_of_groups} replayed
333
+ Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{groups.size} replayed
310
334
  EOS
311
335
  time(msg) do
312
- replay_events(aggregate_prefixes, event_types, exclude_ids, replay_persistor, &insert_ids)
336
+ replay_events(
337
+ aggregate_prefixes,
338
+ event_types,
339
+ minimum_xact_id_inclusive,
340
+ maximum_xact_id_exclusive,
341
+ replay_persistor,
342
+ &on_progress
343
+ )
313
344
  end
314
345
  nil
315
346
  rescue StandardError => e
@@ -324,10 +355,19 @@ module Sequent
324
355
  end
325
356
  end
326
357
 
327
- def replay_events(aggregate_prefixes, event_types, exclude_already_replayed, replay_persistor, &on_progress)
358
+ def replay_events(
359
+ aggregate_prefixes,
360
+ event_types,
361
+ minimum_xact_id_inclusive,
362
+ maximum_xact_id_exclusive,
363
+ replay_persistor,
364
+ &on_progress
365
+ )
328
366
  Sequent.configuration.event_store.replay_events_from_cursor(
329
367
  block_size: 1000,
330
- get_events: -> { event_stream(aggregate_prefixes, event_types, exclude_already_replayed) },
368
+ get_events: -> {
369
+ event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
370
+ },
331
371
  on_progress: on_progress,
332
372
  )
333
373
 
@@ -342,12 +382,16 @@ module Sequent
342
382
  establish_connection
343
383
  drop_old_tables(Sequent.new_version)
344
384
 
345
- truncate_replay_ids_table!
346
385
  executor.reset_table_names(plan)
386
+ Versions.rollback!(Sequent.new_version)
347
387
  end
348
388
 
349
- def truncate_replay_ids_table!
350
- exec_sql("truncate table #{ReplayedIds.table_name}")
389
+ def groups(group_exponent: 3, limit: nil, offset: nil)
390
+ number_of_groups = 16**group_exponent
391
+ groups = groups_of_aggregate_id_prefixes(number_of_groups)
392
+ groups = groups.drop(offset) unless offset.nil?
393
+ groups = groups.take(limit) unless limit.nil?
394
+ groups
351
395
  end
352
396
 
353
397
  def groups_of_aggregate_id_prefixes(number_of_groups)
@@ -382,15 +426,8 @@ module Sequent
382
426
  end
383
427
  end
384
428
 
385
- def insert_ids
429
+ def on_progress
386
430
  ->(progress, done, ids) do
387
- unless ids.empty?
388
- exec_sql(
389
- "insert into #{ReplayedIds.table_name} (event_id) values #{ids.map do |id|
390
- "(#{id})"
391
- end.join(',')}",
392
- )
393
- end
394
431
  Sequent::Core::EventStore::PRINT_PROGRESS[progress, done, ids] if progress > 0
395
432
  end
396
433
  end
@@ -413,18 +450,24 @@ module Sequent
413
450
  Sequent::Configuration.restore(old_config)
414
451
  end
415
452
 
416
- def event_stream(aggregate_prefixes, event_types, exclude_already_replayed)
453
+ def event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
417
454
  fail ArgumentError, 'aggregate_prefixes is mandatory' unless aggregate_prefixes.present?
418
455
 
419
456
  event_stream = Sequent.configuration.event_record_class.where(event_type: event_types)
420
457
  event_stream = event_stream.where(<<~SQL, aggregate_prefixes)
421
- substring(aggregate_id::varchar from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?)
458
+ substring(aggregate_id::text from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?)
422
459
  SQL
423
- if exclude_already_replayed
424
- event_stream = event_stream
425
- .where("NOT EXISTS (SELECT 1 FROM #{ReplayedIds.table_name} WHERE event_id = event_records.id)")
460
+ if minimum_xact_id_inclusive && maximum_xact_id_exclusive
461
+ event_stream = event_stream.where(
462
+ 'xact_id >= ? AND xact_id < ?',
463
+ minimum_xact_id_inclusive,
464
+ maximum_xact_id_exclusive,
465
+ )
466
+ elsif minimum_xact_id_inclusive
467
+ event_stream = event_stream.where('xact_id >= ?', minimum_xact_id_inclusive)
468
+ elsif maximum_xact_id_exclusive
469
+ event_stream = event_stream.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive)
426
470
  end
427
- event_stream = event_stream.where('event_records.created_at > ?', 1.day.ago) if exclude_already_replayed
428
471
  event_stream
429
472
  .order('aggregate_id ASC, sequence_number ASC')
430
473
  .select('id, event_type, event_json, sequence_number')
@@ -31,6 +31,12 @@ module Sequent
31
31
  EOS
32
32
  task init: :set_env_var
33
33
 
34
+ desc 'Creates sequent view schema if not exists and runs internal migrations'
35
+ task create_and_migrate_sequent_view_schema: ['sequent:init', :init] do
36
+ ensure_sequent_env_set!
37
+ Sequent::Migrations::ViewSchema.create_view_schema_if_not_exists(env: @env)
38
+ end
39
+
34
40
  namespace :db do
35
41
  desc 'Creates the database and initializes the event_store schema for the current env'
36
42
  task create: ['sequent:init'] do
@@ -47,9 +53,9 @@ module Sequent
47
53
  ensure_sequent_env_set!
48
54
 
49
55
  if @env == 'production' && args[:production] != 'yes_drop_production'
50
- fail <<~OES
56
+ fail <<~EOS
51
57
  Wont drop db in production unless you whitelist the environment as follows: rake sequent:db:drop[yes_drop_production]
52
- OES
58
+ EOS
53
59
  end
54
60
 
55
61
  db_config = Sequent::Support::Database.read_config(@env)
@@ -91,12 +97,70 @@ module Sequent
91
97
  task :init
92
98
 
93
99
  desc 'Prints the current version in the database'
94
- task current_version: ['sequent:init', :init] do
95
- ensure_sequent_env_set!
100
+ task current_version: [:create_and_migrate_sequent_view_schema] do
101
+ puts "Current version in the database is: #{Sequent::Migrations::Versions.current_version}"
102
+ end
96
103
 
97
- Sequent::Support::Database.connect!(@env)
104
+ desc 'Returns whether a migration is currently running'
105
+ task check_running_migrations: [:create_and_migrate_sequent_view_schema] do
106
+ if Sequent::Migrations::Versions.running.any?
107
+ puts <<~EOS
108
+ Migration is running, current version: #{Sequent::Migrations::Versions.current_version},
109
+ target version #{Sequent::Migrations::Versions.version_currently_migrating}
110
+ EOS
111
+ else
112
+ puts 'No running migrations'
113
+ end
114
+ end
98
115
 
99
- puts "Current version in the database is: #{Sequent::Migrations::ViewSchema::Versions.maximum(:version)}"
116
+ desc 'Returns whether a migration is pending'
117
+ task check_pending_migrations: [:create_and_migrate_sequent_view_schema] do
118
+ if Sequent.new_version != Sequent::Migrations::Versions.current_version
119
+ puts <<~EOS
120
+ Migration is pending, current version: #{Sequent::Migrations::Versions.current_version},
121
+ pending version: #{Sequent.new_version}
122
+ EOS
123
+ else
124
+ puts 'No pending migrations'
125
+ end
126
+ end
127
+
128
+ desc <<-EOS
129
+ Shows the current status of the migrations
130
+ EOS
131
+ task status: ['sequent:init', :init] do
132
+ ensure_sequent_env_set!
133
+ db_config = Sequent::Support::Database.read_config(@env)
134
+ view_schema = Sequent::Migrations::ViewSchema.new(db_config: db_config)
135
+
136
+ latest_done_version = Sequent::Migrations::Versions.done.latest
137
+ latest_version = Sequent::Migrations::Versions.latest
138
+ pending_version = Sequent.new_version
139
+ case latest_version.status
140
+ when Sequent::Migrations::Versions::DONE
141
+ if pending_version == latest_version.version
142
+ puts "Current version #{latest_version.version}, no pending changes"
143
+ else
144
+ puts "Current version #{latest_version.version}, pending version #{pending_version}"
145
+ end
146
+ when Sequent::Migrations::Versions::MIGRATE_ONLINE_RUNNING
147
+ puts "Online migration from #{latest_done_version.version} to #{latest_version.version} is running"
148
+ when Sequent::Migrations::Versions::MIGRATE_ONLINE_FINISHED
149
+ projectors = view_schema.plan.projectors
150
+ event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
151
+
152
+ current_snapshot_xmin_xact_id = Sequent::Migrations::Versions.current_snapshot_xmin_xact_id
153
+ pending_events = Sequent.configuration.event_record_class
154
+ .where(event_type: event_types)
155
+ .where('xact_id >= ?', current_snapshot_xmin_xact_id)
156
+ .count
157
+ print <<~EOS
158
+ Online migration from #{latest_done_version.version} to #{latest_version.version} is finished.
159
+ #{current_snapshot_xmin_xact_id - latest_version.xmin_xact_id} transactions behind current state (#{pending_events} pending events).
160
+ EOS
161
+ when Sequent::Migrations::Versions::MIGRATE_OFFLINE_RUNNING
162
+ puts "Offline migration from #{latest_done_version.version} to #{latest_version.version} is running"
163
+ end
100
164
  end
101
165
 
102
166
  desc <<~EOS
@@ -122,6 +186,25 @@ module Sequent
122
186
 
123
187
  view_schema.migrate_offline
124
188
  end
189
+
190
+ desc <<~EOS
191
+ Runs the projectors in replay mode without making any changes to the database, useful for (performance) testing against real data.
192
+
193
+ Pass a regular expression as parameter to select the projectors to run, otherwise all projectors are selected.
194
+ EOS
195
+ task :dryrun, %i[regex group_exponent limit offset] => ['sequent:init', :init] do |_task, args|
196
+ ensure_sequent_env_set!
197
+
198
+ db_config = Sequent::Support::Database.read_config(@env)
199
+ view_schema = Sequent::DryRun::ViewSchema.new(db_config: db_config)
200
+
201
+ view_schema.migrate_dryrun(
202
+ regex: args[:regex],
203
+ group_exponent: (args[:group_exponent] || 3).to_i,
204
+ limit: args[:limit]&.to_i,
205
+ offset: args[:offset]&.to_i,
206
+ )
207
+ end
125
208
  end
126
209
 
127
210
  namespace :snapshots do
@@ -53,7 +53,7 @@ module Sequent
53
53
  db_config = db_config.deep_merge(
54
54
  Sequent.configuration.primary_database_key => db_config_overrides,
55
55
  ).stringify_keys
56
- if Gem.loaded_specs['activerecord'].version < Gem::Version.create('7.1.0')
56
+ if ActiveRecord::VERSION::MAJOR >= 7 && ActiveRecord::VERSION::MINOR < 1
57
57
  ActiveRecord.legacy_connection_handling = false
58
58
  end
59
59
  ActiveRecord::Base.configurations = db_config.stringify_keys
@@ -132,16 +132,6 @@ module Sequent
132
132
  self.class.execute_sql(sql)
133
133
  end
134
134
 
135
- def migrate(migrations_path, schema_migration: ActiveRecord::SchemaMigration, verbose: true)
136
- ActiveRecord::Migration.verbose = verbose
137
- if ActiveRecord::VERSION::MAJOR >= 6
138
- ActiveRecord::MigrationContext.new([migrations_path], schema_migration).up
139
- elsif ActiveRecord::VERSION::MAJOR >= 5 && ActiveRecord::VERSION::MINOR >= 2
140
- ActiveRecord::MigrationContext.new([migrations_path]).up
141
- else
142
- ActiveRecord::Migrator.migrate(migrations_path)
143
- end
144
- end
145
135
  end
146
136
  end
147
137
  end