sequent 7.0.0 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/db/sequent_schema.rb +5 -0
- data/lib/sequent/configuration.rb +8 -13
- data/lib/sequent/core/event_record.rb +1 -0
- data/lib/sequent/core/event_store.rb +4 -0
- data/lib/sequent/core/helpers/message_handler.rb +10 -11
- data/lib/sequent/core/helpers/message_router.rb +13 -7
- data/lib/sequent/core/persistors/active_record_persistor.rb +4 -0
- data/lib/sequent/core/persistors/persistor.rb +5 -0
- data/lib/sequent/core/persistors/replay_optimized_postgres_persistor.rb +140 -133
- data/lib/sequent/dry_run/dry_run.rb +4 -0
- data/lib/sequent/dry_run/read_only_replay_optimized_postgres_persistor.rb +26 -0
- data/lib/sequent/dry_run/view_schema.rb +36 -0
- data/lib/sequent/generator/template_project/db/sequent_schema.rb +1 -0
- data/lib/sequent/migrations/errors.rb +12 -0
- data/lib/sequent/migrations/planner.rb +7 -3
- data/lib/sequent/migrations/versions.rb +82 -0
- data/lib/sequent/migrations/view_schema.rb +101 -58
- data/lib/sequent/rake/migration_tasks.rb +89 -6
- data/lib/sequent/support/database.rb +1 -11
- data/lib/sequent/support.rb +0 -2
- data/lib/sequent/util/util.rb +1 -0
- data/lib/sequent/util/web/clear_cache.rb +19 -0
- data/lib/sequent.rb +1 -0
- data/lib/version.rb +1 -1
- metadata +20 -29
- data/lib/sequent/core/helpers/message_dispatcher.rb +0 -23
- data/lib/sequent/support/view_projection.rb +0 -61
- data/lib/sequent/support/view_schema.rb +0 -24
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../core/persistors/replay_optimized_postgres_persistor'
|
4
|
+
module Sequent
|
5
|
+
module DryRun
|
6
|
+
# Subclass of ReplayOptimizedPostgresPersistor
|
7
|
+
# This persistor does not persist anything. Mainly usefull for
|
8
|
+
# performance testing migrations.
|
9
|
+
class ReadOnlyReplayOptimizedPostgresPersistor < Core::Persistors::ReplayOptimizedPostgresPersistor
|
10
|
+
def prepare
|
11
|
+
@starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
12
|
+
end
|
13
|
+
|
14
|
+
def commit
|
15
|
+
# Running in dryrun mode, not committing anything.
|
16
|
+
ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
17
|
+
elapsed = ending - @starting
|
18
|
+
count = @record_store.values.map(&:size).sum
|
19
|
+
Sequent.logger.info(
|
20
|
+
"dryrun: processed #{count} records in #{elapsed.round(2)} s (#{(count / elapsed).round(2)} records/s)",
|
21
|
+
)
|
22
|
+
clear
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../migrations/view_schema'
|
4
|
+
require_relative 'read_only_replay_optimized_postgres_persistor'
|
5
|
+
|
6
|
+
module Sequent
|
7
|
+
module DryRun
|
8
|
+
# Subclass of Migrations::ViewSchema to dry run a migration.
|
9
|
+
# This migration does not insert anything into the database, mainly usefull
|
10
|
+
# for performance testing migrations.
|
11
|
+
class ViewSchema < Migrations::ViewSchema
|
12
|
+
def migrate_dryrun(regex:, group_exponent: 3, limit: nil, offset: nil)
|
13
|
+
persistor = DryRun::ReadOnlyReplayOptimizedPostgresPersistor.new
|
14
|
+
|
15
|
+
projectors = Sequent::Core::Migratable.all.select { |p| p.replay_persistor.nil? && p.name.match(regex || /.*/) }
|
16
|
+
if projectors.present?
|
17
|
+
Sequent.logger.info "Dry run using the following projectors: #{projectors.map(&:name).join(', ')}"
|
18
|
+
|
19
|
+
starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
20
|
+
groups = groups(group_exponent: group_exponent, limit: limit, offset: offset)
|
21
|
+
replay!(persistor, projectors: projectors, groups: groups)
|
22
|
+
ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
23
|
+
|
24
|
+
Sequent.logger.info("Done migrate_dryrun for version #{Sequent.new_version} in #{ending - starting} s")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
# override so no ids are inserted.
|
31
|
+
def insert_ids
|
32
|
+
->(progress, done, ids) {}
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sequent
|
4
|
+
module Migrations
|
5
|
+
class MigrationError < RuntimeError; end
|
6
|
+
class MigrationNotStarted < MigrationError; end
|
7
|
+
class MigrationDone < MigrationError; end
|
8
|
+
class ConcurrentMigration < MigrationError; end
|
9
|
+
|
10
|
+
class InvalidMigrationDefinition < MigrationError; end
|
11
|
+
end
|
12
|
+
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative 'errors'
|
4
|
+
|
3
5
|
module Sequent
|
4
6
|
module Migrations
|
5
7
|
class Planner
|
@@ -100,7 +102,8 @@ module Sequent
|
|
100
102
|
def map_to_migrations(migrations)
|
101
103
|
migrations.reduce({}) do |memo, (version, ms)|
|
102
104
|
unless ms.is_a?(Array)
|
103
|
-
fail
|
105
|
+
fail InvalidMigrationDefinition,
|
106
|
+
"Declared migrations for version #{version} must be an Array. For example: {'3' => [FooProjector]}"
|
104
107
|
end
|
105
108
|
|
106
109
|
memo[version] = ms.flat_map do |migration|
|
@@ -109,14 +112,15 @@ module Sequent
|
|
109
112
|
#{Sequent.configuration.migration_sql_files_directory}/#{migration.table_name}_#{version}.sql
|
110
113
|
EOS
|
111
114
|
unless File.exist?(alter_table_sql_file_name)
|
112
|
-
fail
|
115
|
+
fail InvalidMigrationDefinition,
|
116
|
+
"Missing file #{alter_table_sql_file_name} to apply for version #{version}"
|
113
117
|
end
|
114
118
|
|
115
119
|
migration.copy(version)
|
116
120
|
elsif migration < Sequent::Projector
|
117
121
|
migration.managed_tables.map { |table| ReplayTable.create(table, version) }
|
118
122
|
else
|
119
|
-
fail "Unknown Migration #{migration}"
|
123
|
+
fail InvalidMigrationDefinition, "Unknown Migration #{migration}"
|
120
124
|
end
|
121
125
|
end
|
122
126
|
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sequent
|
4
|
+
module Migrations
|
5
|
+
class Versions < Sequent::ApplicationRecord
|
6
|
+
MIGRATE_ONLINE_RUNNING = 1
|
7
|
+
MIGRATE_ONLINE_FINISHED = 2
|
8
|
+
MIGRATE_OFFLINE_RUNNING = 3
|
9
|
+
DONE = nil
|
10
|
+
|
11
|
+
def self.migration_sql
|
12
|
+
<<~SQL.chomp
|
13
|
+
CREATE TABLE IF NOT EXISTS #{table_name} (version integer NOT NULL, CONSTRAINT version_pk PRIMARY KEY(version));
|
14
|
+
ALTER TABLE #{table_name} drop constraint if exists only_one_running;
|
15
|
+
ALTER TABLE #{table_name} ADD COLUMN IF NOT EXISTS status INTEGER DEFAULT NULL CONSTRAINT only_one_running CHECK (status in (1,2,3));
|
16
|
+
ALTER TABLE #{table_name} ADD COLUMN IF NOT EXISTS xmin_xact_id BIGINT;
|
17
|
+
DROP INDEX IF EXISTS single_migration_running;
|
18
|
+
CREATE UNIQUE INDEX single_migration_running ON #{table_name} ((status * 0)) where status is not null;
|
19
|
+
SQL
|
20
|
+
end
|
21
|
+
|
22
|
+
scope :done, -> { where(status: DONE) }
|
23
|
+
scope :running,
|
24
|
+
-> {
|
25
|
+
where(status: [MIGRATE_ONLINE_RUNNING, MIGRATE_ONLINE_FINISHED, MIGRATE_OFFLINE_RUNNING])
|
26
|
+
}
|
27
|
+
|
28
|
+
def self.current_version
|
29
|
+
done.latest_version || 0
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.version_currently_migrating
|
33
|
+
running.latest_version
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.latest_version
|
37
|
+
latest&.version
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.latest
|
41
|
+
order('version desc').limit(1).first
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.start_online!(new_version)
|
45
|
+
create!(version: new_version, status: MIGRATE_ONLINE_RUNNING, xmin_xact_id: current_snapshot_xmin_xact_id)
|
46
|
+
rescue ActiveRecord::RecordNotUnique
|
47
|
+
raise ConcurrentMigration, "Migration for version #{new_version} is already running"
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.end_online!(new_version)
|
51
|
+
find_by!(version: new_version, status: MIGRATE_ONLINE_RUNNING).update(status: MIGRATE_ONLINE_FINISHED)
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.rollback!(new_version)
|
55
|
+
running.where(version: new_version).delete_all
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.start_offline!(new_version)
|
59
|
+
current_migration = find_by(version: new_version)
|
60
|
+
fail MigrationNotStarted if current_migration.blank?
|
61
|
+
|
62
|
+
current_migration.with_lock('FOR UPDATE NOWAIT') do
|
63
|
+
current_migration.reload
|
64
|
+
fail MigrationDone if current_migration.status.nil?
|
65
|
+
fail ConcurrentMigration if current_migration.status != MIGRATE_ONLINE_FINISHED
|
66
|
+
|
67
|
+
current_migration.update(status: MIGRATE_OFFLINE_RUNNING)
|
68
|
+
end
|
69
|
+
rescue ActiveRecord::LockWaitTimeout
|
70
|
+
raise ConcurrentMigration
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.end_offline!(new_version)
|
74
|
+
find_by!(version: new_version, status: MIGRATE_OFFLINE_RUNNING).update(status: DONE)
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.current_snapshot_xmin_xact_id
|
78
|
+
connection.execute('SELECT pg_snapshot_xmin(pg_current_snapshot())::text::bigint AS xmin').first['xmin']
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'parallel'
|
4
4
|
require 'postgresql_cursor'
|
5
5
|
|
6
|
+
require_relative 'errors'
|
6
7
|
require_relative '../support/database'
|
7
8
|
require_relative '../sequent'
|
8
9
|
require_relative '../util/timer'
|
@@ -11,11 +12,10 @@ require_relative './projectors'
|
|
11
12
|
require_relative 'planner'
|
12
13
|
require_relative 'executor'
|
13
14
|
require_relative 'sql'
|
15
|
+
require_relative 'versions'
|
14
16
|
|
15
17
|
module Sequent
|
16
18
|
module Migrations
|
17
|
-
class MigrationError < RuntimeError; end
|
18
|
-
|
19
19
|
##
|
20
20
|
# ViewSchema is used for migration of you view_schema. For instance
|
21
21
|
# when you create new Projectors or change existing Projectors.
|
@@ -74,9 +74,6 @@ module Sequent
|
|
74
74
|
include Sequent::Util::Printer
|
75
75
|
include Sql
|
76
76
|
|
77
|
-
class Versions < Sequent::ApplicationRecord; end
|
78
|
-
class ReplayedIds < Sequent::ApplicationRecord; end
|
79
|
-
|
80
77
|
attr_reader :view_schema, :db_config, :logger
|
81
78
|
|
82
79
|
class << self
|
@@ -111,7 +108,7 @@ module Sequent
|
|
111
108
|
##
|
112
109
|
# Returns the current version from the database
|
113
110
|
def current_version
|
114
|
-
Versions.
|
111
|
+
Versions.current_version
|
115
112
|
end
|
116
113
|
|
117
114
|
##
|
@@ -150,7 +147,7 @@ module Sequent
|
|
150
147
|
replay!(
|
151
148
|
Sequent.configuration.online_replay_persistor_class.new,
|
152
149
|
projectors: Core::Migratable.projectors,
|
153
|
-
group_exponent: group_exponent,
|
150
|
+
groups: groups(group_exponent: group_exponent),
|
154
151
|
)
|
155
152
|
end
|
156
153
|
|
@@ -160,14 +157,7 @@ module Sequent
|
|
160
157
|
# This method is mainly useful during an initial setup of the view schema
|
161
158
|
def create_view_schema_if_not_exists
|
162
159
|
exec_sql(%(CREATE SCHEMA IF NOT EXISTS #{view_schema}))
|
163
|
-
|
164
|
-
exec_sql(<<~SQL.chomp)
|
165
|
-
CREATE TABLE IF NOT EXISTS #{Versions.table_name} (version integer NOT NULL, CONSTRAINT version_pk PRIMARY KEY(version))
|
166
|
-
SQL
|
167
|
-
exec_sql(<<~SQL.chomp)
|
168
|
-
CREATE TABLE IF NOT EXISTS #{ReplayedIds.table_name} (event_id bigint NOT NULL, CONSTRAINT event_id_pk PRIMARY KEY(event_id))
|
169
|
-
SQL
|
170
|
-
end
|
160
|
+
migrate_metadata_tables
|
171
161
|
end
|
172
162
|
|
173
163
|
def plan
|
@@ -194,27 +184,45 @@ module Sequent
|
|
194
184
|
#
|
195
185
|
# If anything fails an exception is raised and everything is rolled back
|
196
186
|
#
|
187
|
+
# @raise ConcurrentMigrationError if migration is already running
|
197
188
|
def migrate_online
|
189
|
+
ensure_valid_plan!
|
190
|
+
migrate_metadata_tables
|
191
|
+
|
198
192
|
return if Sequent.new_version == current_version
|
199
193
|
|
200
194
|
ensure_version_correct!
|
201
195
|
|
196
|
+
Sequent.logger.info("Start migrate_online for version #{Sequent.new_version}")
|
197
|
+
|
202
198
|
in_view_schema do
|
203
|
-
|
199
|
+
Versions.start_online!(Sequent.new_version)
|
204
200
|
|
205
201
|
drop_old_tables(Sequent.new_version)
|
206
202
|
executor.execute_online(plan)
|
207
203
|
end
|
208
204
|
|
209
|
-
|
205
|
+
if plan.projectors.any?
|
206
|
+
replay!(
|
207
|
+
Sequent.configuration.online_replay_persistor_class.new,
|
208
|
+
groups: groups,
|
209
|
+
maximum_xact_id_exclusive: Versions.running.first.xmin_xact_id,
|
210
|
+
)
|
211
|
+
end
|
210
212
|
|
211
213
|
in_view_schema do
|
212
214
|
executor.create_indexes_after_execute_online(plan)
|
215
|
+
executor.reset_table_names(plan)
|
216
|
+
Versions.end_online!(Sequent.new_version)
|
213
217
|
end
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
+
Sequent.logger.info("Done migrate_online for version #{Sequent.new_version}")
|
219
|
+
rescue ConcurrentMigration
|
220
|
+
# Do not rollback the migration when this is a concurrent migration as the other one is running
|
221
|
+
raise
|
222
|
+
rescue InvalidMigrationDefinition
|
223
|
+
# Do not rollback the migration when since there is nothing to rollback
|
224
|
+
raise
|
225
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
218
226
|
rollback_migration
|
219
227
|
raise e
|
220
228
|
end
|
@@ -232,7 +240,7 @@ module Sequent
|
|
232
240
|
# 2.1 Rename current tables with the +current version+ as SUFFIX
|
233
241
|
# 2.2 Rename the new tables and remove the +new version+ suffix
|
234
242
|
# 2.3 Add the new version in the +Versions+ table
|
235
|
-
# 3.
|
243
|
+
# 3. Update the versions table to complete the migration
|
236
244
|
#
|
237
245
|
# If anything fails an exception is raised and everything is rolled back
|
238
246
|
#
|
@@ -242,6 +250,8 @@ module Sequent
|
|
242
250
|
return if Sequent.new_version == current_version
|
243
251
|
|
244
252
|
ensure_version_correct!
|
253
|
+
in_view_schema { Versions.start_offline!(Sequent.new_version) }
|
254
|
+
Sequent.logger.info("Start migrate_offline for version #{Sequent.new_version}")
|
245
255
|
|
246
256
|
executor.set_table_names_to_new_version(plan)
|
247
257
|
|
@@ -249,8 +259,8 @@ module Sequent
|
|
249
259
|
if plan.projectors.any?
|
250
260
|
replay!(
|
251
261
|
Sequent.configuration.offline_replay_persistor_class.new,
|
252
|
-
|
253
|
-
|
262
|
+
groups: groups(group_exponent: 1),
|
263
|
+
minimum_xact_id_inclusive: Versions.running.first.xmin_xact_id,
|
254
264
|
)
|
255
265
|
end
|
256
266
|
|
@@ -259,22 +269,33 @@ module Sequent
|
|
259
269
|
# 2.1, 2.2
|
260
270
|
executor.execute_offline(plan, current_version)
|
261
271
|
# 2.3 Create migration record
|
262
|
-
Versions.
|
272
|
+
Versions.end_offline!(Sequent.new_version)
|
263
273
|
end
|
264
|
-
|
265
|
-
# 3. Truncate replayed ids
|
266
|
-
truncate_replay_ids_table!
|
267
274
|
end
|
268
275
|
logger.info "Migrated to version #{Sequent.new_version}"
|
269
|
-
|
270
|
-
|
271
|
-
|
276
|
+
rescue ConcurrentMigration
|
277
|
+
raise
|
278
|
+
rescue MigrationDone
|
279
|
+
# no-op same as Sequent.new_version == current_version
|
280
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
272
281
|
rollback_migration
|
273
282
|
raise e
|
274
283
|
end
|
275
284
|
|
276
285
|
private
|
277
286
|
|
287
|
+
def ensure_valid_plan!
|
288
|
+
plan
|
289
|
+
end
|
290
|
+
|
291
|
+
def migrate_metadata_tables
|
292
|
+
Sequent::ApplicationRecord.transaction do
|
293
|
+
in_view_schema do
|
294
|
+
exec_sql([Versions.migration_sql].join("\n"))
|
295
|
+
end
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
278
299
|
def ensure_version_correct!
|
279
300
|
create_view_schema_if_not_exists
|
280
301
|
new_version = Sequent.new_version
|
@@ -285,19 +306,22 @@ module Sequent
|
|
285
306
|
end
|
286
307
|
end
|
287
308
|
|
288
|
-
def replay!(
|
289
|
-
|
309
|
+
def replay!(
|
310
|
+
replay_persistor,
|
311
|
+
groups:,
|
312
|
+
projectors: plan.projectors,
|
313
|
+
minimum_xact_id_inclusive: nil,
|
314
|
+
maximum_xact_id_exclusive: nil
|
315
|
+
)
|
316
|
+
logger.info "groups: #{groups.size}"
|
290
317
|
|
291
318
|
with_sequent_config(replay_persistor, projectors) do
|
292
319
|
logger.info 'Start replaying events'
|
293
320
|
|
294
|
-
time("#{
|
321
|
+
time("#{groups.size} groups replayed") do
|
295
322
|
event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
|
296
323
|
disconnect!
|
297
324
|
|
298
|
-
number_of_groups = 16**group_exponent
|
299
|
-
groups = groups_of_aggregate_id_prefixes(number_of_groups)
|
300
|
-
|
301
325
|
@connected = false
|
302
326
|
# using `map_with_index` because https://github.com/grosser/parallel/issues/175
|
303
327
|
result = Parallel.map_with_index(
|
@@ -306,10 +330,17 @@ module Sequent
|
|
306
330
|
) do |aggregate_prefixes, index|
|
307
331
|
@connected ||= establish_connection
|
308
332
|
msg = <<~EOS.chomp
|
309
|
-
Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{
|
333
|
+
Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{groups.size} replayed
|
310
334
|
EOS
|
311
335
|
time(msg) do
|
312
|
-
replay_events(
|
336
|
+
replay_events(
|
337
|
+
aggregate_prefixes,
|
338
|
+
event_types,
|
339
|
+
minimum_xact_id_inclusive,
|
340
|
+
maximum_xact_id_exclusive,
|
341
|
+
replay_persistor,
|
342
|
+
&on_progress
|
343
|
+
)
|
313
344
|
end
|
314
345
|
nil
|
315
346
|
rescue StandardError => e
|
@@ -324,10 +355,19 @@ module Sequent
|
|
324
355
|
end
|
325
356
|
end
|
326
357
|
|
327
|
-
def replay_events(
|
358
|
+
def replay_events(
|
359
|
+
aggregate_prefixes,
|
360
|
+
event_types,
|
361
|
+
minimum_xact_id_inclusive,
|
362
|
+
maximum_xact_id_exclusive,
|
363
|
+
replay_persistor,
|
364
|
+
&on_progress
|
365
|
+
)
|
328
366
|
Sequent.configuration.event_store.replay_events_from_cursor(
|
329
367
|
block_size: 1000,
|
330
|
-
get_events: -> {
|
368
|
+
get_events: -> {
|
369
|
+
event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
|
370
|
+
},
|
331
371
|
on_progress: on_progress,
|
332
372
|
)
|
333
373
|
|
@@ -342,12 +382,16 @@ module Sequent
|
|
342
382
|
establish_connection
|
343
383
|
drop_old_tables(Sequent.new_version)
|
344
384
|
|
345
|
-
truncate_replay_ids_table!
|
346
385
|
executor.reset_table_names(plan)
|
386
|
+
Versions.rollback!(Sequent.new_version)
|
347
387
|
end
|
348
388
|
|
349
|
-
def
|
350
|
-
|
389
|
+
def groups(group_exponent: 3, limit: nil, offset: nil)
|
390
|
+
number_of_groups = 16**group_exponent
|
391
|
+
groups = groups_of_aggregate_id_prefixes(number_of_groups)
|
392
|
+
groups = groups.drop(offset) unless offset.nil?
|
393
|
+
groups = groups.take(limit) unless limit.nil?
|
394
|
+
groups
|
351
395
|
end
|
352
396
|
|
353
397
|
def groups_of_aggregate_id_prefixes(number_of_groups)
|
@@ -382,15 +426,8 @@ module Sequent
|
|
382
426
|
end
|
383
427
|
end
|
384
428
|
|
385
|
-
def
|
429
|
+
def on_progress
|
386
430
|
->(progress, done, ids) do
|
387
|
-
unless ids.empty?
|
388
|
-
exec_sql(
|
389
|
-
"insert into #{ReplayedIds.table_name} (event_id) values #{ids.map do |id|
|
390
|
-
"(#{id})"
|
391
|
-
end.join(',')}",
|
392
|
-
)
|
393
|
-
end
|
394
431
|
Sequent::Core::EventStore::PRINT_PROGRESS[progress, done, ids] if progress > 0
|
395
432
|
end
|
396
433
|
end
|
@@ -413,18 +450,24 @@ module Sequent
|
|
413
450
|
Sequent::Configuration.restore(old_config)
|
414
451
|
end
|
415
452
|
|
416
|
-
def event_stream(aggregate_prefixes, event_types,
|
453
|
+
def event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive)
|
417
454
|
fail ArgumentError, 'aggregate_prefixes is mandatory' unless aggregate_prefixes.present?
|
418
455
|
|
419
456
|
event_stream = Sequent.configuration.event_record_class.where(event_type: event_types)
|
420
457
|
event_stream = event_stream.where(<<~SQL, aggregate_prefixes)
|
421
|
-
substring(aggregate_id::
|
458
|
+
substring(aggregate_id::text from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?)
|
422
459
|
SQL
|
423
|
-
if
|
424
|
-
event_stream = event_stream
|
425
|
-
|
460
|
+
if minimum_xact_id_inclusive && maximum_xact_id_exclusive
|
461
|
+
event_stream = event_stream.where(
|
462
|
+
'xact_id >= ? AND xact_id < ?',
|
463
|
+
minimum_xact_id_inclusive,
|
464
|
+
maximum_xact_id_exclusive,
|
465
|
+
)
|
466
|
+
elsif minimum_xact_id_inclusive
|
467
|
+
event_stream = event_stream.where('xact_id >= ?', minimum_xact_id_inclusive)
|
468
|
+
elsif maximum_xact_id_exclusive
|
469
|
+
event_stream = event_stream.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive)
|
426
470
|
end
|
427
|
-
event_stream = event_stream.where('event_records.created_at > ?', 1.day.ago) if exclude_already_replayed
|
428
471
|
event_stream
|
429
472
|
.order('aggregate_id ASC, sequence_number ASC')
|
430
473
|
.select('id, event_type, event_json, sequence_number')
|
@@ -31,6 +31,12 @@ module Sequent
|
|
31
31
|
EOS
|
32
32
|
task init: :set_env_var
|
33
33
|
|
34
|
+
desc 'Creates sequent view schema if not exists and runs internal migrations'
|
35
|
+
task create_and_migrate_sequent_view_schema: ['sequent:init', :init] do
|
36
|
+
ensure_sequent_env_set!
|
37
|
+
Sequent::Migrations::ViewSchema.create_view_schema_if_not_exists(env: @env)
|
38
|
+
end
|
39
|
+
|
34
40
|
namespace :db do
|
35
41
|
desc 'Creates the database and initializes the event_store schema for the current env'
|
36
42
|
task create: ['sequent:init'] do
|
@@ -47,9 +53,9 @@ module Sequent
|
|
47
53
|
ensure_sequent_env_set!
|
48
54
|
|
49
55
|
if @env == 'production' && args[:production] != 'yes_drop_production'
|
50
|
-
fail <<~
|
56
|
+
fail <<~EOS
|
51
57
|
Wont drop db in production unless you whitelist the environment as follows: rake sequent:db:drop[yes_drop_production]
|
52
|
-
|
58
|
+
EOS
|
53
59
|
end
|
54
60
|
|
55
61
|
db_config = Sequent::Support::Database.read_config(@env)
|
@@ -91,12 +97,70 @@ module Sequent
|
|
91
97
|
task :init
|
92
98
|
|
93
99
|
desc 'Prints the current version in the database'
|
94
|
-
task current_version: [
|
95
|
-
|
100
|
+
task current_version: [:create_and_migrate_sequent_view_schema] do
|
101
|
+
puts "Current version in the database is: #{Sequent::Migrations::Versions.current_version}"
|
102
|
+
end
|
96
103
|
|
97
|
-
|
104
|
+
desc 'Returns whether a migration is currently running'
|
105
|
+
task check_running_migrations: [:create_and_migrate_sequent_view_schema] do
|
106
|
+
if Sequent::Migrations::Versions.running.any?
|
107
|
+
puts <<~EOS
|
108
|
+
Migration is running, current version: #{Sequent::Migrations::Versions.current_version},
|
109
|
+
target version #{Sequent::Migrations::Versions.version_currently_migrating}
|
110
|
+
EOS
|
111
|
+
else
|
112
|
+
puts 'No running migrations'
|
113
|
+
end
|
114
|
+
end
|
98
115
|
|
99
|
-
|
116
|
+
desc 'Returns whether a migration is pending'
|
117
|
+
task check_pending_migrations: [:create_and_migrate_sequent_view_schema] do
|
118
|
+
if Sequent.new_version != Sequent::Migrations::Versions.current_version
|
119
|
+
puts <<~EOS
|
120
|
+
Migration is pending, current version: #{Sequent::Migrations::Versions.current_version},
|
121
|
+
pending version: #{Sequent.new_version}
|
122
|
+
EOS
|
123
|
+
else
|
124
|
+
puts 'No pending migrations'
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
desc <<-EOS
|
129
|
+
Shows the current status of the migrations
|
130
|
+
EOS
|
131
|
+
task status: ['sequent:init', :init] do
|
132
|
+
ensure_sequent_env_set!
|
133
|
+
db_config = Sequent::Support::Database.read_config(@env)
|
134
|
+
view_schema = Sequent::Migrations::ViewSchema.new(db_config: db_config)
|
135
|
+
|
136
|
+
latest_done_version = Sequent::Migrations::Versions.done.latest
|
137
|
+
latest_version = Sequent::Migrations::Versions.latest
|
138
|
+
pending_version = Sequent.new_version
|
139
|
+
case latest_version.status
|
140
|
+
when Sequent::Migrations::Versions::DONE
|
141
|
+
if pending_version == latest_version.version
|
142
|
+
puts "Current version #{latest_version.version}, no pending changes"
|
143
|
+
else
|
144
|
+
puts "Current version #{latest_version.version}, pending version #{pending_version}"
|
145
|
+
end
|
146
|
+
when Sequent::Migrations::Versions::MIGRATE_ONLINE_RUNNING
|
147
|
+
puts "Online migration from #{latest_done_version.version} to #{latest_version.version} is running"
|
148
|
+
when Sequent::Migrations::Versions::MIGRATE_ONLINE_FINISHED
|
149
|
+
projectors = view_schema.plan.projectors
|
150
|
+
event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
|
151
|
+
|
152
|
+
current_snapshot_xmin_xact_id = Sequent::Migrations::Versions.current_snapshot_xmin_xact_id
|
153
|
+
pending_events = Sequent.configuration.event_record_class
|
154
|
+
.where(event_type: event_types)
|
155
|
+
.where('xact_id >= ?', current_snapshot_xmin_xact_id)
|
156
|
+
.count
|
157
|
+
print <<~EOS
|
158
|
+
Online migration from #{latest_done_version.version} to #{latest_version.version} is finished.
|
159
|
+
#{current_snapshot_xmin_xact_id - latest_version.xmin_xact_id} transactions behind current state (#{pending_events} pending events).
|
160
|
+
EOS
|
161
|
+
when Sequent::Migrations::Versions::MIGRATE_OFFLINE_RUNNING
|
162
|
+
puts "Offline migration from #{latest_done_version.version} to #{latest_version.version} is running"
|
163
|
+
end
|
100
164
|
end
|
101
165
|
|
102
166
|
desc <<~EOS
|
@@ -122,6 +186,25 @@ module Sequent
|
|
122
186
|
|
123
187
|
view_schema.migrate_offline
|
124
188
|
end
|
189
|
+
|
190
|
+
desc <<~EOS
|
191
|
+
Runs the projectors in replay mode without making any changes to the database, useful for (performance) testing against real data.
|
192
|
+
|
193
|
+
Pass a regular expression as parameter to select the projectors to run, otherwise all projectors are selected.
|
194
|
+
EOS
|
195
|
+
task :dryrun, %i[regex group_exponent limit offset] => ['sequent:init', :init] do |_task, args|
|
196
|
+
ensure_sequent_env_set!
|
197
|
+
|
198
|
+
db_config = Sequent::Support::Database.read_config(@env)
|
199
|
+
view_schema = Sequent::DryRun::ViewSchema.new(db_config: db_config)
|
200
|
+
|
201
|
+
view_schema.migrate_dryrun(
|
202
|
+
regex: args[:regex],
|
203
|
+
group_exponent: (args[:group_exponent] || 3).to_i,
|
204
|
+
limit: args[:limit]&.to_i,
|
205
|
+
offset: args[:offset]&.to_i,
|
206
|
+
)
|
207
|
+
end
|
125
208
|
end
|
126
209
|
|
127
210
|
namespace :snapshots do
|
@@ -53,7 +53,7 @@ module Sequent
|
|
53
53
|
db_config = db_config.deep_merge(
|
54
54
|
Sequent.configuration.primary_database_key => db_config_overrides,
|
55
55
|
).stringify_keys
|
56
|
-
if
|
56
|
+
if ActiveRecord::VERSION::MAJOR >= 7 && ActiveRecord::VERSION::MINOR < 1
|
57
57
|
ActiveRecord.legacy_connection_handling = false
|
58
58
|
end
|
59
59
|
ActiveRecord::Base.configurations = db_config.stringify_keys
|
@@ -132,16 +132,6 @@ module Sequent
|
|
132
132
|
self.class.execute_sql(sql)
|
133
133
|
end
|
134
134
|
|
135
|
-
def migrate(migrations_path, schema_migration: ActiveRecord::SchemaMigration, verbose: true)
|
136
|
-
ActiveRecord::Migration.verbose = verbose
|
137
|
-
if ActiveRecord::VERSION::MAJOR >= 6
|
138
|
-
ActiveRecord::MigrationContext.new([migrations_path], schema_migration).up
|
139
|
-
elsif ActiveRecord::VERSION::MAJOR >= 5 && ActiveRecord::VERSION::MINOR >= 2
|
140
|
-
ActiveRecord::MigrationContext.new([migrations_path]).up
|
141
|
-
else
|
142
|
-
ActiveRecord::Migrator.migrate(migrations_path)
|
143
|
-
end
|
144
|
-
end
|
145
135
|
end
|
146
136
|
end
|
147
137
|
end
|