sequent 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/sequent +31 -25
- data/lib/notices.rb +2 -0
- data/lib/sequent/application_record.rb +2 -0
- data/lib/sequent/configuration.rb +24 -31
- data/lib/sequent/core/aggregate_repository.rb +17 -13
- data/lib/sequent/core/aggregate_root.rb +16 -7
- data/lib/sequent/core/aggregate_roots.rb +24 -0
- data/lib/sequent/core/aggregate_snapshotter.rb +8 -5
- data/lib/sequent/core/base_command_handler.rb +4 -2
- data/lib/sequent/core/command.rb +17 -9
- data/lib/sequent/core/command_record.rb +8 -3
- data/lib/sequent/core/command_service.rb +18 -18
- data/lib/sequent/core/core.rb +2 -0
- data/lib/sequent/core/current_event.rb +2 -0
- data/lib/sequent/core/event.rb +16 -11
- data/lib/sequent/core/event_publisher.rb +16 -15
- data/lib/sequent/core/event_record.rb +7 -7
- data/lib/sequent/core/event_store.rb +57 -50
- data/lib/sequent/core/ext/ext.rb +9 -1
- data/lib/sequent/core/helpers/array_with_type.rb +4 -1
- data/lib/sequent/core/helpers/association_validator.rb +9 -7
- data/lib/sequent/core/helpers/attribute_support.rb +45 -28
- data/lib/sequent/core/helpers/autoset_attributes.rb +4 -4
- data/lib/sequent/core/helpers/boolean_validator.rb +6 -1
- data/lib/sequent/core/helpers/copyable.rb +2 -2
- data/lib/sequent/core/helpers/date_time_validator.rb +4 -1
- data/lib/sequent/core/helpers/date_validator.rb +6 -1
- data/lib/sequent/core/helpers/default_validators.rb +12 -10
- data/lib/sequent/core/helpers/equal_support.rb +8 -6
- data/lib/sequent/core/helpers/helpers.rb +2 -0
- data/lib/sequent/core/helpers/mergable.rb +6 -5
- data/lib/sequent/core/helpers/message_handler.rb +3 -1
- data/lib/sequent/core/helpers/param_support.rb +19 -15
- data/lib/sequent/core/helpers/secret.rb +14 -12
- data/lib/sequent/core/helpers/string_support.rb +5 -4
- data/lib/sequent/core/helpers/string_to_value_parsers.rb +7 -2
- data/lib/sequent/core/helpers/string_validator.rb +6 -1
- data/lib/sequent/core/helpers/type_conversion_support.rb +5 -3
- data/lib/sequent/core/helpers/uuid_helper.rb +5 -2
- data/lib/sequent/core/helpers/value_validators.rb +23 -9
- data/lib/sequent/core/persistors/active_record_persistor.rb +19 -9
- data/lib/sequent/core/persistors/persistor.rb +16 -14
- data/lib/sequent/core/persistors/persistors.rb +2 -0
- data/lib/sequent/core/persistors/replay_optimized_postgres_persistor.rb +70 -47
- data/lib/sequent/core/projector.rb +25 -22
- data/lib/sequent/core/random_uuid_generator.rb +2 -0
- data/lib/sequent/core/sequent_oj.rb +2 -0
- data/lib/sequent/core/stream_record.rb +9 -3
- data/lib/sequent/core/transactions/active_record_transaction_provider.rb +5 -9
- data/lib/sequent/core/transactions/no_transactions.rb +2 -1
- data/lib/sequent/core/transactions/transactions.rb +2 -0
- data/lib/sequent/core/value_object.rb +8 -10
- data/lib/sequent/core/workflow.rb +7 -5
- data/lib/sequent/generator/aggregate.rb +16 -10
- data/lib/sequent/generator/command.rb +26 -19
- data/lib/sequent/generator/event.rb +19 -17
- data/lib/sequent/generator/generator.rb +2 -0
- data/lib/sequent/generator/project.rb +2 -0
- data/lib/sequent/generator/template_project/Gemfile +1 -1
- data/lib/sequent/generator.rb +2 -0
- data/lib/sequent/migrations/executor.rb +22 -13
- data/lib/sequent/migrations/functions.rb +5 -6
- data/lib/sequent/migrations/migrate_events.rb +12 -9
- data/lib/sequent/migrations/migrations.rb +2 -1
- data/lib/sequent/migrations/planner.rb +33 -23
- data/lib/sequent/migrations/projectors.rb +4 -3
- data/lib/sequent/migrations/sql.rb +2 -0
- data/lib/sequent/migrations/view_schema.rb +84 -45
- data/lib/sequent/rake/migration_tasks.rb +58 -22
- data/lib/sequent/rake/tasks.rb +5 -2
- data/lib/sequent/sequent.rb +2 -0
- data/lib/sequent/support/database.rb +30 -15
- data/lib/sequent/support/view_projection.rb +6 -3
- data/lib/sequent/support/view_schema.rb +2 -0
- data/lib/sequent/support.rb +2 -0
- data/lib/sequent/test/command_handler_helpers.rb +35 -17
- data/lib/sequent/test/event_handler_helpers.rb +10 -4
- data/lib/sequent/test/event_stream_helpers.rb +7 -3
- data/lib/sequent/test/time_comparison.rb +12 -5
- data/lib/sequent/test.rb +2 -0
- data/lib/sequent/util/dry_run.rb +11 -8
- data/lib/sequent/util/printer.rb +6 -5
- data/lib/sequent/util/skip_if_already_processing.rb +3 -1
- data/lib/sequent/util/timer.rb +2 -0
- data/lib/sequent/util/util.rb +2 -0
- data/lib/sequent.rb +2 -0
- data/lib/version.rb +3 -1
- metadata +81 -66
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'parallel'
|
|
2
4
|
require 'postgresql_cursor'
|
|
3
5
|
|
|
@@ -24,11 +26,13 @@ module Sequent
|
|
|
24
26
|
# - AlterTable (For instance if you introduce a new column)
|
|
25
27
|
#
|
|
26
28
|
# To maintain your migrations you need to:
|
|
27
|
-
# 1. Create a class that extends `Sequent::Migrations::Projectors`
|
|
29
|
+
# 1. Create a class that extends `Sequent::Migrations::Projectors`
|
|
30
|
+
# and specify in `Sequent.configuration.migrations_class_name`
|
|
28
31
|
# 2. Define per version which migrations you want to execute
|
|
29
32
|
# See the definition of `Sequent::Migrations::Projectors.versions` and `Sequent::Migrations::Projectors.version`
|
|
30
33
|
# 3. Specify in Sequent where your sql files reside (Sequent.configuration.migration_sql_files_directory)
|
|
31
|
-
# 4. Ensure that you add %SUFFIX% to each name that needs to be unique in postgres
|
|
34
|
+
# 4. Ensure that you add %SUFFIX% to each name that needs to be unique in postgres
|
|
35
|
+
# (like TABLE names, INDEX names, PRIMARY KEYS)
|
|
32
36
|
# E.g. `create table foo%SUFFIX% (id serial NOT NULL, CONSTRAINT foo_pkey%SUFFIX% PRIMARY KEY (id))`
|
|
33
37
|
# 5. If you want to run an `alter_table` migration ensure that
|
|
34
38
|
# a sql file named `table_name_VERSION.sql` exists.
|
|
@@ -96,7 +100,10 @@ module Sequent
|
|
|
96
100
|
create_view_schema_if_not_exists
|
|
97
101
|
in_view_schema do
|
|
98
102
|
Sequent::Core::Migratable.all.flat_map(&:managed_tables).each do |table|
|
|
99
|
-
|
|
103
|
+
sql_file = "#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}.sql"
|
|
104
|
+
statements = sql_file_to_statements(sql_file) do |raw_sql|
|
|
105
|
+
raw_sql.remove('%SUFFIX%')
|
|
106
|
+
end
|
|
100
107
|
statements.each { |statement| exec_sql(statement) }
|
|
101
108
|
|
|
102
109
|
indexes_file_name = "#{Sequent.configuration.migration_sql_files_directory}/#{table.table_name}.indexes.sql"
|
|
@@ -121,10 +128,14 @@ module Sequent
|
|
|
121
128
|
#
|
|
122
129
|
# This method is mainly useful during an initial setup of the view schema
|
|
123
130
|
def create_view_schema_if_not_exists
|
|
124
|
-
exec_sql(%
|
|
131
|
+
exec_sql(%(CREATE SCHEMA IF NOT EXISTS #{view_schema}))
|
|
125
132
|
in_view_schema do
|
|
126
|
-
exec_sql(
|
|
127
|
-
|
|
133
|
+
exec_sql(<<~SQL.chomp)
|
|
134
|
+
CREATE TABLE IF NOT EXISTS #{Versions.table_name} (version integer NOT NULL, CONSTRAINT version_pk PRIMARY KEY(version))
|
|
135
|
+
SQL
|
|
136
|
+
exec_sql(<<~SQL.chomp)
|
|
137
|
+
CREATE TABLE IF NOT EXISTS #{ReplayedIds.table_name} (event_id bigint NOT NULL, CONSTRAINT event_id_pk PRIMARY KEY(event_id))
|
|
138
|
+
SQL
|
|
128
139
|
end
|
|
129
140
|
end
|
|
130
141
|
|
|
@@ -162,14 +173,14 @@ module Sequent
|
|
|
162
173
|
executor.execute_online(plan)
|
|
163
174
|
end
|
|
164
175
|
|
|
165
|
-
if plan.projectors.any?
|
|
166
|
-
replay!(Sequent.configuration.online_replay_persistor_class.new)
|
|
167
|
-
end
|
|
176
|
+
replay!(Sequent.configuration.online_replay_persistor_class.new) if plan.projectors.any?
|
|
168
177
|
|
|
169
178
|
in_view_schema do
|
|
170
179
|
executor.create_indexes_after_execute_online(plan)
|
|
171
180
|
end
|
|
181
|
+
# rubocop:disable Lint/RescueException
|
|
172
182
|
rescue Exception => e
|
|
183
|
+
# rubocop:enable Lint/RescueException
|
|
173
184
|
rollback_migration
|
|
174
185
|
raise e
|
|
175
186
|
end
|
|
@@ -201,7 +212,13 @@ module Sequent
|
|
|
201
212
|
executor.set_table_names_to_new_version(plan)
|
|
202
213
|
|
|
203
214
|
# 1 replay events not yet replayed
|
|
204
|
-
|
|
215
|
+
if plan.projectors.any?
|
|
216
|
+
replay!(
|
|
217
|
+
Sequent.configuration.offline_replay_persistor_class.new,
|
|
218
|
+
exclude_ids: true,
|
|
219
|
+
group_exponent: 1,
|
|
220
|
+
)
|
|
221
|
+
end
|
|
205
222
|
|
|
206
223
|
in_view_schema do
|
|
207
224
|
Sequent::ApplicationRecord.transaction do
|
|
@@ -215,50 +232,57 @@ module Sequent
|
|
|
215
232
|
truncate_replay_ids_table!
|
|
216
233
|
end
|
|
217
234
|
logger.info "Migrated to version #{Sequent.new_version}"
|
|
235
|
+
# rubocop:disable Lint/RescueException
|
|
218
236
|
rescue Exception => e
|
|
237
|
+
# rubocop:enable Lint/RescueException
|
|
219
238
|
rollback_migration
|
|
220
239
|
raise e
|
|
221
240
|
end
|
|
222
241
|
|
|
223
242
|
private
|
|
224
243
|
|
|
225
|
-
|
|
226
244
|
def ensure_version_correct!
|
|
227
245
|
create_view_schema_if_not_exists
|
|
228
246
|
new_version = Sequent.new_version
|
|
229
247
|
|
|
230
|
-
|
|
231
|
-
|
|
248
|
+
if new_version < current_version
|
|
249
|
+
fail ArgumentError,
|
|
250
|
+
"new_version [#{new_version}] must be greater or equal to current_version [#{current_version}]"
|
|
251
|
+
end
|
|
232
252
|
end
|
|
233
253
|
|
|
234
254
|
def replay!(replay_persistor, projectors: plan.projectors, exclude_ids: false, group_exponent: 3)
|
|
235
255
|
logger.info "group_exponent: #{group_exponent.inspect}"
|
|
236
256
|
|
|
237
257
|
with_sequent_config(replay_persistor, projectors) do
|
|
238
|
-
logger.info
|
|
258
|
+
logger.info 'Start replaying events'
|
|
239
259
|
|
|
240
|
-
time("#{16
|
|
260
|
+
time("#{16**group_exponent} groups replayed") do
|
|
241
261
|
event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name)
|
|
242
262
|
disconnect!
|
|
243
263
|
|
|
244
|
-
number_of_groups = 16
|
|
264
|
+
number_of_groups = 16**group_exponent
|
|
245
265
|
groups = groups_of_aggregate_id_prefixes(number_of_groups)
|
|
246
266
|
|
|
247
267
|
@connected = false
|
|
248
268
|
# using `map_with_index` because https://github.com/grosser/parallel/issues/175
|
|
249
|
-
result = Parallel.map_with_index(
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
recursively_print(e)
|
|
260
|
-
raise Parallel::Kill # immediately kill all sub-processes
|
|
269
|
+
result = Parallel.map_with_index(
|
|
270
|
+
groups,
|
|
271
|
+
in_processes: Sequent.configuration.number_of_replay_processes,
|
|
272
|
+
) do |aggregate_prefixes, index|
|
|
273
|
+
@connected ||= establish_connection
|
|
274
|
+
msg = <<~EOS.chomp
|
|
275
|
+
Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{number_of_groups} replayed
|
|
276
|
+
EOS
|
|
277
|
+
time(msg) do
|
|
278
|
+
replay_events(aggregate_prefixes, event_types, exclude_ids, replay_persistor, &insert_ids)
|
|
261
279
|
end
|
|
280
|
+
nil
|
|
281
|
+
rescue StandardError => e
|
|
282
|
+
logger.error "Replaying failed for ids: ^#{aggregate_prefixes.first} - #{aggregate_prefixes.last}"
|
|
283
|
+
logger.error '+++++++++++++++ ERROR +++++++++++++++'
|
|
284
|
+
recursively_print(e)
|
|
285
|
+
raise Parallel::Kill # immediately kill all sub-processes
|
|
262
286
|
end
|
|
263
287
|
establish_connection
|
|
264
288
|
fail if result.nil?
|
|
@@ -270,7 +294,7 @@ module Sequent
|
|
|
270
294
|
Sequent.configuration.event_store.replay_events_from_cursor(
|
|
271
295
|
block_size: 1000,
|
|
272
296
|
get_events: -> { event_stream(aggregate_prefixes, event_types, exclude_already_replayed) },
|
|
273
|
-
on_progress: on_progress
|
|
297
|
+
on_progress: on_progress,
|
|
274
298
|
)
|
|
275
299
|
|
|
276
300
|
replay_persistor.commit
|
|
@@ -293,29 +317,31 @@ module Sequent
|
|
|
293
317
|
end
|
|
294
318
|
|
|
295
319
|
def groups_of_aggregate_id_prefixes(number_of_groups)
|
|
296
|
-
all_prefixes = (0...16
|
|
297
|
-
|
|
320
|
+
all_prefixes = (0...16**LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE).to_a.map do |i|
|
|
321
|
+
i.to_s(16)
|
|
322
|
+
end
|
|
323
|
+
all_prefixes = all_prefixes.map { |s| s.length == 3 ? s : "#{'0' * (3 - s.length)}#{s}" }
|
|
298
324
|
|
|
299
325
|
logger.info "Number of groups #{number_of_groups}"
|
|
300
326
|
|
|
301
327
|
logger.debug "Prefixes: #{all_prefixes.length}"
|
|
302
|
-
|
|
328
|
+
if number_of_groups > all_prefixes.length
|
|
329
|
+
fail "Can not have more groups #{number_of_groups} than number of prefixes #{all_prefixes.length}"
|
|
330
|
+
end
|
|
303
331
|
|
|
304
332
|
all_prefixes.each_slice(all_prefixes.length / number_of_groups).to_a
|
|
305
333
|
end
|
|
306
334
|
|
|
307
|
-
def in_view_schema
|
|
308
|
-
Sequent::Support::Database.with_schema_search_path(view_schema, db_config)
|
|
309
|
-
yield
|
|
310
|
-
end
|
|
335
|
+
def in_view_schema(&block)
|
|
336
|
+
Sequent::Support::Database.with_schema_search_path(view_schema, db_config, &block)
|
|
311
337
|
end
|
|
312
338
|
|
|
313
339
|
def drop_old_tables(new_version)
|
|
314
340
|
versions_to_check = (current_version - 10)..new_version
|
|
315
341
|
old_tables = versions_to_check.flat_map do |old_version|
|
|
316
|
-
exec_sql(
|
|
317
|
-
|
|
318
|
-
|
|
342
|
+
exec_sql(<<~SQL).flat_map(&:values)
|
|
343
|
+
select table_name from information_schema.tables where table_schema = '#{Sequent.configuration.view_schema_name}' and table_name LIKE '%_#{old_version}'
|
|
344
|
+
SQL
|
|
319
345
|
end
|
|
320
346
|
old_tables.each do |old_table|
|
|
321
347
|
exec_sql("DROP TABLE #{Sequent.configuration.view_schema_name}.#{old_table} CASCADE")
|
|
@@ -324,7 +350,13 @@ module Sequent
|
|
|
324
350
|
|
|
325
351
|
def insert_ids
|
|
326
352
|
->(progress, done, ids) do
|
|
327
|
-
|
|
353
|
+
unless ids.empty?
|
|
354
|
+
exec_sql(
|
|
355
|
+
"insert into #{ReplayedIds.table_name} (event_id) values #{ids.map do |id|
|
|
356
|
+
"(#{id})"
|
|
357
|
+
end.join(',')}",
|
|
358
|
+
)
|
|
359
|
+
end
|
|
328
360
|
Sequent::Core::EventStore::PRINT_PROGRESS[progress, done, ids] if progress > 0
|
|
329
361
|
end
|
|
330
362
|
end
|
|
@@ -334,7 +366,9 @@ module Sequent
|
|
|
334
366
|
|
|
335
367
|
config = Sequent.configuration.dup
|
|
336
368
|
|
|
337
|
-
replay_projectors = projectors.map
|
|
369
|
+
replay_projectors = projectors.map do |projector_class|
|
|
370
|
+
projector_class.new(projector_class.replay_persistor || replay_persistor)
|
|
371
|
+
end
|
|
338
372
|
config.transaction_provider = Sequent::Core::Transactions::NoTransactions.new
|
|
339
373
|
config.event_handlers = replay_projectors
|
|
340
374
|
|
|
@@ -346,12 +380,17 @@ module Sequent
|
|
|
346
380
|
end
|
|
347
381
|
|
|
348
382
|
def event_stream(aggregate_prefixes, event_types, exclude_already_replayed)
|
|
349
|
-
fail ArgumentError
|
|
383
|
+
fail ArgumentError, 'aggregate_prefixes is mandatory' unless aggregate_prefixes.present?
|
|
350
384
|
|
|
351
385
|
event_stream = Sequent.configuration.event_record_class.where(event_type: event_types)
|
|
352
|
-
event_stream = event_stream.where(
|
|
353
|
-
|
|
354
|
-
|
|
386
|
+
event_stream = event_stream.where(<<~SQL, aggregate_prefixes)
|
|
387
|
+
substring(aggregate_id::varchar from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?)
|
|
388
|
+
SQL
|
|
389
|
+
if exclude_already_replayed
|
|
390
|
+
event_stream = event_stream
|
|
391
|
+
.where("NOT EXISTS (SELECT 1 FROM #{ReplayedIds.table_name} WHERE event_id = event_records.id)")
|
|
392
|
+
end
|
|
393
|
+
event_stream = event_stream.where('event_records.created_at > ?', 1.day.ago) if exclude_already_replayed
|
|
355
394
|
event_stream.order('sequence_number ASC').select('id, event_type, event_json, sequence_number')
|
|
356
395
|
end
|
|
357
396
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'active_record'
|
|
2
4
|
require 'rake'
|
|
3
5
|
require 'rake/tasklib'
|
|
@@ -12,13 +14,14 @@ module Sequent
|
|
|
12
14
|
|
|
13
15
|
def register_tasks!
|
|
14
16
|
namespace :sequent do
|
|
15
|
-
desc
|
|
17
|
+
desc <<~EOS
|
|
18
|
+
Rake task that runs before all sequent rake tasks. Hook applications can use to for instance run other rake tasks.
|
|
19
|
+
EOS
|
|
16
20
|
task :init
|
|
17
21
|
|
|
18
22
|
namespace :db do
|
|
19
|
-
|
|
20
23
|
desc 'Creates the database and initializes the event_store schema for the current env'
|
|
21
|
-
task :
|
|
24
|
+
task create: ['sequent:init'] do
|
|
22
25
|
ensure_rack_env_set!
|
|
23
26
|
|
|
24
27
|
db_config = Sequent::Support::Database.read_config(@env)
|
|
@@ -31,14 +34,18 @@ module Sequent
|
|
|
31
34
|
task :drop, [:production] => ['sequent:init'] do |_t, args|
|
|
32
35
|
ensure_rack_env_set!
|
|
33
36
|
|
|
34
|
-
|
|
37
|
+
if @env == 'production' && args[:production] != 'yes_drop_production'
|
|
38
|
+
fail <<~OES
|
|
39
|
+
Wont drop db in production unless you whitelist the environment as follows: rake sequent:db:drop[yes_drop_production]
|
|
40
|
+
OES
|
|
41
|
+
end
|
|
35
42
|
|
|
36
43
|
db_config = Sequent::Support::Database.read_config(@env)
|
|
37
44
|
Sequent::Support::Database.drop!(db_config)
|
|
38
45
|
end
|
|
39
46
|
|
|
40
47
|
desc 'Creates the view schema for the current env'
|
|
41
|
-
task :
|
|
48
|
+
task create_view_schema: ['sequent:init'] do
|
|
42
49
|
ensure_rack_env_set!
|
|
43
50
|
|
|
44
51
|
db_config = Sequent::Support::Database.read_config(@env)
|
|
@@ -47,33 +54,41 @@ module Sequent
|
|
|
47
54
|
end
|
|
48
55
|
|
|
49
56
|
desc 'Creates the event_store schema for the current env'
|
|
50
|
-
task :
|
|
57
|
+
task create_event_store: ['sequent:init'] do
|
|
51
58
|
ensure_rack_env_set!
|
|
52
59
|
db_config = Sequent::Support::Database.read_config(@env)
|
|
53
60
|
create_event_store(db_config)
|
|
54
61
|
end
|
|
55
62
|
|
|
63
|
+
# rubocop:disable Lint/NestedMethodDefinition
|
|
56
64
|
def create_event_store(db_config)
|
|
57
65
|
event_store_schema = Sequent.configuration.event_store_schema_name
|
|
58
66
|
sequent_schema = File.join(Sequent.configuration.database_schema_directory, "#{event_store_schema}.rb")
|
|
59
|
-
|
|
67
|
+
unless File.exist?(sequent_schema)
|
|
68
|
+
fail "File #{sequent_schema} does not exist. Check your Sequent configuration."
|
|
69
|
+
end
|
|
60
70
|
|
|
61
71
|
Sequent::Support::Database.establish_connection(db_config)
|
|
62
|
-
|
|
72
|
+
if Sequent::Support::Database.schema_exists?(event_store_schema)
|
|
73
|
+
fail "Schema #{event_store_schema} already exists in the database"
|
|
74
|
+
end
|
|
63
75
|
|
|
64
76
|
Sequent::Support::Database.create_schema(event_store_schema)
|
|
65
77
|
Sequent::Support::Database.with_schema_search_path(event_store_schema, db_config, @env) do
|
|
66
78
|
load(sequent_schema)
|
|
67
79
|
end
|
|
68
80
|
end
|
|
81
|
+
# rubocop:enable Lint/NestedMethodDefinition
|
|
69
82
|
end
|
|
70
83
|
|
|
71
84
|
namespace :migrate do
|
|
72
|
-
desc
|
|
85
|
+
desc <<~EOS
|
|
86
|
+
Rake task that runs before all migrate rake tasks. Hook applications can use to for instance run other rake tasks.
|
|
87
|
+
EOS
|
|
73
88
|
task :init
|
|
74
89
|
|
|
75
90
|
desc 'Prints the current version in the database'
|
|
76
|
-
task :
|
|
91
|
+
task current_version: ['sequent:init', :init] do
|
|
77
92
|
ensure_rack_env_set!
|
|
78
93
|
|
|
79
94
|
Sequent::Support::Database.connect!(@env)
|
|
@@ -81,8 +96,10 @@ module Sequent
|
|
|
81
96
|
puts "Current version in the database is: #{Sequent::Migrations::ViewSchema::Versions.maximum(:version)}"
|
|
82
97
|
end
|
|
83
98
|
|
|
84
|
-
desc
|
|
85
|
-
|
|
99
|
+
desc <<~EOS
|
|
100
|
+
Migrates the Projectors while the app is running. Call +sequent:migrate:offline+ after this successfully completed.
|
|
101
|
+
EOS
|
|
102
|
+
task online: ['sequent:init', :init] do
|
|
86
103
|
ensure_rack_env_set!
|
|
87
104
|
|
|
88
105
|
db_config = Sequent::Support::Database.read_config(@env)
|
|
@@ -91,8 +108,10 @@ module Sequent
|
|
|
91
108
|
view_schema.migrate_online
|
|
92
109
|
end
|
|
93
110
|
|
|
94
|
-
desc
|
|
95
|
-
|
|
111
|
+
desc <<~EOS
|
|
112
|
+
Migrates the events inserted while +online+ was running. It is expected +sequent:migrate:online+ ran first.
|
|
113
|
+
EOS
|
|
114
|
+
task offline: ['sequent:init', :init] do
|
|
96
115
|
ensure_rack_env_set!
|
|
97
116
|
|
|
98
117
|
db_config = Sequent::Support::Database.read_config(@env)
|
|
@@ -103,21 +122,35 @@ module Sequent
|
|
|
103
122
|
end
|
|
104
123
|
|
|
105
124
|
namespace :snapshots do
|
|
106
|
-
desc
|
|
125
|
+
desc <<~EOS
|
|
126
|
+
Rake task that runs before all snapshots rake tasks. Hook applications can use to for instance run other rake tasks.
|
|
127
|
+
EOS
|
|
107
128
|
task :init
|
|
108
129
|
|
|
109
|
-
task :set_snapshot_threshold, [
|
|
130
|
+
task :set_snapshot_threshold, %i[aggregate_type threshold] => ['sequent:init', :init] do |_t, args|
|
|
110
131
|
aggregate_type = args['aggregate_type']
|
|
111
132
|
threshold = args['threshold']
|
|
112
133
|
|
|
113
|
-
|
|
114
|
-
|
|
134
|
+
unless aggregate_type
|
|
135
|
+
fail ArgumentError,
|
|
136
|
+
'usage rake sequent:snapshots:set_snapshot_threshold[AggregegateType,threshold]'
|
|
137
|
+
end
|
|
138
|
+
unless threshold
|
|
139
|
+
fail ArgumentError,
|
|
140
|
+
'usage rake sequent:snapshots:set_snapshot_threshold[AggregegateType,threshold]'
|
|
141
|
+
end
|
|
115
142
|
|
|
116
|
-
execute
|
|
143
|
+
execute <<~EOS
|
|
144
|
+
UPDATE #{Sequent.configuration.stream_record_class} SET snapshot_threshold = #{threshold.to_i} WHERE aggregate_type = '#{aggregate_type}'
|
|
145
|
+
EOS
|
|
117
146
|
end
|
|
118
147
|
|
|
119
|
-
task :
|
|
120
|
-
result = Sequent::ApplicationRecord
|
|
148
|
+
task delete_all: ['sequent:init', :init] do
|
|
149
|
+
result = Sequent::ApplicationRecord
|
|
150
|
+
.connection
|
|
151
|
+
.execute(<<~EOS)
|
|
152
|
+
DELETE FROM #{Sequent.configuration.event_record_class.table_name} WHERE event_type = 'Sequent::Core::SnapshotEvent'
|
|
153
|
+
EOS
|
|
121
154
|
Sequent.logger.info "Deleted #{result.cmd_tuples} aggregate snapshots from the event store"
|
|
122
155
|
end
|
|
123
156
|
end
|
|
@@ -125,9 +158,12 @@ module Sequent
|
|
|
125
158
|
end
|
|
126
159
|
|
|
127
160
|
private
|
|
161
|
+
|
|
162
|
+
# rubocop:disable Naming/MemoizedInstanceVariableName
|
|
128
163
|
def ensure_rack_env_set!
|
|
129
|
-
@env ||= ENV['RACK_ENV'] || fail(
|
|
164
|
+
@env ||= ENV['RACK_ENV'] || fail('RACK_ENV not set')
|
|
130
165
|
end
|
|
166
|
+
# rubocop:enable Naming/MemoizedInstanceVariableName
|
|
131
167
|
end
|
|
132
168
|
end
|
|
133
169
|
end
|
data/lib/sequent/rake/tasks.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'active_record'
|
|
2
4
|
require 'rake'
|
|
3
5
|
require 'rake/tasklib'
|
|
@@ -11,12 +13,13 @@ module Sequent
|
|
|
11
13
|
|
|
12
14
|
DEFAULT_OPTIONS = {
|
|
13
15
|
migrations_path: 'db/migrate',
|
|
14
|
-
event_store_schema: 'public'
|
|
15
|
-
}
|
|
16
|
+
event_store_schema: 'public',
|
|
17
|
+
}.freeze
|
|
16
18
|
|
|
17
19
|
attr_reader :options
|
|
18
20
|
|
|
19
21
|
def initialize(options)
|
|
22
|
+
super()
|
|
20
23
|
@options = DEFAULT_OPTIONS.merge(options)
|
|
21
24
|
end
|
|
22
25
|
|
data/lib/sequent/sequent.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'active_support/hash_with_indifferent_access'
|
|
2
4
|
|
|
3
5
|
module Sequent
|
|
@@ -16,20 +18,25 @@ module Sequent
|
|
|
16
18
|
end
|
|
17
19
|
|
|
18
20
|
def self.read_config(env)
|
|
19
|
-
fail ArgumentError
|
|
21
|
+
fail ArgumentError, 'env is mandatory' unless env
|
|
20
22
|
|
|
21
23
|
database_yml = File.join(Sequent.configuration.database_config_directory, 'database.yml')
|
|
22
|
-
YAML.
|
|
24
|
+
config = YAML.safe_load(ERB.new(File.read(database_yml)).result, aliases: true)[env]
|
|
25
|
+
if Gem.loaded_specs['activerecord'].version >= Gem::Version.create('6.1.0')
|
|
26
|
+
ActiveRecord::Base.configurations.resolve(config).configuration_hash.with_indifferent_access
|
|
27
|
+
else
|
|
28
|
+
ActiveRecord::Base.resolve_config_for_connection(config)
|
|
29
|
+
end
|
|
23
30
|
end
|
|
24
31
|
|
|
25
32
|
def self.create!(db_config)
|
|
26
|
-
ActiveRecord::Base.establish_connection(db_config.merge(
|
|
27
|
-
ActiveRecord::Base.connection.create_database(db_config[
|
|
33
|
+
ActiveRecord::Base.establish_connection(db_config.merge(database: 'postgres'))
|
|
34
|
+
ActiveRecord::Base.connection.create_database(db_config[:database])
|
|
28
35
|
end
|
|
29
36
|
|
|
30
37
|
def self.drop!(db_config)
|
|
31
|
-
ActiveRecord::Base.establish_connection(db_config.merge(
|
|
32
|
-
ActiveRecord::Base.connection.drop_database(db_config[
|
|
38
|
+
ActiveRecord::Base.establish_connection(db_config.merge(database: 'postgres'))
|
|
39
|
+
ActiveRecord::Base.connection.drop_database(db_config[:database])
|
|
33
40
|
end
|
|
34
41
|
|
|
35
42
|
def self.establish_connection(db_config)
|
|
@@ -46,9 +53,8 @@ module Sequent
|
|
|
46
53
|
|
|
47
54
|
def self.create_schema(schema)
|
|
48
55
|
sql = "CREATE SCHEMA IF NOT EXISTS #{schema}"
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
end
|
|
56
|
+
user = configuration_hash[:username]
|
|
57
|
+
sql += %( AUTHORIZATION "#{user}") if user
|
|
52
58
|
execute_sql(sql)
|
|
53
59
|
end
|
|
54
60
|
|
|
@@ -57,32 +63,41 @@ module Sequent
|
|
|
57
63
|
end
|
|
58
64
|
|
|
59
65
|
def self.with_schema_search_path(search_path, db_config, env = ENV['RACK_ENV'])
|
|
60
|
-
fail ArgumentError
|
|
66
|
+
fail ArgumentError, 'env is required' unless env
|
|
61
67
|
|
|
62
68
|
disconnect!
|
|
63
|
-
original_search_paths = db_config[
|
|
69
|
+
original_search_paths = db_config[:schema_search_path].dup
|
|
64
70
|
|
|
65
71
|
if ActiveRecord::VERSION::MAJOR < 6
|
|
66
|
-
ActiveRecord::Base.configurations[env.to_s] =
|
|
72
|
+
ActiveRecord::Base.configurations[env.to_s] =
|
|
73
|
+
ActiveSupport::HashWithIndifferentAccess.new(db_config).stringify_keys
|
|
67
74
|
end
|
|
68
75
|
|
|
69
|
-
db_config[
|
|
76
|
+
db_config[:schema_search_path] = search_path
|
|
70
77
|
|
|
71
78
|
ActiveRecord::Base.establish_connection db_config
|
|
72
79
|
|
|
73
80
|
yield
|
|
74
81
|
ensure
|
|
75
82
|
disconnect!
|
|
76
|
-
db_config[
|
|
83
|
+
db_config[:schema_search_path] = original_search_paths
|
|
77
84
|
establish_connection(db_config)
|
|
78
85
|
end
|
|
79
86
|
|
|
80
87
|
def self.schema_exists?(schema)
|
|
81
88
|
ActiveRecord::Base.connection.execute(
|
|
82
|
-
"SELECT schema_name FROM information_schema.schemata WHERE schema_name like '#{schema}'"
|
|
89
|
+
"SELECT schema_name FROM information_schema.schemata WHERE schema_name like '#{schema}'",
|
|
83
90
|
).count == 1
|
|
84
91
|
end
|
|
85
92
|
|
|
93
|
+
def self.configuration_hash
|
|
94
|
+
if Gem.loaded_specs['activesupport'].version >= Gem::Version.create('6.1.0')
|
|
95
|
+
ActiveRecord::Base.connection_db_config.configuration_hash
|
|
96
|
+
else
|
|
97
|
+
ActiveRecord::Base.connection_config
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
86
101
|
def schema_exists?(schema)
|
|
87
102
|
self.class.schema_exists?(schema)
|
|
88
103
|
end
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'postgresql_cursor'
|
|
2
4
|
|
|
3
5
|
module Sequent
|
|
4
6
|
module Support
|
|
5
7
|
class ViewProjection
|
|
6
8
|
attr_reader :name, :version, :schema_definition
|
|
9
|
+
|
|
7
10
|
def initialize(options)
|
|
8
11
|
@name = options.fetch(:name)
|
|
9
12
|
@version = options.fetch(:version)
|
|
@@ -20,7 +23,7 @@ module Sequent
|
|
|
20
23
|
ordering = Events::ORDERED_BY_STREAM
|
|
21
24
|
event_store.replay_events_from_cursor(
|
|
22
25
|
block_size: 10_000,
|
|
23
|
-
get_events: ->
|
|
26
|
+
get_events: -> { ordering[event_store] },
|
|
24
27
|
)
|
|
25
28
|
end
|
|
26
29
|
end
|
|
@@ -42,13 +45,13 @@ module Sequent
|
|
|
42
45
|
module Events
|
|
43
46
|
extend ActiveRecord::ConnectionAdapters::Quoting
|
|
44
47
|
|
|
45
|
-
ORDERED_BY_STREAM =
|
|
48
|
+
ORDERED_BY_STREAM = ->(_event_store) do
|
|
46
49
|
event_records = quote_table_name(Sequent.configuration.event_record_class.table_name)
|
|
47
50
|
stream_records = quote_table_name(Sequent.configuration.stream_record_class.table_name)
|
|
48
51
|
snapshot_event_type = quote(Sequent.configuration.snapshot_event_class)
|
|
49
52
|
|
|
50
53
|
Sequent.configuration.event_record_class
|
|
51
|
-
.select(
|
|
54
|
+
.select('event_type, event_json')
|
|
52
55
|
.joins("INNER JOIN #{stream_records} ON #{event_records}.stream_record_id = #{stream_records}.id")
|
|
53
56
|
.where("event_type <> #{snapshot_event_type}")
|
|
54
57
|
.order!("#{stream_records}.id, #{event_records}.sequence_number")
|