activerecord 7.2.3.1 → 8.0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +475 -946
- data/README.rdoc +1 -1
- data/lib/active_record/association_relation.rb +1 -0
- data/lib/active_record/associations/association.rb +34 -10
- data/lib/active_record/associations/builder/association.rb +7 -6
- data/lib/active_record/associations/collection_association.rb +1 -1
- data/lib/active_record/associations/disable_joins_association_scope.rb +1 -1
- data/lib/active_record/associations/has_many_through_association.rb +3 -2
- data/lib/active_record/associations/join_dependency.rb +2 -2
- data/lib/active_record/associations/preloader/association.rb +2 -2
- data/lib/active_record/associations/preloader/batch.rb +7 -1
- data/lib/active_record/associations/singular_association.rb +8 -3
- data/lib/active_record/associations.rb +34 -4
- data/lib/active_record/asynchronous_queries_tracker.rb +28 -24
- data/lib/active_record/attribute_methods/primary_key.rb +4 -8
- data/lib/active_record/attribute_methods/query.rb +34 -0
- data/lib/active_record/attribute_methods/time_zone_conversion.rb +2 -12
- data/lib/active_record/autosave_association.rb +69 -27
- data/lib/active_record/connection_adapters/abstract/connection_handler.rb +34 -25
- data/lib/active_record/connection_adapters/abstract/connection_pool/queue.rb +0 -1
- data/lib/active_record/connection_adapters/abstract/connection_pool/reaper.rb +0 -1
- data/lib/active_record/connection_adapters/abstract/connection_pool.rb +8 -17
- data/lib/active_record/connection_adapters/abstract/database_statements.rb +91 -44
- data/lib/active_record/connection_adapters/abstract/query_cache.rb +8 -2
- data/lib/active_record/connection_adapters/abstract/quoting.rb +1 -1
- data/lib/active_record/connection_adapters/abstract/schema_creation.rb +4 -5
- data/lib/active_record/connection_adapters/abstract/schema_definitions.rb +7 -2
- data/lib/active_record/connection_adapters/abstract/schema_statements.rb +37 -10
- data/lib/active_record/connection_adapters/abstract/transaction.rb +15 -5
- data/lib/active_record/connection_adapters/abstract_adapter.rb +67 -74
- data/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +21 -40
- data/lib/active_record/connection_adapters/mysql/quoting.rb +0 -8
- data/lib/active_record/connection_adapters/mysql/schema_definitions.rb +2 -8
- data/lib/active_record/connection_adapters/mysql/schema_statements.rb +50 -45
- data/lib/active_record/connection_adapters/mysql2/database_statements.rb +84 -94
- data/lib/active_record/connection_adapters/mysql2_adapter.rb +1 -8
- data/lib/active_record/connection_adapters/pool_config.rb +7 -7
- data/lib/active_record/connection_adapters/postgresql/column.rb +4 -2
- data/lib/active_record/connection_adapters/postgresql/database_statements.rb +72 -43
- data/lib/active_record/connection_adapters/postgresql/oid/array.rb +1 -1
- data/lib/active_record/connection_adapters/postgresql/oid/point.rb +10 -0
- data/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +2 -4
- data/lib/active_record/connection_adapters/postgresql/schema_creation.rb +1 -11
- data/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +6 -12
- data/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +2 -1
- data/lib/active_record/connection_adapters/postgresql/schema_statements.rb +59 -16
- data/lib/active_record/connection_adapters/postgresql_adapter.rb +53 -96
- data/lib/active_record/connection_adapters/schema_cache.rb +1 -3
- data/lib/active_record/connection_adapters/sqlite3/column.rb +8 -2
- data/lib/active_record/connection_adapters/sqlite3/database_statements.rb +80 -100
- data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +0 -6
- data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb +27 -2
- data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb +9 -1
- data/lib/active_record/connection_adapters/sqlite3_adapter.rb +53 -12
- data/lib/active_record/connection_adapters/statement_pool.rb +4 -2
- data/lib/active_record/connection_adapters/trilogy/database_statements.rb +37 -67
- data/lib/active_record/connection_adapters/trilogy_adapter.rb +0 -17
- data/lib/active_record/connection_adapters.rb +0 -56
- data/lib/active_record/connection_handling.rb +23 -1
- data/lib/active_record/core.rb +29 -14
- data/lib/active_record/database_configurations/database_config.rb +4 -0
- data/lib/active_record/database_configurations/hash_config.rb +16 -2
- data/lib/active_record/encryption/config.rb +3 -1
- data/lib/active_record/encryption/encryptable_record.rb +4 -4
- data/lib/active_record/encryption/encrypted_attribute_type.rb +10 -1
- data/lib/active_record/encryption/encryptor.rb +16 -8
- data/lib/active_record/encryption/extended_deterministic_queries.rb +4 -2
- data/lib/active_record/encryption/scheme.rb +8 -1
- data/lib/active_record/enum.rb +9 -22
- data/lib/active_record/errors.rb +13 -5
- data/lib/active_record/fixtures.rb +0 -2
- data/lib/active_record/future_result.rb +15 -9
- data/lib/active_record/gem_version.rb +3 -3
- data/lib/active_record/insert_all.rb +3 -3
- data/lib/active_record/locking/optimistic.rb +1 -1
- data/lib/active_record/log_subscriber.rb +5 -11
- data/lib/active_record/migration/command_recorder.rb +32 -12
- data/lib/active_record/migration/compatibility.rb +5 -2
- data/lib/active_record/migration.rb +38 -42
- data/lib/active_record/model_schema.rb +3 -4
- data/lib/active_record/nested_attributes.rb +4 -6
- data/lib/active_record/persistence.rb +128 -130
- data/lib/active_record/query_logs.rb +102 -50
- data/lib/active_record/query_logs_formatter.rb +17 -28
- data/lib/active_record/querying.rb +8 -8
- data/lib/active_record/railtie.rb +2 -26
- data/lib/active_record/railties/databases.rake +11 -35
- data/lib/active_record/reflection.rb +18 -21
- data/lib/active_record/relation/batches/batch_enumerator.rb +4 -3
- data/lib/active_record/relation/batches.rb +132 -72
- data/lib/active_record/relation/calculations.rb +40 -39
- data/lib/active_record/relation/delegation.rb +25 -14
- data/lib/active_record/relation/finder_methods.rb +18 -18
- data/lib/active_record/relation/merger.rb +8 -8
- data/lib/active_record/relation/predicate_builder/array_handler.rb +3 -1
- data/lib/active_record/relation/predicate_builder/polymorphic_array_value.rb +1 -1
- data/lib/active_record/relation/predicate_builder/relation_handler.rb +4 -3
- data/lib/active_record/relation/predicate_builder.rb +13 -0
- data/lib/active_record/relation/query_methods.rb +105 -61
- data/lib/active_record/relation/spawn_methods.rb +7 -7
- data/lib/active_record/relation/where_clause.rb +1 -1
- data/lib/active_record/relation.rb +80 -62
- data/lib/active_record/result.rb +66 -4
- data/lib/active_record/sanitization.rb +7 -6
- data/lib/active_record/schema_dumper.rb +5 -0
- data/lib/active_record/schema_migration.rb +2 -1
- data/lib/active_record/scoping/named.rb +5 -2
- data/lib/active_record/statement_cache.rb +14 -14
- data/lib/active_record/store.rb +7 -3
- data/lib/active_record/table_metadata.rb +1 -3
- data/lib/active_record/tasks/database_tasks.rb +69 -60
- data/lib/active_record/tasks/mysql_database_tasks.rb +0 -2
- data/lib/active_record/tasks/postgresql_database_tasks.rb +2 -1
- data/lib/active_record/tasks/sqlite_database_tasks.rb +2 -2
- data/lib/active_record/test_databases.rb +1 -1
- data/lib/active_record/test_fixtures.rb +12 -0
- data/lib/active_record/token_for.rb +1 -1
- data/lib/active_record/transactions.rb +5 -6
- data/lib/active_record/type/serialized.rb +5 -0
- data/lib/active_record/validations/uniqueness.rb +8 -8
- data/lib/active_record.rb +21 -48
- data/lib/arel/collectors/bind.rb +2 -2
- data/lib/arel/collectors/sql_string.rb +1 -1
- data/lib/arel/collectors/substitute_binds.rb +2 -2
- data/lib/arel/nodes/binary.rb +1 -1
- data/lib/arel/nodes/node.rb +1 -1
- data/lib/arel/nodes/sql_literal.rb +1 -1
- data/lib/arel/predications.rb +1 -3
- data/lib/arel/table.rb +3 -7
- metadata +10 -11
- data/lib/active_record/relation/record_fetch_warning.rb +0 -52
|
@@ -87,22 +87,7 @@ db_namespace = namespace :db do
|
|
|
87
87
|
|
|
88
88
|
desc "Migrate the database (options: VERSION=x, VERBOSE=false, SCOPE=blog)."
|
|
89
89
|
task migrate: :load_config do
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
if db_configs.size == 1 && db_configs.first.primary?
|
|
93
|
-
ActiveRecord::Tasks::DatabaseTasks.migrate
|
|
94
|
-
else
|
|
95
|
-
mapped_versions = ActiveRecord::Tasks::DatabaseTasks.db_configs_with_versions
|
|
96
|
-
|
|
97
|
-
mapped_versions.sort.each do |version, db_configs|
|
|
98
|
-
db_configs.each do |db_config|
|
|
99
|
-
ActiveRecord::Tasks::DatabaseTasks.with_temporary_connection(db_config) do
|
|
100
|
-
ActiveRecord::Tasks::DatabaseTasks.migrate(version)
|
|
101
|
-
end
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
end
|
|
105
|
-
|
|
90
|
+
ActiveRecord::Tasks::DatabaseTasks.migrate_all
|
|
106
91
|
db_namespace["_dump"].invoke
|
|
107
92
|
end
|
|
108
93
|
|
|
@@ -175,8 +160,8 @@ db_namespace = namespace :db do
|
|
|
175
160
|
end
|
|
176
161
|
end
|
|
177
162
|
|
|
178
|
-
|
|
179
|
-
task reset: ["db:drop", "db:create", "db:migrate"]
|
|
163
|
+
desc "Resets your database using your migrations for the current environment"
|
|
164
|
+
task reset: ["db:drop", "db:create", "db:schema:dump", "db:migrate"]
|
|
180
165
|
|
|
181
166
|
desc 'Run the "up" for a given migration VERSION.'
|
|
182
167
|
task up: :load_config do
|
|
@@ -462,19 +447,14 @@ db_namespace = namespace :db do
|
|
|
462
447
|
namespace :schema do
|
|
463
448
|
desc "Create a database schema file (either db/schema.rb or db/structure.sql, depending on `ENV['SCHEMA_FORMAT']` or `config.active_record.schema_format`)"
|
|
464
449
|
task dump: :load_config do
|
|
465
|
-
ActiveRecord::Tasks::DatabaseTasks.
|
|
466
|
-
db_config = pool.db_config
|
|
467
|
-
schema_format = ENV.fetch("SCHEMA_FORMAT", ActiveRecord.schema_format).to_sym
|
|
468
|
-
ActiveRecord::Tasks::DatabaseTasks.dump_schema(db_config, schema_format)
|
|
469
|
-
end
|
|
450
|
+
ActiveRecord::Tasks::DatabaseTasks.dump_all
|
|
470
451
|
|
|
471
452
|
db_namespace["schema:dump"].reenable
|
|
472
453
|
end
|
|
473
454
|
|
|
474
455
|
desc "Load a database schema file (either db/schema.rb or db/structure.sql, depending on `ENV['SCHEMA_FORMAT']` or `config.active_record.schema_format`) into the database"
|
|
475
456
|
task load: [:load_config, :check_protected_environments] do
|
|
476
|
-
|
|
477
|
-
ActiveRecord::Tasks::DatabaseTasks.load_schema_current(schema_format, ENV["SCHEMA"])
|
|
457
|
+
ActiveRecord::Tasks::DatabaseTasks.load_schema_current(ENV["SCHEMA_FORMAT"], ENV["SCHEMA"])
|
|
478
458
|
end
|
|
479
459
|
|
|
480
460
|
namespace :dump do
|
|
@@ -483,8 +463,7 @@ db_namespace = namespace :db do
|
|
|
483
463
|
task name => :load_config do
|
|
484
464
|
ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each(name: name) do |pool|
|
|
485
465
|
db_config = pool.db_config
|
|
486
|
-
|
|
487
|
-
ActiveRecord::Tasks::DatabaseTasks.dump_schema(db_config, schema_format)
|
|
466
|
+
ActiveRecord::Tasks::DatabaseTasks.dump_schema(db_config, ENV["SCHEMA_FORMAT"] || db_config.schema_format)
|
|
488
467
|
end
|
|
489
468
|
|
|
490
469
|
db_namespace["schema:dump:#{name}"].reenable
|
|
@@ -494,12 +473,11 @@ db_namespace = namespace :db do
|
|
|
494
473
|
|
|
495
474
|
namespace :load do
|
|
496
475
|
ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
|
|
497
|
-
desc "Load a database schema file (either db/schema.rb or db/structure.sql, depending on
|
|
476
|
+
desc "Load a database schema file (either db/schema.rb or db/structure.sql, depending on configuration) into the #{name} database"
|
|
498
477
|
task name => "db:test:purge:#{name}" do
|
|
499
478
|
ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each(name: name) do |pool|
|
|
500
479
|
db_config = pool.db_config
|
|
501
|
-
|
|
502
|
-
ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, schema_format)
|
|
480
|
+
ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, ENV["SCHEMA_FORMAT"] || db_config.schema_format)
|
|
503
481
|
end
|
|
504
482
|
end
|
|
505
483
|
end
|
|
@@ -543,13 +521,12 @@ db_namespace = namespace :db do
|
|
|
543
521
|
end
|
|
544
522
|
|
|
545
523
|
namespace :test do
|
|
546
|
-
# desc "Recreate the test database from an existent schema file (schema.rb or structure.sql, depending on
|
|
524
|
+
# desc "Recreate the test database from an existent schema file (schema.rb or structure.sql, depending on configuration)"
|
|
547
525
|
task load_schema: %w(db:test:purge) do
|
|
548
526
|
ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each(env: "test") do |pool|
|
|
549
527
|
db_config = pool.db_config
|
|
550
528
|
ActiveRecord::Schema.verbose = false
|
|
551
|
-
|
|
552
|
-
ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, schema_format)
|
|
529
|
+
ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, ENV["SCHEMA_FORMAT"] || db_config.schema_format)
|
|
553
530
|
end
|
|
554
531
|
end
|
|
555
532
|
|
|
@@ -574,8 +551,7 @@ db_namespace = namespace :db do
|
|
|
574
551
|
ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each(env: "test", name: name) do |pool|
|
|
575
552
|
db_config = pool.db_config
|
|
576
553
|
ActiveRecord::Schema.verbose = false
|
|
577
|
-
|
|
578
|
-
ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, schema_format)
|
|
554
|
+
ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, ENV["SCHEMA_FORMAT"] || db_config.schema_format)
|
|
579
555
|
end
|
|
580
556
|
end
|
|
581
557
|
end
|
|
@@ -198,7 +198,7 @@ module ActiveRecord
|
|
|
198
198
|
end
|
|
199
199
|
|
|
200
200
|
def join_scope(table, foreign_table, foreign_klass)
|
|
201
|
-
predicate_builder = predicate_builder(table)
|
|
201
|
+
predicate_builder = klass.predicate_builder.with(TableMetadata.new(klass, table))
|
|
202
202
|
scope_chain_items = join_scopes(table, predicate_builder)
|
|
203
203
|
klass_scope = klass_join_scope(table, predicate_builder)
|
|
204
204
|
|
|
@@ -224,7 +224,7 @@ module ActiveRecord
|
|
|
224
224
|
klass_scope
|
|
225
225
|
end
|
|
226
226
|
|
|
227
|
-
def join_scopes(table, predicate_builder, klass = self.klass, record = nil) # :nodoc:
|
|
227
|
+
def join_scopes(table, predicate_builder = nil, klass = self.klass, record = nil) # :nodoc:
|
|
228
228
|
if scope
|
|
229
229
|
[scope_for(build_scope(table, predicate_builder, klass), record)]
|
|
230
230
|
else
|
|
@@ -232,7 +232,7 @@ module ActiveRecord
|
|
|
232
232
|
end
|
|
233
233
|
end
|
|
234
234
|
|
|
235
|
-
def klass_join_scope(table, predicate_builder) # :nodoc:
|
|
235
|
+
def klass_join_scope(table, predicate_builder = nil) # :nodoc:
|
|
236
236
|
relation = build_scope(table, predicate_builder)
|
|
237
237
|
klass.scope_for_association(relation)
|
|
238
238
|
end
|
|
@@ -333,12 +333,8 @@ module ActiveRecord
|
|
|
333
333
|
collect_join_chain
|
|
334
334
|
end
|
|
335
335
|
|
|
336
|
-
def build_scope(table, predicate_builder =
|
|
337
|
-
Relation.create(
|
|
338
|
-
klass,
|
|
339
|
-
table: table,
|
|
340
|
-
predicate_builder: predicate_builder
|
|
341
|
-
)
|
|
336
|
+
def build_scope(table, predicate_builder = nil, klass = self.klass)
|
|
337
|
+
Relation.create(klass, table:, predicate_builder:)
|
|
342
338
|
end
|
|
343
339
|
|
|
344
340
|
def strict_loading?
|
|
@@ -357,10 +353,6 @@ module ActiveRecord
|
|
|
357
353
|
end
|
|
358
354
|
|
|
359
355
|
private
|
|
360
|
-
def predicate_builder(table)
|
|
361
|
-
PredicateBuilder.new(TableMetadata.new(klass, table))
|
|
362
|
-
end
|
|
363
|
-
|
|
364
356
|
def primary_key(klass)
|
|
365
357
|
klass.primary_key || raise(UnknownPrimaryKey.new(klass))
|
|
366
358
|
end
|
|
@@ -531,9 +523,9 @@ module ActiveRecord
|
|
|
531
523
|
@association_foreign_key = nil
|
|
532
524
|
@association_primary_key = nil
|
|
533
525
|
if options[:query_constraints]
|
|
534
|
-
|
|
535
|
-
Setting `query_constraints:` option on `#{active_record}.#{macro} :#{name}` is
|
|
536
|
-
To
|
|
526
|
+
raise ConfigurationError, <<~MSG.squish
|
|
527
|
+
Setting `query_constraints:` option on `#{active_record}.#{macro} :#{name}` is not allowed.
|
|
528
|
+
To get the same behavior, use the `foreign_key` option instead.
|
|
537
529
|
MSG
|
|
538
530
|
end
|
|
539
531
|
|
|
@@ -562,12 +554,12 @@ module ActiveRecord
|
|
|
562
554
|
def foreign_key(infer_from_inverse_of: true)
|
|
563
555
|
@foreign_key ||= if options[:foreign_key]
|
|
564
556
|
if options[:foreign_key].is_a?(Array)
|
|
565
|
-
options[:foreign_key].map { |fk| fk.to_s.freeze }.freeze
|
|
557
|
+
options[:foreign_key].map { |fk| -fk.to_s.freeze }.freeze
|
|
566
558
|
else
|
|
567
559
|
options[:foreign_key].to_s.freeze
|
|
568
560
|
end
|
|
569
561
|
elsif options[:query_constraints]
|
|
570
|
-
options[:query_constraints].map { |fk| fk.to_s.freeze }.freeze
|
|
562
|
+
options[:query_constraints].map { |fk| -fk.to_s.freeze }.freeze
|
|
571
563
|
else
|
|
572
564
|
derived_fk = derive_foreign_key(infer_from_inverse_of: infer_from_inverse_of)
|
|
573
565
|
|
|
@@ -575,7 +567,12 @@ module ActiveRecord
|
|
|
575
567
|
derived_fk = derive_fk_query_constraints(derived_fk)
|
|
576
568
|
end
|
|
577
569
|
|
|
578
|
-
derived_fk
|
|
570
|
+
if derived_fk.is_a?(Array)
|
|
571
|
+
derived_fk.map! { |fk| -fk.freeze }
|
|
572
|
+
derived_fk.freeze
|
|
573
|
+
else
|
|
574
|
+
-derived_fk.freeze
|
|
575
|
+
end
|
|
579
576
|
end
|
|
580
577
|
end
|
|
581
578
|
|
|
@@ -1065,7 +1062,7 @@ module ActiveRecord
|
|
|
1065
1062
|
source_reflection.scopes + super
|
|
1066
1063
|
end
|
|
1067
1064
|
|
|
1068
|
-
def join_scopes(table, predicate_builder, klass = self.klass, record = nil) # :nodoc:
|
|
1065
|
+
def join_scopes(table, predicate_builder = nil, klass = self.klass, record = nil) # :nodoc:
|
|
1069
1066
|
source_reflection.join_scopes(table, predicate_builder, klass, record) + super
|
|
1070
1067
|
end
|
|
1071
1068
|
|
|
@@ -1238,7 +1235,7 @@ module ActiveRecord
|
|
|
1238
1235
|
@previous_reflection = previous_reflection
|
|
1239
1236
|
end
|
|
1240
1237
|
|
|
1241
|
-
def join_scopes(table, predicate_builder, klass = self.klass, record = nil) # :nodoc:
|
|
1238
|
+
def join_scopes(table, predicate_builder = nil, klass = self.klass, record = nil) # :nodoc:
|
|
1242
1239
|
scopes = super
|
|
1243
1240
|
unless @previous_reflection.through_reflection?
|
|
1244
1241
|
scopes += @previous_reflection.join_scopes(table, predicate_builder, klass, record)
|
|
@@ -5,11 +5,12 @@ module ActiveRecord
|
|
|
5
5
|
class BatchEnumerator
|
|
6
6
|
include Enumerable
|
|
7
7
|
|
|
8
|
-
def initialize(of: 1000, start: nil, finish: nil, relation:, order: :asc, use_ranges: nil) # :nodoc:
|
|
8
|
+
def initialize(of: 1000, start: nil, finish: nil, relation:, cursor:, order: :asc, use_ranges: nil) # :nodoc:
|
|
9
9
|
@of = of
|
|
10
10
|
@relation = relation
|
|
11
11
|
@start = start
|
|
12
12
|
@finish = finish
|
|
13
|
+
@cursor = cursor
|
|
13
14
|
@order = order
|
|
14
15
|
@use_ranges = use_ranges
|
|
15
16
|
end
|
|
@@ -52,7 +53,7 @@ module ActiveRecord
|
|
|
52
53
|
def each_record(&block)
|
|
53
54
|
return to_enum(:each_record) unless block_given?
|
|
54
55
|
|
|
55
|
-
@relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, order: @order).each do |relation|
|
|
56
|
+
@relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, cursor: @cursor, order: @order).each do |relation|
|
|
56
57
|
relation.records.each(&block)
|
|
57
58
|
end
|
|
58
59
|
end
|
|
@@ -105,7 +106,7 @@ module ActiveRecord
|
|
|
105
106
|
# relation.update_all(awesome: true)
|
|
106
107
|
# end
|
|
107
108
|
def each(&block)
|
|
108
|
-
enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, order: @order, use_ranges: @use_ranges)
|
|
109
|
+
enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, cursor: @cursor, order: @order, use_ranges: @use_ranges)
|
|
109
110
|
return enum.each(&block) if block_given?
|
|
110
111
|
enum
|
|
111
112
|
end
|
|
@@ -5,7 +5,7 @@ require "active_record/relation/batches/batch_enumerator"
|
|
|
5
5
|
module ActiveRecord
|
|
6
6
|
# = Active Record \Batches
|
|
7
7
|
module Batches
|
|
8
|
-
ORDER_IGNORE_MESSAGE = "Scoped order is ignored,
|
|
8
|
+
ORDER_IGNORE_MESSAGE = "Scoped order is ignored, use :cursor with :order to configure custom order."
|
|
9
9
|
DEFAULT_ORDER = :asc
|
|
10
10
|
|
|
11
11
|
# Looping through a collection of records from the database
|
|
@@ -35,11 +35,13 @@ module ActiveRecord
|
|
|
35
35
|
#
|
|
36
36
|
# ==== Options
|
|
37
37
|
# * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
|
|
38
|
-
# * <tt>:start</tt> - Specifies the
|
|
39
|
-
# * <tt>:finish</tt> - Specifies the
|
|
38
|
+
# * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
|
|
39
|
+
# * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
|
|
40
40
|
# * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
|
|
41
41
|
# an order is present in the relation.
|
|
42
|
-
# * <tt>:
|
|
42
|
+
# * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
|
|
43
|
+
# of column names). Defaults to primary key.
|
|
44
|
+
# * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
|
|
43
45
|
# of :asc or :desc). Defaults to +:asc+.
|
|
44
46
|
#
|
|
45
47
|
# class Order < ActiveRecord::Base
|
|
@@ -71,20 +73,25 @@ module ActiveRecord
|
|
|
71
73
|
#
|
|
72
74
|
# NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
|
|
73
75
|
# ascending on the primary key ("id ASC").
|
|
74
|
-
# This also means that this method only works when the
|
|
76
|
+
# This also means that this method only works when the cursor column is
|
|
75
77
|
# orderable (e.g. an integer or string).
|
|
76
78
|
#
|
|
79
|
+
# NOTE: When using custom columns for batching, they should include at least one unique column
|
|
80
|
+
# (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
|
|
81
|
+
# all columns should be static (unchangeable after it was set).
|
|
82
|
+
#
|
|
77
83
|
# NOTE: By its nature, batch processing is subject to race conditions if
|
|
78
84
|
# other processes are modifying the database.
|
|
79
|
-
def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER, &block)
|
|
85
|
+
def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER, &block)
|
|
80
86
|
if block_given?
|
|
81
|
-
find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do |records|
|
|
87
|
+
find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do |records|
|
|
82
88
|
records.each(&block)
|
|
83
89
|
end
|
|
84
90
|
else
|
|
85
|
-
enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
|
|
91
|
+
enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do
|
|
86
92
|
relation = self
|
|
87
|
-
|
|
93
|
+
cursor = Array(cursor)
|
|
94
|
+
apply_limits(relation, cursor, start, finish, build_batch_orders(cursor, order)).size
|
|
88
95
|
end
|
|
89
96
|
end
|
|
90
97
|
end
|
|
@@ -109,11 +116,13 @@ module ActiveRecord
|
|
|
109
116
|
#
|
|
110
117
|
# ==== Options
|
|
111
118
|
# * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
|
|
112
|
-
# * <tt>:start</tt> - Specifies the
|
|
113
|
-
# * <tt>:finish</tt> - Specifies the
|
|
119
|
+
# * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
|
|
120
|
+
# * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
|
|
114
121
|
# * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
|
|
115
122
|
# an order is present in the relation.
|
|
116
|
-
# * <tt>:
|
|
123
|
+
# * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
|
|
124
|
+
# of column names). Defaults to primary key.
|
|
125
|
+
# * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
|
|
117
126
|
# of :asc or :desc). Defaults to +:asc+.
|
|
118
127
|
#
|
|
119
128
|
# class Order < ActiveRecord::Base
|
|
@@ -140,21 +149,26 @@ module ActiveRecord
|
|
|
140
149
|
#
|
|
141
150
|
# NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
|
|
142
151
|
# ascending on the primary key ("id ASC").
|
|
143
|
-
# This also means that this method only works when the
|
|
152
|
+
# This also means that this method only works when the cursor column is
|
|
144
153
|
# orderable (e.g. an integer or string).
|
|
145
154
|
#
|
|
155
|
+
# NOTE: When using custom columns for batching, they should include at least one unique column
|
|
156
|
+
# (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
|
|
157
|
+
# all columns should be static (unchangeable after it was set).
|
|
158
|
+
#
|
|
146
159
|
# NOTE: By its nature, batch processing is subject to race conditions if
|
|
147
160
|
# other processes are modifying the database.
|
|
148
|
-
def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER)
|
|
161
|
+
def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER)
|
|
149
162
|
relation = self
|
|
150
163
|
unless block_given?
|
|
151
|
-
return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
|
|
152
|
-
|
|
164
|
+
return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do
|
|
165
|
+
cursor = Array(cursor)
|
|
166
|
+
total = apply_limits(relation, cursor, start, finish, build_batch_orders(cursor, order)).size
|
|
153
167
|
(total - 1).div(batch_size) + 1
|
|
154
168
|
end
|
|
155
169
|
end
|
|
156
170
|
|
|
157
|
-
in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, order: order) do |batch|
|
|
171
|
+
in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do |batch|
|
|
158
172
|
yield batch.to_a
|
|
159
173
|
end
|
|
160
174
|
end
|
|
@@ -183,11 +197,13 @@ module ActiveRecord
|
|
|
183
197
|
# ==== Options
|
|
184
198
|
# * <tt>:of</tt> - Specifies the size of the batch. Defaults to 1000.
|
|
185
199
|
# * <tt>:load</tt> - Specifies if the relation should be loaded. Defaults to false.
|
|
186
|
-
# * <tt>:start</tt> - Specifies the
|
|
187
|
-
# * <tt>:finish</tt> - Specifies the
|
|
200
|
+
# * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
|
|
201
|
+
# * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
|
|
188
202
|
# * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
|
|
189
203
|
# an order is present in the relation.
|
|
190
|
-
# * <tt>:
|
|
204
|
+
# * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
|
|
205
|
+
# of column names). Defaults to primary key.
|
|
206
|
+
# * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
|
|
191
207
|
# of :asc or :desc). Defaults to +:asc+.
|
|
192
208
|
#
|
|
193
209
|
# class Order < ActiveRecord::Base
|
|
@@ -231,22 +247,25 @@ module ActiveRecord
|
|
|
231
247
|
#
|
|
232
248
|
# NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
|
|
233
249
|
# ascending on the primary key ("id ASC").
|
|
234
|
-
# This also means that this method only works when the
|
|
250
|
+
# This also means that this method only works when the cursor column is
|
|
235
251
|
# orderable (e.g. an integer or string).
|
|
236
252
|
#
|
|
253
|
+
# NOTE: When using custom columns for batching, they should include at least one unique column
|
|
254
|
+
# (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
|
|
255
|
+
# all columns should be static (unchangeable after it was set).
|
|
256
|
+
#
|
|
237
257
|
# NOTE: By its nature, batch processing is subject to race conditions if
|
|
238
258
|
# other processes are modifying the database.
|
|
239
|
-
def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, order: DEFAULT_ORDER, use_ranges: nil, &block)
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
end
|
|
259
|
+
def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER, use_ranges: nil, &block)
|
|
260
|
+
cursor = Array(cursor).map(&:to_s)
|
|
261
|
+
ensure_valid_options_for_batching!(cursor, start, finish, order)
|
|
243
262
|
|
|
244
263
|
if arel.orders.present?
|
|
245
264
|
act_on_ignored_order(error_on_ignore)
|
|
246
265
|
end
|
|
247
266
|
|
|
248
267
|
unless block
|
|
249
|
-
return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, order: order, use_ranges: use_ranges)
|
|
268
|
+
return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, cursor: cursor, order: order, use_ranges: use_ranges)
|
|
250
269
|
end
|
|
251
270
|
|
|
252
271
|
batch_limit = of
|
|
@@ -261,6 +280,7 @@ module ActiveRecord
|
|
|
261
280
|
relation: self,
|
|
262
281
|
start: start,
|
|
263
282
|
finish: finish,
|
|
283
|
+
cursor: cursor,
|
|
264
284
|
order: order,
|
|
265
285
|
batch_limit: batch_limit,
|
|
266
286
|
&block
|
|
@@ -271,6 +291,7 @@ module ActiveRecord
|
|
|
271
291
|
start: start,
|
|
272
292
|
finish: finish,
|
|
273
293
|
load: load,
|
|
294
|
+
cursor: cursor,
|
|
274
295
|
order: order,
|
|
275
296
|
use_ranges: use_ranges,
|
|
276
297
|
remaining: remaining,
|
|
@@ -281,28 +302,51 @@ module ActiveRecord
|
|
|
281
302
|
end
|
|
282
303
|
|
|
283
304
|
private
|
|
284
|
-
def
|
|
285
|
-
|
|
286
|
-
|
|
305
|
+
def ensure_valid_options_for_batching!(cursor, start, finish, order)
|
|
306
|
+
if start && Array(start).size != cursor.size
|
|
307
|
+
raise ArgumentError, ":start must contain one value per cursor column"
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
if finish && Array(finish).size != cursor.size
|
|
311
|
+
raise ArgumentError, ":finish must contain one value per cursor column"
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
if (Array(primary_key) - cursor).any?
|
|
315
|
+
indexes = model.schema_cache.indexes(table_name)
|
|
316
|
+
unique_index = indexes.find { |index| index.unique && index.where.nil? && (Array(index.columns) - cursor).empty? }
|
|
317
|
+
|
|
318
|
+
unless unique_index
|
|
319
|
+
raise ArgumentError, ":cursor must include a primary key or other unique column(s)"
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
if (Array(order) - [:asc, :desc]).any?
|
|
324
|
+
raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
def apply_limits(relation, cursor, start, finish, batch_orders)
|
|
329
|
+
relation = apply_start_limit(relation, cursor, start, batch_orders) if start
|
|
330
|
+
relation = apply_finish_limit(relation, cursor, finish, batch_orders) if finish
|
|
287
331
|
relation
|
|
288
332
|
end
|
|
289
333
|
|
|
290
|
-
def apply_start_limit(relation, start, batch_orders)
|
|
334
|
+
def apply_start_limit(relation, cursor, start, batch_orders)
|
|
291
335
|
operators = batch_orders.map do |_column, order|
|
|
292
336
|
order == :desc ? :lteq : :gteq
|
|
293
337
|
end
|
|
294
|
-
batch_condition(relation,
|
|
338
|
+
batch_condition(relation, cursor, start, operators)
|
|
295
339
|
end
|
|
296
340
|
|
|
297
|
-
def apply_finish_limit(relation, finish, batch_orders)
|
|
341
|
+
def apply_finish_limit(relation, cursor, finish, batch_orders)
|
|
298
342
|
operators = batch_orders.map do |_column, order|
|
|
299
343
|
order == :desc ? :gteq : :lteq
|
|
300
344
|
end
|
|
301
|
-
batch_condition(relation,
|
|
345
|
+
batch_condition(relation, cursor, finish, operators)
|
|
302
346
|
end
|
|
303
347
|
|
|
304
|
-
def batch_condition(relation,
|
|
305
|
-
cursor_positions =
|
|
348
|
+
def batch_condition(relation, cursor, values, operators)
|
|
349
|
+
cursor_positions = cursor.zip(Array(values), operators)
|
|
306
350
|
|
|
307
351
|
first_clause_column, first_clause_value, operator = cursor_positions.pop
|
|
308
352
|
where_clause = predicate_builder[first_clause_column, first_clause_value, operator]
|
|
@@ -316,9 +360,9 @@ module ActiveRecord
|
|
|
316
360
|
relation.where(where_clause)
|
|
317
361
|
end
|
|
318
362
|
|
|
319
|
-
def build_batch_orders(order)
|
|
320
|
-
|
|
321
|
-
[column,
|
|
363
|
+
def build_batch_orders(cursor, order)
|
|
364
|
+
cursor.zip(Array(order)).map do |column, order_|
|
|
365
|
+
[column, order_ || DEFAULT_ORDER]
|
|
322
366
|
end
|
|
323
367
|
end
|
|
324
368
|
|
|
@@ -327,34 +371,28 @@ module ActiveRecord
|
|
|
327
371
|
|
|
328
372
|
if raise_error
|
|
329
373
|
raise ArgumentError.new(ORDER_IGNORE_MESSAGE)
|
|
330
|
-
elsif logger
|
|
331
|
-
logger.warn(ORDER_IGNORE_MESSAGE)
|
|
374
|
+
elsif model.logger
|
|
375
|
+
model.logger.warn(ORDER_IGNORE_MESSAGE)
|
|
332
376
|
end
|
|
333
377
|
end
|
|
334
378
|
|
|
335
|
-
def
|
|
336
|
-
Array(primary_key).zip(Array(order))
|
|
337
|
-
end
|
|
338
|
-
|
|
339
|
-
def batch_on_loaded_relation(relation:, start:, finish:, order:, batch_limit:)
|
|
379
|
+
def batch_on_loaded_relation(relation:, start:, finish:, cursor:, order:, batch_limit:)
|
|
340
380
|
records = relation.to_a
|
|
381
|
+
order = build_batch_orders(cursor, order).map(&:second)
|
|
341
382
|
|
|
342
383
|
if start || finish
|
|
343
384
|
records = records.filter do |record|
|
|
344
|
-
|
|
385
|
+
values = record_cursor_values(record, cursor)
|
|
345
386
|
|
|
346
|
-
|
|
347
|
-
(
|
|
348
|
-
else
|
|
349
|
-
(start.nil? || id <= start) && (finish.nil? || id >= finish)
|
|
350
|
-
end
|
|
387
|
+
(start.nil? || compare_values_for_order(values, Array(start), order) >= 0) &&
|
|
388
|
+
(finish.nil? || compare_values_for_order(values, Array(finish), order) <= 0)
|
|
351
389
|
end
|
|
352
390
|
end
|
|
353
391
|
|
|
354
|
-
records.
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
392
|
+
records.sort! do |record1, record2|
|
|
393
|
+
values1 = record_cursor_values(record1, cursor)
|
|
394
|
+
values2 = record_cursor_values(record2, cursor)
|
|
395
|
+
compare_values_for_order(values1, values2, order)
|
|
358
396
|
end
|
|
359
397
|
|
|
360
398
|
records.each_slice(batch_limit) do |subrecords|
|
|
@@ -367,44 +405,65 @@ module ActiveRecord
|
|
|
367
405
|
nil
|
|
368
406
|
end
|
|
369
407
|
|
|
370
|
-
def
|
|
371
|
-
|
|
408
|
+
def record_cursor_values(record, cursor)
|
|
409
|
+
record.attributes.slice(*cursor).values
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# This is a custom implementation of `<=>` operator,
|
|
413
|
+
# which also takes into account how the collection will be ordered.
|
|
414
|
+
def compare_values_for_order(values1, values2, order)
|
|
415
|
+
values1.each_with_index do |element1, index|
|
|
416
|
+
element2 = values2[index]
|
|
417
|
+
direction = order[index]
|
|
418
|
+
comparison = element1 <=> element2
|
|
419
|
+
comparison = -comparison if direction == :desc
|
|
420
|
+
return comparison if comparison != 0
|
|
421
|
+
end
|
|
422
|
+
|
|
423
|
+
0
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
def batch_on_unloaded_relation(relation:, start:, finish:, load:, cursor:, order:, use_ranges:, remaining:, batch_limit:)
|
|
427
|
+
batch_orders = build_batch_orders(cursor, order)
|
|
372
428
|
relation = relation.reorder(batch_orders.to_h).limit(batch_limit)
|
|
373
|
-
relation = apply_limits(relation, start, finish, batch_orders)
|
|
429
|
+
relation = apply_limits(relation, cursor, start, finish, batch_orders)
|
|
374
430
|
relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
|
|
375
431
|
batch_relation = relation
|
|
376
|
-
empty_scope = to_sql ==
|
|
432
|
+
empty_scope = to_sql == model.unscoped.all.to_sql
|
|
377
433
|
|
|
378
434
|
loop do
|
|
379
435
|
if load
|
|
380
436
|
records = batch_relation.records
|
|
381
|
-
|
|
382
|
-
yielded_relation = where(
|
|
437
|
+
values = records.pluck(*cursor)
|
|
438
|
+
yielded_relation = where(cursor => values)
|
|
383
439
|
yielded_relation.load_records(records)
|
|
384
440
|
elsif (empty_scope && use_ranges != false) || use_ranges
|
|
385
|
-
|
|
386
|
-
|
|
441
|
+
values = batch_relation.pluck(*cursor)
|
|
442
|
+
|
|
443
|
+
finish = values.last
|
|
387
444
|
if finish
|
|
388
|
-
yielded_relation = apply_finish_limit(batch_relation, finish, batch_orders)
|
|
445
|
+
yielded_relation = apply_finish_limit(batch_relation, cursor, finish, batch_orders)
|
|
389
446
|
yielded_relation = yielded_relation.except(:limit, :order)
|
|
390
447
|
yielded_relation.skip_query_cache!(false)
|
|
391
448
|
end
|
|
392
449
|
else
|
|
393
|
-
|
|
394
|
-
yielded_relation = where(
|
|
450
|
+
values = batch_relation.pluck(*cursor)
|
|
451
|
+
yielded_relation = where(cursor => values)
|
|
395
452
|
end
|
|
396
453
|
|
|
397
|
-
break if
|
|
454
|
+
break if values.empty?
|
|
398
455
|
|
|
399
|
-
|
|
400
|
-
|
|
456
|
+
if values.flatten.any?(nil)
|
|
457
|
+
raise ArgumentError, "Not all of the batch cursor columns were included in the custom select clause "\
|
|
458
|
+
"or some columns contain nil."
|
|
459
|
+
end
|
|
401
460
|
|
|
402
461
|
yield yielded_relation
|
|
403
462
|
|
|
404
|
-
break if
|
|
463
|
+
break if values.length < batch_limit
|
|
405
464
|
|
|
406
465
|
if limit_value
|
|
407
|
-
remaining -=
|
|
466
|
+
remaining -= values.length
|
|
408
467
|
|
|
409
468
|
if remaining == 0
|
|
410
469
|
# Saves a useless iteration when the limit is a multiple of the
|
|
@@ -422,7 +481,8 @@ module ActiveRecord
|
|
|
422
481
|
end
|
|
423
482
|
operators << (last_order == :desc ? :lt : :gt)
|
|
424
483
|
|
|
425
|
-
|
|
484
|
+
cursor_value = values.last
|
|
485
|
+
batch_relation = batch_condition(relation, cursor, cursor_value, operators)
|
|
426
486
|
end
|
|
427
487
|
|
|
428
488
|
nil
|