activerecord 7.2.3.1 → 8.0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +475 -946
  3. data/README.rdoc +1 -1
  4. data/lib/active_record/association_relation.rb +1 -0
  5. data/lib/active_record/associations/association.rb +34 -10
  6. data/lib/active_record/associations/builder/association.rb +7 -6
  7. data/lib/active_record/associations/collection_association.rb +1 -1
  8. data/lib/active_record/associations/disable_joins_association_scope.rb +1 -1
  9. data/lib/active_record/associations/has_many_through_association.rb +3 -2
  10. data/lib/active_record/associations/join_dependency.rb +2 -2
  11. data/lib/active_record/associations/preloader/association.rb +2 -2
  12. data/lib/active_record/associations/preloader/batch.rb +7 -1
  13. data/lib/active_record/associations/singular_association.rb +8 -3
  14. data/lib/active_record/associations.rb +34 -4
  15. data/lib/active_record/asynchronous_queries_tracker.rb +28 -24
  16. data/lib/active_record/attribute_methods/primary_key.rb +4 -8
  17. data/lib/active_record/attribute_methods/query.rb +34 -0
  18. data/lib/active_record/attribute_methods/time_zone_conversion.rb +2 -12
  19. data/lib/active_record/autosave_association.rb +69 -27
  20. data/lib/active_record/connection_adapters/abstract/connection_handler.rb +34 -25
  21. data/lib/active_record/connection_adapters/abstract/connection_pool/queue.rb +0 -1
  22. data/lib/active_record/connection_adapters/abstract/connection_pool/reaper.rb +0 -1
  23. data/lib/active_record/connection_adapters/abstract/connection_pool.rb +8 -17
  24. data/lib/active_record/connection_adapters/abstract/database_statements.rb +91 -44
  25. data/lib/active_record/connection_adapters/abstract/query_cache.rb +8 -2
  26. data/lib/active_record/connection_adapters/abstract/quoting.rb +1 -1
  27. data/lib/active_record/connection_adapters/abstract/schema_creation.rb +4 -5
  28. data/lib/active_record/connection_adapters/abstract/schema_definitions.rb +7 -2
  29. data/lib/active_record/connection_adapters/abstract/schema_statements.rb +37 -10
  30. data/lib/active_record/connection_adapters/abstract/transaction.rb +15 -5
  31. data/lib/active_record/connection_adapters/abstract_adapter.rb +67 -74
  32. data/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +21 -40
  33. data/lib/active_record/connection_adapters/mysql/quoting.rb +0 -8
  34. data/lib/active_record/connection_adapters/mysql/schema_definitions.rb +2 -8
  35. data/lib/active_record/connection_adapters/mysql/schema_statements.rb +50 -45
  36. data/lib/active_record/connection_adapters/mysql2/database_statements.rb +84 -94
  37. data/lib/active_record/connection_adapters/mysql2_adapter.rb +1 -8
  38. data/lib/active_record/connection_adapters/pool_config.rb +7 -7
  39. data/lib/active_record/connection_adapters/postgresql/column.rb +4 -2
  40. data/lib/active_record/connection_adapters/postgresql/database_statements.rb +72 -43
  41. data/lib/active_record/connection_adapters/postgresql/oid/array.rb +1 -1
  42. data/lib/active_record/connection_adapters/postgresql/oid/point.rb +10 -0
  43. data/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +2 -4
  44. data/lib/active_record/connection_adapters/postgresql/schema_creation.rb +1 -11
  45. data/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +6 -12
  46. data/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +2 -1
  47. data/lib/active_record/connection_adapters/postgresql/schema_statements.rb +59 -16
  48. data/lib/active_record/connection_adapters/postgresql_adapter.rb +53 -96
  49. data/lib/active_record/connection_adapters/schema_cache.rb +1 -3
  50. data/lib/active_record/connection_adapters/sqlite3/column.rb +8 -2
  51. data/lib/active_record/connection_adapters/sqlite3/database_statements.rb +80 -100
  52. data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +0 -6
  53. data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb +27 -2
  54. data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb +9 -1
  55. data/lib/active_record/connection_adapters/sqlite3_adapter.rb +53 -12
  56. data/lib/active_record/connection_adapters/statement_pool.rb +4 -2
  57. data/lib/active_record/connection_adapters/trilogy/database_statements.rb +37 -67
  58. data/lib/active_record/connection_adapters/trilogy_adapter.rb +0 -17
  59. data/lib/active_record/connection_adapters.rb +0 -56
  60. data/lib/active_record/connection_handling.rb +23 -1
  61. data/lib/active_record/core.rb +29 -14
  62. data/lib/active_record/database_configurations/database_config.rb +4 -0
  63. data/lib/active_record/database_configurations/hash_config.rb +16 -2
  64. data/lib/active_record/encryption/config.rb +3 -1
  65. data/lib/active_record/encryption/encryptable_record.rb +4 -4
  66. data/lib/active_record/encryption/encrypted_attribute_type.rb +10 -1
  67. data/lib/active_record/encryption/encryptor.rb +16 -8
  68. data/lib/active_record/encryption/extended_deterministic_queries.rb +4 -2
  69. data/lib/active_record/encryption/scheme.rb +8 -1
  70. data/lib/active_record/enum.rb +9 -22
  71. data/lib/active_record/errors.rb +13 -5
  72. data/lib/active_record/fixtures.rb +0 -2
  73. data/lib/active_record/future_result.rb +15 -9
  74. data/lib/active_record/gem_version.rb +3 -3
  75. data/lib/active_record/insert_all.rb +3 -3
  76. data/lib/active_record/locking/optimistic.rb +1 -1
  77. data/lib/active_record/log_subscriber.rb +5 -11
  78. data/lib/active_record/migration/command_recorder.rb +32 -12
  79. data/lib/active_record/migration/compatibility.rb +5 -2
  80. data/lib/active_record/migration.rb +38 -42
  81. data/lib/active_record/model_schema.rb +3 -4
  82. data/lib/active_record/nested_attributes.rb +4 -6
  83. data/lib/active_record/persistence.rb +128 -130
  84. data/lib/active_record/query_logs.rb +102 -50
  85. data/lib/active_record/query_logs_formatter.rb +17 -28
  86. data/lib/active_record/querying.rb +8 -8
  87. data/lib/active_record/railtie.rb +2 -26
  88. data/lib/active_record/railties/databases.rake +11 -35
  89. data/lib/active_record/reflection.rb +18 -21
  90. data/lib/active_record/relation/batches/batch_enumerator.rb +4 -3
  91. data/lib/active_record/relation/batches.rb +132 -72
  92. data/lib/active_record/relation/calculations.rb +40 -39
  93. data/lib/active_record/relation/delegation.rb +25 -14
  94. data/lib/active_record/relation/finder_methods.rb +18 -18
  95. data/lib/active_record/relation/merger.rb +8 -8
  96. data/lib/active_record/relation/predicate_builder/array_handler.rb +3 -1
  97. data/lib/active_record/relation/predicate_builder/polymorphic_array_value.rb +1 -1
  98. data/lib/active_record/relation/predicate_builder/relation_handler.rb +4 -3
  99. data/lib/active_record/relation/predicate_builder.rb +13 -0
  100. data/lib/active_record/relation/query_methods.rb +105 -61
  101. data/lib/active_record/relation/spawn_methods.rb +7 -7
  102. data/lib/active_record/relation/where_clause.rb +1 -1
  103. data/lib/active_record/relation.rb +80 -62
  104. data/lib/active_record/result.rb +66 -4
  105. data/lib/active_record/sanitization.rb +7 -6
  106. data/lib/active_record/schema_dumper.rb +5 -0
  107. data/lib/active_record/schema_migration.rb +2 -1
  108. data/lib/active_record/scoping/named.rb +5 -2
  109. data/lib/active_record/statement_cache.rb +14 -14
  110. data/lib/active_record/store.rb +7 -3
  111. data/lib/active_record/table_metadata.rb +1 -3
  112. data/lib/active_record/tasks/database_tasks.rb +69 -60
  113. data/lib/active_record/tasks/mysql_database_tasks.rb +0 -2
  114. data/lib/active_record/tasks/postgresql_database_tasks.rb +2 -1
  115. data/lib/active_record/tasks/sqlite_database_tasks.rb +2 -2
  116. data/lib/active_record/test_databases.rb +1 -1
  117. data/lib/active_record/test_fixtures.rb +12 -0
  118. data/lib/active_record/token_for.rb +1 -1
  119. data/lib/active_record/transactions.rb +5 -6
  120. data/lib/active_record/type/serialized.rb +5 -0
  121. data/lib/active_record/validations/uniqueness.rb +8 -8
  122. data/lib/active_record.rb +21 -48
  123. data/lib/arel/collectors/bind.rb +2 -2
  124. data/lib/arel/collectors/sql_string.rb +1 -1
  125. data/lib/arel/collectors/substitute_binds.rb +2 -2
  126. data/lib/arel/nodes/binary.rb +1 -1
  127. data/lib/arel/nodes/node.rb +1 -1
  128. data/lib/arel/nodes/sql_literal.rb +1 -1
  129. data/lib/arel/predications.rb +1 -3
  130. data/lib/arel/table.rb +3 -7
  131. metadata +10 -11
  132. data/lib/active_record/relation/record_fetch_warning.rb +0 -52
@@ -87,22 +87,7 @@ db_namespace = namespace :db do
87
87
 
88
88
  desc "Migrate the database (options: VERSION=x, VERBOSE=false, SCOPE=blog)."
89
89
  task migrate: :load_config do
90
- db_configs = ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env)
91
-
92
- if db_configs.size == 1 && db_configs.first.primary?
93
- ActiveRecord::Tasks::DatabaseTasks.migrate
94
- else
95
- mapped_versions = ActiveRecord::Tasks::DatabaseTasks.db_configs_with_versions
96
-
97
- mapped_versions.sort.each do |version, db_configs|
98
- db_configs.each do |db_config|
99
- ActiveRecord::Tasks::DatabaseTasks.with_temporary_connection(db_config) do
100
- ActiveRecord::Tasks::DatabaseTasks.migrate(version)
101
- end
102
- end
103
- end
104
- end
105
-
90
+ ActiveRecord::Tasks::DatabaseTasks.migrate_all
106
91
  db_namespace["_dump"].invoke
107
92
  end
108
93
 
@@ -175,8 +160,8 @@ db_namespace = namespace :db do
175
160
  end
176
161
  end
177
162
 
178
- # desc 'Resets your database using your migrations for the current environment'
179
- task reset: ["db:drop", "db:create", "db:migrate"]
163
+ desc "Resets your database using your migrations for the current environment"
164
+ task reset: ["db:drop", "db:create", "db:schema:dump", "db:migrate"]
180
165
 
181
166
  desc 'Run the "up" for a given migration VERSION.'
182
167
  task up: :load_config do
@@ -462,19 +447,14 @@ db_namespace = namespace :db do
462
447
  namespace :schema do
463
448
  desc "Create a database schema file (either db/schema.rb or db/structure.sql, depending on `ENV['SCHEMA_FORMAT']` or `config.active_record.schema_format`)"
464
449
  task dump: :load_config do
465
- ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each do |pool|
466
- db_config = pool.db_config
467
- schema_format = ENV.fetch("SCHEMA_FORMAT", ActiveRecord.schema_format).to_sym
468
- ActiveRecord::Tasks::DatabaseTasks.dump_schema(db_config, schema_format)
469
- end
450
+ ActiveRecord::Tasks::DatabaseTasks.dump_all
470
451
 
471
452
  db_namespace["schema:dump"].reenable
472
453
  end
473
454
 
474
455
  desc "Load a database schema file (either db/schema.rb or db/structure.sql, depending on `ENV['SCHEMA_FORMAT']` or `config.active_record.schema_format`) into the database"
475
456
  task load: [:load_config, :check_protected_environments] do
476
- schema_format = ENV.fetch("SCHEMA_FORMAT", ActiveRecord.schema_format).to_sym
477
- ActiveRecord::Tasks::DatabaseTasks.load_schema_current(schema_format, ENV["SCHEMA"])
457
+ ActiveRecord::Tasks::DatabaseTasks.load_schema_current(ENV["SCHEMA_FORMAT"], ENV["SCHEMA"])
478
458
  end
479
459
 
480
460
  namespace :dump do
@@ -483,8 +463,7 @@ db_namespace = namespace :db do
483
463
  task name => :load_config do
484
464
  ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each(name: name) do |pool|
485
465
  db_config = pool.db_config
486
- schema_format = ENV.fetch("SCHEMA_FORMAT", ActiveRecord.schema_format).to_sym
487
- ActiveRecord::Tasks::DatabaseTasks.dump_schema(db_config, schema_format)
466
+ ActiveRecord::Tasks::DatabaseTasks.dump_schema(db_config, ENV["SCHEMA_FORMAT"] || db_config.schema_format)
488
467
  end
489
468
 
490
469
  db_namespace["schema:dump:#{name}"].reenable
@@ -494,12 +473,11 @@ db_namespace = namespace :db do
494
473
 
495
474
  namespace :load do
496
475
  ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
497
- desc "Load a database schema file (either db/schema.rb or db/structure.sql, depending on `ENV['SCHEMA_FORMAT']` or `config.active_record.schema_format`) into the #{name} database"
476
+ desc "Load a database schema file (either db/schema.rb or db/structure.sql, depending on configuration) into the #{name} database"
498
477
  task name => "db:test:purge:#{name}" do
499
478
  ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each(name: name) do |pool|
500
479
  db_config = pool.db_config
501
- schema_format = ENV.fetch("SCHEMA_FORMAT", ActiveRecord.schema_format).to_sym
502
- ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, schema_format)
480
+ ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, ENV["SCHEMA_FORMAT"] || db_config.schema_format)
503
481
  end
504
482
  end
505
483
  end
@@ -543,13 +521,12 @@ db_namespace = namespace :db do
543
521
  end
544
522
 
545
523
  namespace :test do
546
- # desc "Recreate the test database from an existent schema file (schema.rb or structure.sql, depending on `ENV['SCHEMA_FORMAT']` or `config.active_record.schema_format`)"
524
+ # desc "Recreate the test database from an existent schema file (schema.rb or structure.sql, depending on configuration)"
547
525
  task load_schema: %w(db:test:purge) do
548
526
  ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each(env: "test") do |pool|
549
527
  db_config = pool.db_config
550
528
  ActiveRecord::Schema.verbose = false
551
- schema_format = ENV.fetch("SCHEMA_FORMAT", ActiveRecord.schema_format).to_sym
552
- ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, schema_format)
529
+ ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, ENV["SCHEMA_FORMAT"] || db_config.schema_format)
553
530
  end
554
531
  end
555
532
 
@@ -574,8 +551,7 @@ db_namespace = namespace :db do
574
551
  ActiveRecord::Tasks::DatabaseTasks.with_temporary_pool_for_each(env: "test", name: name) do |pool|
575
552
  db_config = pool.db_config
576
553
  ActiveRecord::Schema.verbose = false
577
- schema_format = ENV.fetch("SCHEMA_FORMAT", ActiveRecord.schema_format).to_sym
578
- ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, schema_format)
554
+ ActiveRecord::Tasks::DatabaseTasks.load_schema(db_config, ENV["SCHEMA_FORMAT"] || db_config.schema_format)
579
555
  end
580
556
  end
581
557
  end
@@ -198,7 +198,7 @@ module ActiveRecord
198
198
  end
199
199
 
200
200
  def join_scope(table, foreign_table, foreign_klass)
201
- predicate_builder = predicate_builder(table)
201
+ predicate_builder = klass.predicate_builder.with(TableMetadata.new(klass, table))
202
202
  scope_chain_items = join_scopes(table, predicate_builder)
203
203
  klass_scope = klass_join_scope(table, predicate_builder)
204
204
 
@@ -224,7 +224,7 @@ module ActiveRecord
224
224
  klass_scope
225
225
  end
226
226
 
227
- def join_scopes(table, predicate_builder, klass = self.klass, record = nil) # :nodoc:
227
+ def join_scopes(table, predicate_builder = nil, klass = self.klass, record = nil) # :nodoc:
228
228
  if scope
229
229
  [scope_for(build_scope(table, predicate_builder, klass), record)]
230
230
  else
@@ -232,7 +232,7 @@ module ActiveRecord
232
232
  end
233
233
  end
234
234
 
235
- def klass_join_scope(table, predicate_builder) # :nodoc:
235
+ def klass_join_scope(table, predicate_builder = nil) # :nodoc:
236
236
  relation = build_scope(table, predicate_builder)
237
237
  klass.scope_for_association(relation)
238
238
  end
@@ -333,12 +333,8 @@ module ActiveRecord
333
333
  collect_join_chain
334
334
  end
335
335
 
336
- def build_scope(table, predicate_builder = predicate_builder(table), klass = self.klass)
337
- Relation.create(
338
- klass,
339
- table: table,
340
- predicate_builder: predicate_builder
341
- )
336
+ def build_scope(table, predicate_builder = nil, klass = self.klass)
337
+ Relation.create(klass, table:, predicate_builder:)
342
338
  end
343
339
 
344
340
  def strict_loading?
@@ -357,10 +353,6 @@ module ActiveRecord
357
353
  end
358
354
 
359
355
  private
360
- def predicate_builder(table)
361
- PredicateBuilder.new(TableMetadata.new(klass, table))
362
- end
363
-
364
356
  def primary_key(klass)
365
357
  klass.primary_key || raise(UnknownPrimaryKey.new(klass))
366
358
  end
@@ -531,9 +523,9 @@ module ActiveRecord
531
523
  @association_foreign_key = nil
532
524
  @association_primary_key = nil
533
525
  if options[:query_constraints]
534
- ActiveRecord.deprecator.warn <<~MSG.squish
535
- Setting `query_constraints:` option on `#{active_record}.#{macro} :#{name}` is deprecated.
536
- To maintain current behavior, use the `foreign_key` option instead.
526
+ raise ConfigurationError, <<~MSG.squish
527
+ Setting `query_constraints:` option on `#{active_record}.#{macro} :#{name}` is not allowed.
528
+ To get the same behavior, use the `foreign_key` option instead.
537
529
  MSG
538
530
  end
539
531
 
@@ -562,12 +554,12 @@ module ActiveRecord
562
554
  def foreign_key(infer_from_inverse_of: true)
563
555
  @foreign_key ||= if options[:foreign_key]
564
556
  if options[:foreign_key].is_a?(Array)
565
- options[:foreign_key].map { |fk| fk.to_s.freeze }.freeze
557
+ options[:foreign_key].map { |fk| -fk.to_s.freeze }.freeze
566
558
  else
567
559
  options[:foreign_key].to_s.freeze
568
560
  end
569
561
  elsif options[:query_constraints]
570
- options[:query_constraints].map { |fk| fk.to_s.freeze }.freeze
562
+ options[:query_constraints].map { |fk| -fk.to_s.freeze }.freeze
571
563
  else
572
564
  derived_fk = derive_foreign_key(infer_from_inverse_of: infer_from_inverse_of)
573
565
 
@@ -575,7 +567,12 @@ module ActiveRecord
575
567
  derived_fk = derive_fk_query_constraints(derived_fk)
576
568
  end
577
569
 
578
- derived_fk
570
+ if derived_fk.is_a?(Array)
571
+ derived_fk.map! { |fk| -fk.freeze }
572
+ derived_fk.freeze
573
+ else
574
+ -derived_fk.freeze
575
+ end
579
576
  end
580
577
  end
581
578
 
@@ -1065,7 +1062,7 @@ module ActiveRecord
1065
1062
  source_reflection.scopes + super
1066
1063
  end
1067
1064
 
1068
- def join_scopes(table, predicate_builder, klass = self.klass, record = nil) # :nodoc:
1065
+ def join_scopes(table, predicate_builder = nil, klass = self.klass, record = nil) # :nodoc:
1069
1066
  source_reflection.join_scopes(table, predicate_builder, klass, record) + super
1070
1067
  end
1071
1068
 
@@ -1238,7 +1235,7 @@ module ActiveRecord
1238
1235
  @previous_reflection = previous_reflection
1239
1236
  end
1240
1237
 
1241
- def join_scopes(table, predicate_builder, klass = self.klass, record = nil) # :nodoc:
1238
+ def join_scopes(table, predicate_builder = nil, klass = self.klass, record = nil) # :nodoc:
1242
1239
  scopes = super
1243
1240
  unless @previous_reflection.through_reflection?
1244
1241
  scopes += @previous_reflection.join_scopes(table, predicate_builder, klass, record)
@@ -5,11 +5,12 @@ module ActiveRecord
5
5
  class BatchEnumerator
6
6
  include Enumerable
7
7
 
8
- def initialize(of: 1000, start: nil, finish: nil, relation:, order: :asc, use_ranges: nil) # :nodoc:
8
+ def initialize(of: 1000, start: nil, finish: nil, relation:, cursor:, order: :asc, use_ranges: nil) # :nodoc:
9
9
  @of = of
10
10
  @relation = relation
11
11
  @start = start
12
12
  @finish = finish
13
+ @cursor = cursor
13
14
  @order = order
14
15
  @use_ranges = use_ranges
15
16
  end
@@ -52,7 +53,7 @@ module ActiveRecord
52
53
  def each_record(&block)
53
54
  return to_enum(:each_record) unless block_given?
54
55
 
55
- @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, order: @order).each do |relation|
56
+ @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, cursor: @cursor, order: @order).each do |relation|
56
57
  relation.records.each(&block)
57
58
  end
58
59
  end
@@ -105,7 +106,7 @@ module ActiveRecord
105
106
  # relation.update_all(awesome: true)
106
107
  # end
107
108
  def each(&block)
108
- enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, order: @order, use_ranges: @use_ranges)
109
+ enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, cursor: @cursor, order: @order, use_ranges: @use_ranges)
109
110
  return enum.each(&block) if block_given?
110
111
  enum
111
112
  end
@@ -5,7 +5,7 @@ require "active_record/relation/batches/batch_enumerator"
5
5
  module ActiveRecord
6
6
  # = Active Record \Batches
7
7
  module Batches
8
- ORDER_IGNORE_MESSAGE = "Scoped order is ignored, it's forced to be batch order."
8
+ ORDER_IGNORE_MESSAGE = "Scoped order is ignored, use :cursor with :order to configure custom order."
9
9
  DEFAULT_ORDER = :asc
10
10
 
11
11
  # Looping through a collection of records from the database
@@ -35,11 +35,13 @@ module ActiveRecord
35
35
  #
36
36
  # ==== Options
37
37
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
38
- # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
39
- # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
38
+ # * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
39
+ # * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
40
40
  # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
41
41
  # an order is present in the relation.
42
- # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
42
+ # * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
43
+ # of column names). Defaults to primary key.
44
+ # * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
43
45
  # of :asc or :desc). Defaults to +:asc+.
44
46
  #
45
47
  # class Order < ActiveRecord::Base
@@ -71,20 +73,25 @@ module ActiveRecord
71
73
  #
72
74
  # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
73
75
  # ascending on the primary key ("id ASC").
74
- # This also means that this method only works when the primary key is
76
+ # This also means that this method only works when the cursor column is
75
77
  # orderable (e.g. an integer or string).
76
78
  #
79
+ # NOTE: When using custom columns for batching, they should include at least one unique column
80
+ # (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
81
+ # all columns should be static (unchangeable after it was set).
82
+ #
77
83
  # NOTE: By its nature, batch processing is subject to race conditions if
78
84
  # other processes are modifying the database.
79
- def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER, &block)
85
+ def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER, &block)
80
86
  if block_given?
81
- find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do |records|
87
+ find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do |records|
82
88
  records.each(&block)
83
89
  end
84
90
  else
85
- enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
91
+ enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do
86
92
  relation = self
87
- apply_limits(relation, start, finish, build_batch_orders(order)).size
93
+ cursor = Array(cursor)
94
+ apply_limits(relation, cursor, start, finish, build_batch_orders(cursor, order)).size
88
95
  end
89
96
  end
90
97
  end
@@ -109,11 +116,13 @@ module ActiveRecord
109
116
  #
110
117
  # ==== Options
111
118
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
112
- # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
113
- # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
119
+ # * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
120
+ # * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
114
121
  # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
115
122
  # an order is present in the relation.
116
- # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
123
+ # * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
124
+ # of column names). Defaults to primary key.
125
+ # * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
117
126
  # of :asc or :desc). Defaults to +:asc+.
118
127
  #
119
128
  # class Order < ActiveRecord::Base
@@ -140,21 +149,26 @@ module ActiveRecord
140
149
  #
141
150
  # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
142
151
  # ascending on the primary key ("id ASC").
143
- # This also means that this method only works when the primary key is
152
+ # This also means that this method only works when the cursor column is
144
153
  # orderable (e.g. an integer or string).
145
154
  #
155
+ # NOTE: When using custom columns for batching, they should include at least one unique column
156
+ # (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
157
+ # all columns should be static (unchangeable after it was set).
158
+ #
146
159
  # NOTE: By its nature, batch processing is subject to race conditions if
147
160
  # other processes are modifying the database.
148
- def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER)
161
+ def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER)
149
162
  relation = self
150
163
  unless block_given?
151
- return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
152
- total = apply_limits(relation, start, finish, build_batch_orders(order)).size
164
+ return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do
165
+ cursor = Array(cursor)
166
+ total = apply_limits(relation, cursor, start, finish, build_batch_orders(cursor, order)).size
153
167
  (total - 1).div(batch_size) + 1
154
168
  end
155
169
  end
156
170
 
157
- in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, order: order) do |batch|
171
+ in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do |batch|
158
172
  yield batch.to_a
159
173
  end
160
174
  end
@@ -183,11 +197,13 @@ module ActiveRecord
183
197
  # ==== Options
184
198
  # * <tt>:of</tt> - Specifies the size of the batch. Defaults to 1000.
185
199
  # * <tt>:load</tt> - Specifies if the relation should be loaded. Defaults to false.
186
- # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
187
- # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
200
+ # * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
201
+ # * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
188
202
  # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
189
203
  # an order is present in the relation.
190
- # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
204
+ # * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
205
+ # of column names). Defaults to primary key.
206
+ # * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
191
207
  # of :asc or :desc). Defaults to +:asc+.
192
208
  #
193
209
  # class Order < ActiveRecord::Base
@@ -231,22 +247,25 @@ module ActiveRecord
231
247
  #
232
248
  # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
233
249
  # ascending on the primary key ("id ASC").
234
- # This also means that this method only works when the primary key is
250
+ # This also means that this method only works when the cursor column is
235
251
  # orderable (e.g. an integer or string).
236
252
  #
253
+ # NOTE: When using custom columns for batching, they should include at least one unique column
254
+ # (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
255
+ # all columns should be static (unchangeable after it was set).
256
+ #
237
257
  # NOTE: By its nature, batch processing is subject to race conditions if
238
258
  # other processes are modifying the database.
239
- def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, order: DEFAULT_ORDER, use_ranges: nil, &block)
240
- unless Array(order).all? { |ord| [:asc, :desc].include?(ord) }
241
- raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
242
- end
259
+ def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER, use_ranges: nil, &block)
260
+ cursor = Array(cursor).map(&:to_s)
261
+ ensure_valid_options_for_batching!(cursor, start, finish, order)
243
262
 
244
263
  if arel.orders.present?
245
264
  act_on_ignored_order(error_on_ignore)
246
265
  end
247
266
 
248
267
  unless block
249
- return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, order: order, use_ranges: use_ranges)
268
+ return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, cursor: cursor, order: order, use_ranges: use_ranges)
250
269
  end
251
270
 
252
271
  batch_limit = of
@@ -261,6 +280,7 @@ module ActiveRecord
261
280
  relation: self,
262
281
  start: start,
263
282
  finish: finish,
283
+ cursor: cursor,
264
284
  order: order,
265
285
  batch_limit: batch_limit,
266
286
  &block
@@ -271,6 +291,7 @@ module ActiveRecord
271
291
  start: start,
272
292
  finish: finish,
273
293
  load: load,
294
+ cursor: cursor,
274
295
  order: order,
275
296
  use_ranges: use_ranges,
276
297
  remaining: remaining,
@@ -281,28 +302,51 @@ module ActiveRecord
281
302
  end
282
303
 
283
304
  private
284
- def apply_limits(relation, start, finish, batch_orders)
285
- relation = apply_start_limit(relation, start, batch_orders) if start
286
- relation = apply_finish_limit(relation, finish, batch_orders) if finish
305
+ def ensure_valid_options_for_batching!(cursor, start, finish, order)
306
+ if start && Array(start).size != cursor.size
307
+ raise ArgumentError, ":start must contain one value per cursor column"
308
+ end
309
+
310
+ if finish && Array(finish).size != cursor.size
311
+ raise ArgumentError, ":finish must contain one value per cursor column"
312
+ end
313
+
314
+ if (Array(primary_key) - cursor).any?
315
+ indexes = model.schema_cache.indexes(table_name)
316
+ unique_index = indexes.find { |index| index.unique && index.where.nil? && (Array(index.columns) - cursor).empty? }
317
+
318
+ unless unique_index
319
+ raise ArgumentError, ":cursor must include a primary key or other unique column(s)"
320
+ end
321
+ end
322
+
323
+ if (Array(order) - [:asc, :desc]).any?
324
+ raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
325
+ end
326
+ end
327
+
328
+ def apply_limits(relation, cursor, start, finish, batch_orders)
329
+ relation = apply_start_limit(relation, cursor, start, batch_orders) if start
330
+ relation = apply_finish_limit(relation, cursor, finish, batch_orders) if finish
287
331
  relation
288
332
  end
289
333
 
290
- def apply_start_limit(relation, start, batch_orders)
334
+ def apply_start_limit(relation, cursor, start, batch_orders)
291
335
  operators = batch_orders.map do |_column, order|
292
336
  order == :desc ? :lteq : :gteq
293
337
  end
294
- batch_condition(relation, primary_key, start, operators)
338
+ batch_condition(relation, cursor, start, operators)
295
339
  end
296
340
 
297
- def apply_finish_limit(relation, finish, batch_orders)
341
+ def apply_finish_limit(relation, cursor, finish, batch_orders)
298
342
  operators = batch_orders.map do |_column, order|
299
343
  order == :desc ? :gteq : :lteq
300
344
  end
301
- batch_condition(relation, primary_key, finish, operators)
345
+ batch_condition(relation, cursor, finish, operators)
302
346
  end
303
347
 
304
- def batch_condition(relation, columns, values, operators)
305
- cursor_positions = Array(columns).zip(Array(values), operators)
348
+ def batch_condition(relation, cursor, values, operators)
349
+ cursor_positions = cursor.zip(Array(values), operators)
306
350
 
307
351
  first_clause_column, first_clause_value, operator = cursor_positions.pop
308
352
  where_clause = predicate_builder[first_clause_column, first_clause_value, operator]
@@ -316,9 +360,9 @@ module ActiveRecord
316
360
  relation.where(where_clause)
317
361
  end
318
362
 
319
- def build_batch_orders(order)
320
- get_the_order_of_primary_key(order).map do |column, ord|
321
- [column, ord || DEFAULT_ORDER]
363
+ def build_batch_orders(cursor, order)
364
+ cursor.zip(Array(order)).map do |column, order_|
365
+ [column, order_ || DEFAULT_ORDER]
322
366
  end
323
367
  end
324
368
 
@@ -327,34 +371,28 @@ module ActiveRecord
327
371
 
328
372
  if raise_error
329
373
  raise ArgumentError.new(ORDER_IGNORE_MESSAGE)
330
- elsif logger
331
- logger.warn(ORDER_IGNORE_MESSAGE)
374
+ elsif model.logger
375
+ model.logger.warn(ORDER_IGNORE_MESSAGE)
332
376
  end
333
377
  end
334
378
 
335
- def get_the_order_of_primary_key(order)
336
- Array(primary_key).zip(Array(order))
337
- end
338
-
339
- def batch_on_loaded_relation(relation:, start:, finish:, order:, batch_limit:)
379
+ def batch_on_loaded_relation(relation:, start:, finish:, cursor:, order:, batch_limit:)
340
380
  records = relation.to_a
381
+ order = build_batch_orders(cursor, order).map(&:second)
341
382
 
342
383
  if start || finish
343
384
  records = records.filter do |record|
344
- id = record.id
385
+ values = record_cursor_values(record, cursor)
345
386
 
346
- if order == :asc
347
- (start.nil? || id >= start) && (finish.nil? || id <= finish)
348
- else
349
- (start.nil? || id <= start) && (finish.nil? || id >= finish)
350
- end
387
+ (start.nil? || compare_values_for_order(values, Array(start), order) >= 0) &&
388
+ (finish.nil? || compare_values_for_order(values, Array(finish), order) <= 0)
351
389
  end
352
390
  end
353
391
 
354
- records.sort_by!(&:id)
355
-
356
- if order == :desc
357
- records.reverse!
392
+ records.sort! do |record1, record2|
393
+ values1 = record_cursor_values(record1, cursor)
394
+ values2 = record_cursor_values(record2, cursor)
395
+ compare_values_for_order(values1, values2, order)
358
396
  end
359
397
 
360
398
  records.each_slice(batch_limit) do |subrecords|
@@ -367,44 +405,65 @@ module ActiveRecord
367
405
  nil
368
406
  end
369
407
 
370
- def batch_on_unloaded_relation(relation:, start:, finish:, load:, order:, use_ranges:, remaining:, batch_limit:)
371
- batch_orders = build_batch_orders(order)
408
+ def record_cursor_values(record, cursor)
409
+ record.attributes.slice(*cursor).values
410
+ end
411
+
412
+ # This is a custom implementation of `<=>` operator,
413
+ # which also takes into account how the collection will be ordered.
414
+ def compare_values_for_order(values1, values2, order)
415
+ values1.each_with_index do |element1, index|
416
+ element2 = values2[index]
417
+ direction = order[index]
418
+ comparison = element1 <=> element2
419
+ comparison = -comparison if direction == :desc
420
+ return comparison if comparison != 0
421
+ end
422
+
423
+ 0
424
+ end
425
+
426
+ def batch_on_unloaded_relation(relation:, start:, finish:, load:, cursor:, order:, use_ranges:, remaining:, batch_limit:)
427
+ batch_orders = build_batch_orders(cursor, order)
372
428
  relation = relation.reorder(batch_orders.to_h).limit(batch_limit)
373
- relation = apply_limits(relation, start, finish, batch_orders)
429
+ relation = apply_limits(relation, cursor, start, finish, batch_orders)
374
430
  relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
375
431
  batch_relation = relation
376
- empty_scope = to_sql == klass.unscoped.all.to_sql
432
+ empty_scope = to_sql == model.unscoped.all.to_sql
377
433
 
378
434
  loop do
379
435
  if load
380
436
  records = batch_relation.records
381
- ids = records.map(&:id)
382
- yielded_relation = where(primary_key => ids)
437
+ values = records.pluck(*cursor)
438
+ yielded_relation = where(cursor => values)
383
439
  yielded_relation.load_records(records)
384
440
  elsif (empty_scope && use_ranges != false) || use_ranges
385
- ids = batch_relation.ids
386
- finish = ids.last
441
+ values = batch_relation.pluck(*cursor)
442
+
443
+ finish = values.last
387
444
  if finish
388
- yielded_relation = apply_finish_limit(batch_relation, finish, batch_orders)
445
+ yielded_relation = apply_finish_limit(batch_relation, cursor, finish, batch_orders)
389
446
  yielded_relation = yielded_relation.except(:limit, :order)
390
447
  yielded_relation.skip_query_cache!(false)
391
448
  end
392
449
  else
393
- ids = batch_relation.ids
394
- yielded_relation = where(primary_key => ids)
450
+ values = batch_relation.pluck(*cursor)
451
+ yielded_relation = where(cursor => values)
395
452
  end
396
453
 
397
- break if ids.empty?
454
+ break if values.empty?
398
455
 
399
- primary_key_offset = ids.last
400
- raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
456
+ if values.flatten.any?(nil)
457
+ raise ArgumentError, "Not all of the batch cursor columns were included in the custom select clause "\
458
+ "or some columns contain nil."
459
+ end
401
460
 
402
461
  yield yielded_relation
403
462
 
404
- break if ids.length < batch_limit
463
+ break if values.length < batch_limit
405
464
 
406
465
  if limit_value
407
- remaining -= ids.length
466
+ remaining -= values.length
408
467
 
409
468
  if remaining == 0
410
469
  # Saves a useless iteration when the limit is a multiple of the
@@ -422,7 +481,8 @@ module ActiveRecord
422
481
  end
423
482
  operators << (last_order == :desc ? :lt : :gt)
424
483
 
425
- batch_relation = batch_condition(relation, primary_key, primary_key_offset, operators)
484
+ cursor_value = values.last
485
+ batch_relation = batch_condition(relation, cursor, cursor_value, operators)
426
486
  end
427
487
 
428
488
  nil