activerecord 7.1.5.1 → 8.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +369 -2484
  3. data/README.rdoc +15 -15
  4. data/examples/performance.rb +2 -2
  5. data/lib/active_record/association_relation.rb +2 -1
  6. data/lib/active_record/associations/alias_tracker.rb +31 -23
  7. data/lib/active_record/associations/association.rb +43 -12
  8. data/lib/active_record/associations/belongs_to_association.rb +21 -8
  9. data/lib/active_record/associations/belongs_to_polymorphic_association.rb +3 -2
  10. data/lib/active_record/associations/builder/association.rb +7 -6
  11. data/lib/active_record/associations/builder/belongs_to.rb +1 -0
  12. data/lib/active_record/associations/builder/has_and_belongs_to_many.rb +2 -2
  13. data/lib/active_record/associations/builder/has_many.rb +3 -4
  14. data/lib/active_record/associations/builder/has_one.rb +3 -4
  15. data/lib/active_record/associations/collection_association.rb +17 -9
  16. data/lib/active_record/associations/collection_proxy.rb +14 -1
  17. data/lib/active_record/associations/disable_joins_association_scope.rb +1 -1
  18. data/lib/active_record/associations/errors.rb +265 -0
  19. data/lib/active_record/associations/has_many_association.rb +1 -1
  20. data/lib/active_record/associations/has_many_through_association.rb +10 -3
  21. data/lib/active_record/associations/join_dependency/join_association.rb +1 -1
  22. data/lib/active_record/associations/nested_error.rb +47 -0
  23. data/lib/active_record/associations/preloader/association.rb +4 -3
  24. data/lib/active_record/associations/preloader/branch.rb +7 -1
  25. data/lib/active_record/associations/preloader/through_association.rb +1 -3
  26. data/lib/active_record/associations/singular_association.rb +14 -3
  27. data/lib/active_record/associations/through_association.rb +1 -1
  28. data/lib/active_record/associations.rb +92 -295
  29. data/lib/active_record/asynchronous_queries_tracker.rb +28 -24
  30. data/lib/active_record/attribute_assignment.rb +0 -2
  31. data/lib/active_record/attribute_methods/composite_primary_key.rb +84 -0
  32. data/lib/active_record/attribute_methods/primary_key.rb +25 -61
  33. data/lib/active_record/attribute_methods/read.rb +1 -13
  34. data/lib/active_record/attribute_methods/serialization.rb +4 -24
  35. data/lib/active_record/attribute_methods/time_zone_conversion.rb +9 -18
  36. data/lib/active_record/attribute_methods.rb +71 -75
  37. data/lib/active_record/attributes.rb +63 -49
  38. data/lib/active_record/autosave_association.rb +92 -57
  39. data/lib/active_record/base.rb +2 -3
  40. data/lib/active_record/callbacks.rb +1 -1
  41. data/lib/active_record/connection_adapters/abstract/connection_handler.rb +48 -122
  42. data/lib/active_record/connection_adapters/abstract/connection_pool/queue.rb +0 -1
  43. data/lib/active_record/connection_adapters/abstract/connection_pool/reaper.rb +1 -1
  44. data/lib/active_record/connection_adapters/abstract/connection_pool.rb +286 -77
  45. data/lib/active_record/connection_adapters/abstract/database_statements.rb +119 -55
  46. data/lib/active_record/connection_adapters/abstract/query_cache.rb +197 -76
  47. data/lib/active_record/connection_adapters/abstract/quoting.rb +66 -92
  48. data/lib/active_record/connection_adapters/abstract/schema_creation.rb +4 -5
  49. data/lib/active_record/connection_adapters/abstract/schema_definitions.rb +12 -3
  50. data/lib/active_record/connection_adapters/abstract/schema_statements.rb +48 -12
  51. data/lib/active_record/connection_adapters/abstract/transaction.rb +140 -67
  52. data/lib/active_record/connection_adapters/abstract_adapter.rb +85 -90
  53. data/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +71 -52
  54. data/lib/active_record/connection_adapters/mysql/database_statements.rb +9 -1
  55. data/lib/active_record/connection_adapters/mysql/quoting.rb +50 -57
  56. data/lib/active_record/connection_adapters/mysql/schema_definitions.rb +2 -8
  57. data/lib/active_record/connection_adapters/mysql/schema_statements.rb +56 -45
  58. data/lib/active_record/connection_adapters/mysql2/database_statements.rb +92 -101
  59. data/lib/active_record/connection_adapters/mysql2_adapter.rb +13 -31
  60. data/lib/active_record/connection_adapters/pool_config.rb +14 -13
  61. data/lib/active_record/connection_adapters/postgresql/database_statements.rb +86 -41
  62. data/lib/active_record/connection_adapters/postgresql/oid/array.rb +1 -1
  63. data/lib/active_record/connection_adapters/postgresql/oid/interval.rb +1 -1
  64. data/lib/active_record/connection_adapters/postgresql/oid/point.rb +10 -0
  65. data/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +14 -4
  66. data/lib/active_record/connection_adapters/postgresql/quoting.rb +58 -58
  67. data/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +2 -4
  68. data/lib/active_record/connection_adapters/postgresql/schema_creation.rb +1 -11
  69. data/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +36 -20
  70. data/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +3 -2
  71. data/lib/active_record/connection_adapters/postgresql/schema_statements.rb +75 -28
  72. data/lib/active_record/connection_adapters/postgresql_adapter.rb +73 -113
  73. data/lib/active_record/connection_adapters/schema_cache.rb +124 -131
  74. data/lib/active_record/connection_adapters/sqlite3/column.rb +14 -1
  75. data/lib/active_record/connection_adapters/sqlite3/database_statements.rb +81 -97
  76. data/lib/active_record/connection_adapters/sqlite3/quoting.rb +57 -46
  77. data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +16 -0
  78. data/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb +13 -0
  79. data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb +29 -0
  80. data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb +35 -3
  81. data/lib/active_record/connection_adapters/sqlite3_adapter.rb +183 -87
  82. data/lib/active_record/connection_adapters/statement_pool.rb +4 -2
  83. data/lib/active_record/connection_adapters/trilogy/database_statements.rb +39 -69
  84. data/lib/active_record/connection_adapters/trilogy_adapter.rb +19 -65
  85. data/lib/active_record/connection_adapters.rb +65 -0
  86. data/lib/active_record/connection_handling.rb +74 -37
  87. data/lib/active_record/core.rb +132 -51
  88. data/lib/active_record/counter_cache.rb +19 -10
  89. data/lib/active_record/database_configurations/connection_url_resolver.rb +9 -2
  90. data/lib/active_record/database_configurations/database_config.rb +23 -4
  91. data/lib/active_record/database_configurations/hash_config.rb +46 -34
  92. data/lib/active_record/database_configurations/url_config.rb +20 -1
  93. data/lib/active_record/database_configurations.rb +1 -1
  94. data/lib/active_record/delegated_type.rb +41 -17
  95. data/lib/active_record/dynamic_matchers.rb +2 -2
  96. data/lib/active_record/encryption/config.rb +3 -1
  97. data/lib/active_record/encryption/encryptable_record.rb +7 -7
  98. data/lib/active_record/encryption/encrypted_attribute_type.rb +33 -4
  99. data/lib/active_record/encryption/encryptor.rb +28 -6
  100. data/lib/active_record/encryption/extended_deterministic_queries.rb +4 -2
  101. data/lib/active_record/encryption/key_provider.rb +1 -1
  102. data/lib/active_record/encryption/message_pack_message_serializer.rb +76 -0
  103. data/lib/active_record/encryption/message_serializer.rb +4 -0
  104. data/lib/active_record/encryption/null_encryptor.rb +4 -0
  105. data/lib/active_record/encryption/read_only_null_encryptor.rb +4 -0
  106. data/lib/active_record/encryption/scheme.rb +8 -1
  107. data/lib/active_record/enum.rb +20 -16
  108. data/lib/active_record/errors.rb +54 -20
  109. data/lib/active_record/explain.rb +13 -24
  110. data/lib/active_record/fixtures.rb +37 -33
  111. data/lib/active_record/future_result.rb +21 -13
  112. data/lib/active_record/gem_version.rb +4 -4
  113. data/lib/active_record/inheritance.rb +4 -2
  114. data/lib/active_record/insert_all.rb +19 -16
  115. data/lib/active_record/integration.rb +4 -1
  116. data/lib/active_record/internal_metadata.rb +48 -34
  117. data/lib/active_record/locking/optimistic.rb +8 -7
  118. data/lib/active_record/log_subscriber.rb +5 -32
  119. data/lib/active_record/message_pack.rb +1 -1
  120. data/lib/active_record/migration/command_recorder.rb +33 -14
  121. data/lib/active_record/migration/compatibility.rb +8 -3
  122. data/lib/active_record/migration/default_strategy.rb +4 -5
  123. data/lib/active_record/migration/pending_migration_connection.rb +2 -2
  124. data/lib/active_record/migration.rb +104 -98
  125. data/lib/active_record/model_schema.rb +32 -70
  126. data/lib/active_record/nested_attributes.rb +15 -9
  127. data/lib/active_record/normalization.rb +3 -7
  128. data/lib/active_record/persistence.rb +127 -451
  129. data/lib/active_record/query_cache.rb +19 -8
  130. data/lib/active_record/query_logs.rb +104 -37
  131. data/lib/active_record/query_logs_formatter.rb +17 -28
  132. data/lib/active_record/querying.rb +24 -12
  133. data/lib/active_record/railtie.rb +26 -68
  134. data/lib/active_record/railties/controller_runtime.rb +13 -4
  135. data/lib/active_record/railties/databases.rake +43 -61
  136. data/lib/active_record/reflection.rb +112 -53
  137. data/lib/active_record/relation/batches/batch_enumerator.rb +19 -5
  138. data/lib/active_record/relation/batches.rb +138 -72
  139. data/lib/active_record/relation/calculations.rb +122 -82
  140. data/lib/active_record/relation/delegation.rb +30 -22
  141. data/lib/active_record/relation/finder_methods.rb +32 -18
  142. data/lib/active_record/relation/merger.rb +12 -14
  143. data/lib/active_record/relation/predicate_builder/array_handler.rb +2 -2
  144. data/lib/active_record/relation/predicate_builder/association_query_value.rb +10 -2
  145. data/lib/active_record/relation/predicate_builder/polymorphic_array_value.rb +1 -1
  146. data/lib/active_record/relation/predicate_builder/relation_handler.rb +4 -3
  147. data/lib/active_record/relation/predicate_builder.rb +16 -3
  148. data/lib/active_record/relation/query_attribute.rb +1 -1
  149. data/lib/active_record/relation/query_methods.rb +317 -101
  150. data/lib/active_record/relation/spawn_methods.rb +3 -19
  151. data/lib/active_record/relation/where_clause.rb +7 -19
  152. data/lib/active_record/relation.rb +561 -119
  153. data/lib/active_record/result.rb +95 -46
  154. data/lib/active_record/runtime_registry.rb +39 -0
  155. data/lib/active_record/sanitization.rb +31 -25
  156. data/lib/active_record/schema.rb +8 -6
  157. data/lib/active_record/schema_dumper.rb +53 -20
  158. data/lib/active_record/schema_migration.rb +31 -14
  159. data/lib/active_record/scoping/named.rb +6 -2
  160. data/lib/active_record/signed_id.rb +24 -4
  161. data/lib/active_record/statement_cache.rb +19 -19
  162. data/lib/active_record/store.rb +7 -3
  163. data/lib/active_record/table_metadata.rb +2 -13
  164. data/lib/active_record/tasks/database_tasks.rb +87 -58
  165. data/lib/active_record/tasks/mysql_database_tasks.rb +1 -3
  166. data/lib/active_record/tasks/postgresql_database_tasks.rb +1 -1
  167. data/lib/active_record/tasks/sqlite_database_tasks.rb +4 -3
  168. data/lib/active_record/test_fixtures.rb +98 -89
  169. data/lib/active_record/testing/query_assertions.rb +121 -0
  170. data/lib/active_record/timestamp.rb +2 -2
  171. data/lib/active_record/token_for.rb +22 -12
  172. data/lib/active_record/touch_later.rb +1 -1
  173. data/lib/active_record/transaction.rb +132 -0
  174. data/lib/active_record/transactions.rb +72 -17
  175. data/lib/active_record/translation.rb +0 -2
  176. data/lib/active_record/type/serialized.rb +1 -3
  177. data/lib/active_record/type_caster/connection.rb +4 -4
  178. data/lib/active_record/validations/associated.rb +9 -3
  179. data/lib/active_record/validations/uniqueness.rb +23 -18
  180. data/lib/active_record/validations.rb +4 -1
  181. data/lib/active_record.rb +138 -57
  182. data/lib/arel/alias_predication.rb +1 -1
  183. data/lib/arel/collectors/bind.rb +4 -2
  184. data/lib/arel/collectors/composite.rb +7 -0
  185. data/lib/arel/collectors/sql_string.rb +2 -2
  186. data/lib/arel/collectors/substitute_binds.rb +3 -3
  187. data/lib/arel/nodes/binary.rb +1 -7
  188. data/lib/arel/nodes/bound_sql_literal.rb +9 -5
  189. data/lib/arel/nodes/{and.rb → nary.rb} +5 -2
  190. data/lib/arel/nodes/node.rb +5 -4
  191. data/lib/arel/nodes/sql_literal.rb +8 -1
  192. data/lib/arel/nodes.rb +2 -2
  193. data/lib/arel/predications.rb +1 -1
  194. data/lib/arel/select_manager.rb +1 -1
  195. data/lib/arel/table.rb +3 -7
  196. data/lib/arel/tree_manager.rb +3 -2
  197. data/lib/arel/update_manager.rb +2 -1
  198. data/lib/arel/visitors/dot.rb +1 -0
  199. data/lib/arel/visitors/mysql.rb +9 -4
  200. data/lib/arel/visitors/postgresql.rb +1 -12
  201. data/lib/arel/visitors/sqlite.rb +25 -0
  202. data/lib/arel/visitors/to_sql.rb +29 -16
  203. data/lib/arel.rb +7 -3
  204. data/lib/rails/generators/active_record/migration/templates/create_table_migration.rb.tt +4 -1
  205. metadata +18 -16
  206. data/lib/active_record/relation/record_fetch_warning.rb +0 -49
@@ -5,11 +5,12 @@ module ActiveRecord
5
5
  class BatchEnumerator
6
6
  include Enumerable
7
7
 
8
- def initialize(of: 1000, start: nil, finish: nil, relation:, order: :asc, use_ranges: nil) # :nodoc:
8
+ def initialize(of: 1000, start: nil, finish: nil, relation:, cursor:, order: :asc, use_ranges: nil) # :nodoc:
9
9
  @of = of
10
10
  @relation = relation
11
11
  @start = start
12
12
  @finish = finish
13
+ @cursor = cursor
13
14
  @order = order
14
15
  @use_ranges = use_ranges
15
16
  end
@@ -52,7 +53,7 @@ module ActiveRecord
52
53
  def each_record(&block)
53
54
  return to_enum(:each_record) unless block_given?
54
55
 
55
- @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, order: @order).each do |relation|
56
+ @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, cursor: @cursor, order: @order).each do |relation|
56
57
  relation.records.each(&block)
57
58
  end
58
59
  end
@@ -77,13 +78,26 @@ module ActiveRecord
77
78
  end
78
79
  end
79
80
 
80
- # Destroys records in batches.
81
+ # Touches records in batches. Returns the total number of rows affected.
82
+ #
83
+ # Person.in_batches.touch_all
84
+ #
85
+ # See Relation#touch_all for details of how each batch is touched.
86
+ def touch_all(...)
87
+ sum do |relation|
88
+ relation.touch_all(...)
89
+ end
90
+ end
91
+
92
+ # Destroys records in batches. Returns the total number of rows affected.
81
93
  #
82
94
  # Person.where("age < 10").in_batches.destroy_all
83
95
  #
84
96
  # See Relation#destroy_all for details of how each batch is destroyed.
85
97
  def destroy_all
86
- each(&:destroy_all)
98
+ sum do |relation|
99
+ relation.destroy_all.count(&:destroyed?)
100
+ end
87
101
  end
88
102
 
89
103
  # Yields an ActiveRecord::Relation object for each batch of records.
@@ -92,7 +106,7 @@ module ActiveRecord
92
106
  # relation.update_all(awesome: true)
93
107
  # end
94
108
  def each(&block)
95
- enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, order: @order, use_ranges: @use_ranges)
109
+ enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, cursor: @cursor, order: @order, use_ranges: @use_ranges)
96
110
  return enum.each(&block) if block_given?
97
111
  enum
98
112
  end
@@ -5,7 +5,7 @@ require "active_record/relation/batches/batch_enumerator"
5
5
  module ActiveRecord
6
6
  # = Active Record \Batches
7
7
  module Batches
8
- ORDER_IGNORE_MESSAGE = "Scoped order is ignored, it's forced to be batch order."
8
+ ORDER_IGNORE_MESSAGE = "Scoped order is ignored, use :cursor with :order to configure custom order."
9
9
  DEFAULT_ORDER = :asc
10
10
 
11
11
  # Looping through a collection of records from the database
@@ -35,11 +35,13 @@ module ActiveRecord
35
35
  #
36
36
  # ==== Options
37
37
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
38
- # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
39
- # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
38
+ # * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
39
+ # * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
40
40
  # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
41
41
  # an order is present in the relation.
42
- # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
42
+ # * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
43
+ # of column names). Defaults to primary key.
44
+ # * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
43
45
  # of :asc or :desc). Defaults to +:asc+.
44
46
  #
45
47
  # class Order < ActiveRecord::Base
@@ -71,20 +73,25 @@ module ActiveRecord
71
73
  #
72
74
  # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
73
75
  # ascending on the primary key ("id ASC").
74
- # This also means that this method only works when the primary key is
76
+ # This also means that this method only works when the cursor column is
75
77
  # orderable (e.g. an integer or string).
76
78
  #
79
+ # NOTE: When using custom columns for batching, they should include at least one unique column
80
+ # (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
81
+ # all columns should be static (unchangeable after it was set).
82
+ #
77
83
  # NOTE: By its nature, batch processing is subject to race conditions if
78
84
  # other processes are modifying the database.
79
- def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER, &block)
85
+ def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER, &block)
80
86
  if block_given?
81
- find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do |records|
87
+ find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do |records|
82
88
  records.each(&block)
83
89
  end
84
90
  else
85
- enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
91
+ enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do
86
92
  relation = self
87
- apply_limits(relation, start, finish, build_batch_orders(order)).size
93
+ cursor = Array(cursor)
94
+ apply_limits(relation, cursor, start, finish, build_batch_orders(cursor, order)).size
88
95
  end
89
96
  end
90
97
  end
@@ -109,11 +116,13 @@ module ActiveRecord
109
116
  #
110
117
  # ==== Options
111
118
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
112
- # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
113
- # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
119
+ # * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
120
+ # * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
114
121
  # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
115
122
  # an order is present in the relation.
116
- # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
123
+ # * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
124
+ # of column names). Defaults to primary key.
125
+ # * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
117
126
  # of :asc or :desc). Defaults to +:asc+.
118
127
  #
119
128
  # class Order < ActiveRecord::Base
@@ -140,21 +149,26 @@ module ActiveRecord
140
149
  #
141
150
  # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
142
151
  # ascending on the primary key ("id ASC").
143
- # This also means that this method only works when the primary key is
152
+ # This also means that this method only works when the cursor column is
144
153
  # orderable (e.g. an integer or string).
145
154
  #
155
+ # NOTE: When using custom columns for batching, they should include at least one unique column
156
+ # (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
157
+ # all columns should be static (unchangeable after it was set).
158
+ #
146
159
  # NOTE: By its nature, batch processing is subject to race conditions if
147
160
  # other processes are modifying the database.
148
- def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER)
161
+ def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER)
149
162
  relation = self
150
163
  unless block_given?
151
- return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
152
- total = apply_limits(relation, start, finish, build_batch_orders(order)).size
164
+ return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do
165
+ cursor = Array(cursor)
166
+ total = apply_limits(relation, cursor, start, finish, build_batch_orders(cursor, order)).size
153
167
  (total - 1).div(batch_size) + 1
154
168
  end
155
169
  end
156
170
 
157
- in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, order: order) do |batch|
171
+ in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do |batch|
158
172
  yield batch.to_a
159
173
  end
160
174
  end
@@ -183,11 +197,13 @@ module ActiveRecord
183
197
  # ==== Options
184
198
  # * <tt>:of</tt> - Specifies the size of the batch. Defaults to 1000.
185
199
  # * <tt>:load</tt> - Specifies if the relation should be loaded. Defaults to false.
186
- # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
187
- # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
200
+ # * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
201
+ # * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
188
202
  # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
189
203
  # an order is present in the relation.
190
- # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
204
+ # * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
205
+ # of column names). Defaults to primary key.
206
+ # * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
191
207
  # of :asc or :desc). Defaults to +:asc+.
192
208
  #
193
209
  # class Order < ActiveRecord::Base
@@ -231,24 +247,27 @@ module ActiveRecord
231
247
  #
232
248
  # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
233
249
  # ascending on the primary key ("id ASC").
234
- # This also means that this method only works when the primary key is
250
+ # This also means that this method only works when the cursor column is
235
251
  # orderable (e.g. an integer or string).
236
252
  #
253
+ # NOTE: When using custom columns for batching, they should include at least one unique column
254
+ # (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
255
+ # all columns should be static (unchangeable after it was set).
256
+ #
237
257
  # NOTE: By its nature, batch processing is subject to race conditions if
238
258
  # other processes are modifying the database.
239
- def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, order: DEFAULT_ORDER, use_ranges: nil, &block)
240
- unless Array(order).all? { |ord| [:asc, :desc].include?(ord) }
241
- raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
242
- end
243
-
244
- unless block
245
- return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, order: order, use_ranges: use_ranges)
246
- end
259
+ def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER, use_ranges: nil, &block)
260
+ cursor = Array(cursor).map(&:to_s)
261
+ ensure_valid_options_for_batching!(cursor, start, finish, order)
247
262
 
248
263
  if arel.orders.present?
249
264
  act_on_ignored_order(error_on_ignore)
250
265
  end
251
266
 
267
+ unless block
268
+ return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, cursor: cursor, order: order, use_ranges: use_ranges)
269
+ end
270
+
252
271
  batch_limit = of
253
272
 
254
273
  if limit_value
@@ -261,6 +280,7 @@ module ActiveRecord
261
280
  relation: self,
262
281
  start: start,
263
282
  finish: finish,
283
+ cursor: cursor,
264
284
  order: order,
265
285
  batch_limit: batch_limit,
266
286
  &block
@@ -271,6 +291,7 @@ module ActiveRecord
271
291
  start: start,
272
292
  finish: finish,
273
293
  load: load,
294
+ cursor: cursor,
274
295
  order: order,
275
296
  use_ranges: use_ranges,
276
297
  remaining: remaining,
@@ -281,28 +302,51 @@ module ActiveRecord
281
302
  end
282
303
 
283
304
  private
284
- def apply_limits(relation, start, finish, batch_orders)
285
- relation = apply_start_limit(relation, start, batch_orders) if start
286
- relation = apply_finish_limit(relation, finish, batch_orders) if finish
305
+ def ensure_valid_options_for_batching!(cursor, start, finish, order)
306
+ if start && Array(start).size != cursor.size
307
+ raise ArgumentError, ":start must contain one value per cursor column"
308
+ end
309
+
310
+ if finish && Array(finish).size != cursor.size
311
+ raise ArgumentError, ":finish must contain one value per cursor column"
312
+ end
313
+
314
+ if (Array(primary_key) - cursor).any?
315
+ indexes = model.schema_cache.indexes(table_name)
316
+ unique_index = indexes.find { |index| index.unique && index.where.nil? && (Array(index.columns) - cursor).empty? }
317
+
318
+ unless unique_index
319
+ raise ArgumentError, ":cursor must include a primary key or other unique column(s)"
320
+ end
321
+ end
322
+
323
+ if (Array(order) - [:asc, :desc]).any?
324
+ raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
325
+ end
326
+ end
327
+
328
+ def apply_limits(relation, cursor, start, finish, batch_orders)
329
+ relation = apply_start_limit(relation, cursor, start, batch_orders) if start
330
+ relation = apply_finish_limit(relation, cursor, finish, batch_orders) if finish
287
331
  relation
288
332
  end
289
333
 
290
- def apply_start_limit(relation, start, batch_orders)
334
+ def apply_start_limit(relation, cursor, start, batch_orders)
291
335
  operators = batch_orders.map do |_column, order|
292
336
  order == :desc ? :lteq : :gteq
293
337
  end
294
- batch_condition(relation, primary_key, start, operators)
338
+ batch_condition(relation, cursor, start, operators)
295
339
  end
296
340
 
297
- def apply_finish_limit(relation, finish, batch_orders)
341
+ def apply_finish_limit(relation, cursor, finish, batch_orders)
298
342
  operators = batch_orders.map do |_column, order|
299
343
  order == :desc ? :gteq : :lteq
300
344
  end
301
- batch_condition(relation, primary_key, finish, operators)
345
+ batch_condition(relation, cursor, finish, operators)
302
346
  end
303
347
 
304
- def batch_condition(relation, columns, values, operators)
305
- cursor_positions = Array(columns).zip(Array(values), operators)
348
+ def batch_condition(relation, cursor, values, operators)
349
+ cursor_positions = cursor.zip(Array(values), operators)
306
350
 
307
351
  first_clause_column, first_clause_value, operator = cursor_positions.pop
308
352
  where_clause = predicate_builder[first_clause_column, first_clause_value, operator]
@@ -316,9 +360,9 @@ module ActiveRecord
316
360
  relation.where(where_clause)
317
361
  end
318
362
 
319
- def build_batch_orders(order)
320
- get_the_order_of_primary_key(order).map do |column, ord|
321
- [column, ord || DEFAULT_ORDER]
363
+ def build_batch_orders(cursor, order)
364
+ cursor.zip(Array(order)).map do |column, order_|
365
+ [column, order_ || DEFAULT_ORDER]
322
366
  end
323
367
  end
324
368
 
@@ -327,33 +371,33 @@ module ActiveRecord
327
371
 
328
372
  if raise_error
329
373
  raise ArgumentError.new(ORDER_IGNORE_MESSAGE)
330
- elsif logger
331
- logger.warn(ORDER_IGNORE_MESSAGE)
374
+ elsif model.logger
375
+ model.logger.warn(ORDER_IGNORE_MESSAGE)
332
376
  end
333
377
  end
334
378
 
335
- def get_the_order_of_primary_key(order)
336
- Array(primary_key).zip(Array(order))
337
- end
338
-
339
- def batch_on_loaded_relation(relation:, start:, finish:, order:, batch_limit:)
379
+ def batch_on_loaded_relation(relation:, start:, finish:, cursor:, order:, batch_limit:)
340
380
  records = relation.to_a
381
+ order = build_batch_orders(cursor, order).map(&:second)
341
382
 
342
383
  if start || finish
343
384
  records = records.filter do |record|
344
- (start.nil? || record.id >= start) && (finish.nil? || record.id <= finish)
385
+ values = record_cursor_values(record, cursor)
386
+
387
+ (start.nil? || compare_values_for_order(values, Array(start), order) >= 0) &&
388
+ (finish.nil? || compare_values_for_order(values, Array(finish), order) <= 0)
345
389
  end
346
390
  end
347
391
 
348
- records = records.sort_by { |record| record.id }
349
-
350
- if order == :desc
351
- records.reverse!
392
+ records.sort! do |record1, record2|
393
+ values1 = record_cursor_values(record1, cursor)
394
+ values2 = record_cursor_values(record2, cursor)
395
+ compare_values_for_order(values1, values2, order)
352
396
  end
353
397
 
354
- (0...records.size).step(batch_limit).each do |start|
398
+ records.each_slice(batch_limit) do |subrecords|
355
399
  subrelation = relation.spawn
356
- subrelation.load_records(records[start, batch_limit])
400
+ subrelation.load_records(subrecords)
357
401
 
358
402
  yield subrelation
359
403
  end
@@ -361,44 +405,65 @@ module ActiveRecord
361
405
  nil
362
406
  end
363
407
 
364
- def batch_on_unloaded_relation(relation:, start:, finish:, load:, order:, use_ranges:, remaining:, batch_limit:)
365
- batch_orders = build_batch_orders(order)
408
+ def record_cursor_values(record, cursor)
409
+ record.attributes.slice(*cursor).values
410
+ end
411
+
412
+ # This is a custom implementation of `<=>` operator,
413
+ # which also takes into account how the collection will be ordered.
414
+ def compare_values_for_order(values1, values2, order)
415
+ values1.each_with_index do |element1, index|
416
+ element2 = values2[index]
417
+ direction = order[index]
418
+ comparison = element1 <=> element2
419
+ comparison = -comparison if direction == :desc
420
+ return comparison if comparison != 0
421
+ end
422
+
423
+ 0
424
+ end
425
+
426
+ def batch_on_unloaded_relation(relation:, start:, finish:, load:, cursor:, order:, use_ranges:, remaining:, batch_limit:)
427
+ batch_orders = build_batch_orders(cursor, order)
366
428
  relation = relation.reorder(batch_orders.to_h).limit(batch_limit)
367
- relation = apply_limits(relation, start, finish, batch_orders)
429
+ relation = apply_limits(relation, cursor, start, finish, batch_orders)
368
430
  relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
369
431
  batch_relation = relation
370
- empty_scope = to_sql == klass.unscoped.all.to_sql
432
+ empty_scope = to_sql == model.unscoped.all.to_sql
371
433
 
372
434
  loop do
373
435
  if load
374
436
  records = batch_relation.records
375
- ids = records.map(&:id)
376
- yielded_relation = where(primary_key => ids)
437
+ values = records.pluck(*cursor)
438
+ yielded_relation = where(cursor => values)
377
439
  yielded_relation.load_records(records)
378
440
  elsif (empty_scope && use_ranges != false) || use_ranges
379
- ids = batch_relation.ids
380
- finish = ids.last
441
+ values = batch_relation.pluck(*cursor)
442
+
443
+ finish = values.last
381
444
  if finish
382
- yielded_relation = apply_finish_limit(batch_relation, finish, batch_orders)
445
+ yielded_relation = apply_finish_limit(batch_relation, cursor, finish, batch_orders)
383
446
  yielded_relation = yielded_relation.except(:limit, :order)
384
447
  yielded_relation.skip_query_cache!(false)
385
448
  end
386
449
  else
387
- ids = batch_relation.ids
388
- yielded_relation = where(primary_key => ids)
450
+ values = batch_relation.pluck(*cursor)
451
+ yielded_relation = where(cursor => values)
389
452
  end
390
453
 
391
- break if ids.empty?
454
+ break if values.empty?
392
455
 
393
- primary_key_offset = ids.last
394
- raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
456
+ if values.flatten.any?(nil)
457
+ raise ArgumentError, "Not all of the batch cursor columns were included in the custom select clause "\
458
+ "or some columns contain nil."
459
+ end
395
460
 
396
461
  yield yielded_relation
397
462
 
398
- break if ids.length < batch_limit
463
+ break if values.length < batch_limit
399
464
 
400
465
  if limit_value
401
- remaining -= ids.length
466
+ remaining -= values.length
402
467
 
403
468
  if remaining == 0
404
469
  # Saves a useless iteration when the limit is a multiple of the
@@ -416,7 +481,8 @@ module ActiveRecord
416
481
  end
417
482
  operators << (last_order == :desc ? :lt : :gt)
418
483
 
419
- batch_relation = batch_condition(relation, primary_key, primary_key_offset, operators)
484
+ cursor_value = values.last
485
+ batch_relation = batch_condition(relation, cursor, cursor_value, operators)
420
486
  end
421
487
 
422
488
  nil