activerecord 7.1.5.1 → 8.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +369 -2484
- data/README.rdoc +15 -15
- data/examples/performance.rb +2 -2
- data/lib/active_record/association_relation.rb +2 -1
- data/lib/active_record/associations/alias_tracker.rb +31 -23
- data/lib/active_record/associations/association.rb +43 -12
- data/lib/active_record/associations/belongs_to_association.rb +21 -8
- data/lib/active_record/associations/belongs_to_polymorphic_association.rb +3 -2
- data/lib/active_record/associations/builder/association.rb +7 -6
- data/lib/active_record/associations/builder/belongs_to.rb +1 -0
- data/lib/active_record/associations/builder/has_and_belongs_to_many.rb +2 -2
- data/lib/active_record/associations/builder/has_many.rb +3 -4
- data/lib/active_record/associations/builder/has_one.rb +3 -4
- data/lib/active_record/associations/collection_association.rb +17 -9
- data/lib/active_record/associations/collection_proxy.rb +14 -1
- data/lib/active_record/associations/disable_joins_association_scope.rb +1 -1
- data/lib/active_record/associations/errors.rb +265 -0
- data/lib/active_record/associations/has_many_association.rb +1 -1
- data/lib/active_record/associations/has_many_through_association.rb +10 -3
- data/lib/active_record/associations/join_dependency/join_association.rb +1 -1
- data/lib/active_record/associations/nested_error.rb +47 -0
- data/lib/active_record/associations/preloader/association.rb +4 -3
- data/lib/active_record/associations/preloader/branch.rb +7 -1
- data/lib/active_record/associations/preloader/through_association.rb +1 -3
- data/lib/active_record/associations/singular_association.rb +14 -3
- data/lib/active_record/associations/through_association.rb +1 -1
- data/lib/active_record/associations.rb +92 -295
- data/lib/active_record/asynchronous_queries_tracker.rb +28 -24
- data/lib/active_record/attribute_assignment.rb +0 -2
- data/lib/active_record/attribute_methods/composite_primary_key.rb +84 -0
- data/lib/active_record/attribute_methods/primary_key.rb +25 -61
- data/lib/active_record/attribute_methods/read.rb +1 -13
- data/lib/active_record/attribute_methods/serialization.rb +4 -24
- data/lib/active_record/attribute_methods/time_zone_conversion.rb +9 -18
- data/lib/active_record/attribute_methods.rb +71 -75
- data/lib/active_record/attributes.rb +63 -49
- data/lib/active_record/autosave_association.rb +92 -57
- data/lib/active_record/base.rb +2 -3
- data/lib/active_record/callbacks.rb +1 -1
- data/lib/active_record/connection_adapters/abstract/connection_handler.rb +48 -122
- data/lib/active_record/connection_adapters/abstract/connection_pool/queue.rb +0 -1
- data/lib/active_record/connection_adapters/abstract/connection_pool/reaper.rb +1 -1
- data/lib/active_record/connection_adapters/abstract/connection_pool.rb +286 -77
- data/lib/active_record/connection_adapters/abstract/database_statements.rb +119 -55
- data/lib/active_record/connection_adapters/abstract/query_cache.rb +197 -76
- data/lib/active_record/connection_adapters/abstract/quoting.rb +66 -92
- data/lib/active_record/connection_adapters/abstract/schema_creation.rb +4 -5
- data/lib/active_record/connection_adapters/abstract/schema_definitions.rb +12 -3
- data/lib/active_record/connection_adapters/abstract/schema_statements.rb +48 -12
- data/lib/active_record/connection_adapters/abstract/transaction.rb +140 -67
- data/lib/active_record/connection_adapters/abstract_adapter.rb +85 -90
- data/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +71 -52
- data/lib/active_record/connection_adapters/mysql/database_statements.rb +9 -1
- data/lib/active_record/connection_adapters/mysql/quoting.rb +50 -57
- data/lib/active_record/connection_adapters/mysql/schema_definitions.rb +2 -8
- data/lib/active_record/connection_adapters/mysql/schema_statements.rb +56 -45
- data/lib/active_record/connection_adapters/mysql2/database_statements.rb +92 -101
- data/lib/active_record/connection_adapters/mysql2_adapter.rb +13 -31
- data/lib/active_record/connection_adapters/pool_config.rb +14 -13
- data/lib/active_record/connection_adapters/postgresql/database_statements.rb +86 -41
- data/lib/active_record/connection_adapters/postgresql/oid/array.rb +1 -1
- data/lib/active_record/connection_adapters/postgresql/oid/interval.rb +1 -1
- data/lib/active_record/connection_adapters/postgresql/oid/point.rb +10 -0
- data/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +14 -4
- data/lib/active_record/connection_adapters/postgresql/quoting.rb +58 -58
- data/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +2 -4
- data/lib/active_record/connection_adapters/postgresql/schema_creation.rb +1 -11
- data/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +36 -20
- data/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +3 -2
- data/lib/active_record/connection_adapters/postgresql/schema_statements.rb +75 -28
- data/lib/active_record/connection_adapters/postgresql_adapter.rb +73 -113
- data/lib/active_record/connection_adapters/schema_cache.rb +124 -131
- data/lib/active_record/connection_adapters/sqlite3/column.rb +14 -1
- data/lib/active_record/connection_adapters/sqlite3/database_statements.rb +81 -97
- data/lib/active_record/connection_adapters/sqlite3/quoting.rb +57 -46
- data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +16 -0
- data/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb +13 -0
- data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb +29 -0
- data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb +35 -3
- data/lib/active_record/connection_adapters/sqlite3_adapter.rb +183 -87
- data/lib/active_record/connection_adapters/statement_pool.rb +4 -2
- data/lib/active_record/connection_adapters/trilogy/database_statements.rb +39 -69
- data/lib/active_record/connection_adapters/trilogy_adapter.rb +19 -65
- data/lib/active_record/connection_adapters.rb +65 -0
- data/lib/active_record/connection_handling.rb +74 -37
- data/lib/active_record/core.rb +132 -51
- data/lib/active_record/counter_cache.rb +19 -10
- data/lib/active_record/database_configurations/connection_url_resolver.rb +9 -2
- data/lib/active_record/database_configurations/database_config.rb +23 -4
- data/lib/active_record/database_configurations/hash_config.rb +46 -34
- data/lib/active_record/database_configurations/url_config.rb +20 -1
- data/lib/active_record/database_configurations.rb +1 -1
- data/lib/active_record/delegated_type.rb +41 -17
- data/lib/active_record/dynamic_matchers.rb +2 -2
- data/lib/active_record/encryption/config.rb +3 -1
- data/lib/active_record/encryption/encryptable_record.rb +7 -7
- data/lib/active_record/encryption/encrypted_attribute_type.rb +33 -4
- data/lib/active_record/encryption/encryptor.rb +28 -6
- data/lib/active_record/encryption/extended_deterministic_queries.rb +4 -2
- data/lib/active_record/encryption/key_provider.rb +1 -1
- data/lib/active_record/encryption/message_pack_message_serializer.rb +76 -0
- data/lib/active_record/encryption/message_serializer.rb +4 -0
- data/lib/active_record/encryption/null_encryptor.rb +4 -0
- data/lib/active_record/encryption/read_only_null_encryptor.rb +4 -0
- data/lib/active_record/encryption/scheme.rb +8 -1
- data/lib/active_record/enum.rb +20 -16
- data/lib/active_record/errors.rb +54 -20
- data/lib/active_record/explain.rb +13 -24
- data/lib/active_record/fixtures.rb +37 -33
- data/lib/active_record/future_result.rb +21 -13
- data/lib/active_record/gem_version.rb +4 -4
- data/lib/active_record/inheritance.rb +4 -2
- data/lib/active_record/insert_all.rb +19 -16
- data/lib/active_record/integration.rb +4 -1
- data/lib/active_record/internal_metadata.rb +48 -34
- data/lib/active_record/locking/optimistic.rb +8 -7
- data/lib/active_record/log_subscriber.rb +5 -32
- data/lib/active_record/message_pack.rb +1 -1
- data/lib/active_record/migration/command_recorder.rb +33 -14
- data/lib/active_record/migration/compatibility.rb +8 -3
- data/lib/active_record/migration/default_strategy.rb +4 -5
- data/lib/active_record/migration/pending_migration_connection.rb +2 -2
- data/lib/active_record/migration.rb +104 -98
- data/lib/active_record/model_schema.rb +32 -70
- data/lib/active_record/nested_attributes.rb +15 -9
- data/lib/active_record/normalization.rb +3 -7
- data/lib/active_record/persistence.rb +127 -451
- data/lib/active_record/query_cache.rb +19 -8
- data/lib/active_record/query_logs.rb +104 -37
- data/lib/active_record/query_logs_formatter.rb +17 -28
- data/lib/active_record/querying.rb +24 -12
- data/lib/active_record/railtie.rb +26 -68
- data/lib/active_record/railties/controller_runtime.rb +13 -4
- data/lib/active_record/railties/databases.rake +43 -61
- data/lib/active_record/reflection.rb +112 -53
- data/lib/active_record/relation/batches/batch_enumerator.rb +19 -5
- data/lib/active_record/relation/batches.rb +138 -72
- data/lib/active_record/relation/calculations.rb +122 -82
- data/lib/active_record/relation/delegation.rb +30 -22
- data/lib/active_record/relation/finder_methods.rb +32 -18
- data/lib/active_record/relation/merger.rb +12 -14
- data/lib/active_record/relation/predicate_builder/array_handler.rb +2 -2
- data/lib/active_record/relation/predicate_builder/association_query_value.rb +10 -2
- data/lib/active_record/relation/predicate_builder/polymorphic_array_value.rb +1 -1
- data/lib/active_record/relation/predicate_builder/relation_handler.rb +4 -3
- data/lib/active_record/relation/predicate_builder.rb +16 -3
- data/lib/active_record/relation/query_attribute.rb +1 -1
- data/lib/active_record/relation/query_methods.rb +317 -101
- data/lib/active_record/relation/spawn_methods.rb +3 -19
- data/lib/active_record/relation/where_clause.rb +7 -19
- data/lib/active_record/relation.rb +561 -119
- data/lib/active_record/result.rb +95 -46
- data/lib/active_record/runtime_registry.rb +39 -0
- data/lib/active_record/sanitization.rb +31 -25
- data/lib/active_record/schema.rb +8 -6
- data/lib/active_record/schema_dumper.rb +53 -20
- data/lib/active_record/schema_migration.rb +31 -14
- data/lib/active_record/scoping/named.rb +6 -2
- data/lib/active_record/signed_id.rb +24 -4
- data/lib/active_record/statement_cache.rb +19 -19
- data/lib/active_record/store.rb +7 -3
- data/lib/active_record/table_metadata.rb +2 -13
- data/lib/active_record/tasks/database_tasks.rb +87 -58
- data/lib/active_record/tasks/mysql_database_tasks.rb +1 -3
- data/lib/active_record/tasks/postgresql_database_tasks.rb +1 -1
- data/lib/active_record/tasks/sqlite_database_tasks.rb +4 -3
- data/lib/active_record/test_fixtures.rb +98 -89
- data/lib/active_record/testing/query_assertions.rb +121 -0
- data/lib/active_record/timestamp.rb +2 -2
- data/lib/active_record/token_for.rb +22 -12
- data/lib/active_record/touch_later.rb +1 -1
- data/lib/active_record/transaction.rb +132 -0
- data/lib/active_record/transactions.rb +72 -17
- data/lib/active_record/translation.rb +0 -2
- data/lib/active_record/type/serialized.rb +1 -3
- data/lib/active_record/type_caster/connection.rb +4 -4
- data/lib/active_record/validations/associated.rb +9 -3
- data/lib/active_record/validations/uniqueness.rb +23 -18
- data/lib/active_record/validations.rb +4 -1
- data/lib/active_record.rb +138 -57
- data/lib/arel/alias_predication.rb +1 -1
- data/lib/arel/collectors/bind.rb +4 -2
- data/lib/arel/collectors/composite.rb +7 -0
- data/lib/arel/collectors/sql_string.rb +2 -2
- data/lib/arel/collectors/substitute_binds.rb +3 -3
- data/lib/arel/nodes/binary.rb +1 -7
- data/lib/arel/nodes/bound_sql_literal.rb +9 -5
- data/lib/arel/nodes/{and.rb → nary.rb} +5 -2
- data/lib/arel/nodes/node.rb +5 -4
- data/lib/arel/nodes/sql_literal.rb +8 -1
- data/lib/arel/nodes.rb +2 -2
- data/lib/arel/predications.rb +1 -1
- data/lib/arel/select_manager.rb +1 -1
- data/lib/arel/table.rb +3 -7
- data/lib/arel/tree_manager.rb +3 -2
- data/lib/arel/update_manager.rb +2 -1
- data/lib/arel/visitors/dot.rb +1 -0
- data/lib/arel/visitors/mysql.rb +9 -4
- data/lib/arel/visitors/postgresql.rb +1 -12
- data/lib/arel/visitors/sqlite.rb +25 -0
- data/lib/arel/visitors/to_sql.rb +29 -16
- data/lib/arel.rb +7 -3
- data/lib/rails/generators/active_record/migration/templates/create_table_migration.rb.tt +4 -1
- metadata +18 -16
- data/lib/active_record/relation/record_fetch_warning.rb +0 -49
@@ -5,11 +5,12 @@ module ActiveRecord
|
|
5
5
|
class BatchEnumerator
|
6
6
|
include Enumerable
|
7
7
|
|
8
|
-
def initialize(of: 1000, start: nil, finish: nil, relation:, order: :asc, use_ranges: nil) # :nodoc:
|
8
|
+
def initialize(of: 1000, start: nil, finish: nil, relation:, cursor:, order: :asc, use_ranges: nil) # :nodoc:
|
9
9
|
@of = of
|
10
10
|
@relation = relation
|
11
11
|
@start = start
|
12
12
|
@finish = finish
|
13
|
+
@cursor = cursor
|
13
14
|
@order = order
|
14
15
|
@use_ranges = use_ranges
|
15
16
|
end
|
@@ -52,7 +53,7 @@ module ActiveRecord
|
|
52
53
|
def each_record(&block)
|
53
54
|
return to_enum(:each_record) unless block_given?
|
54
55
|
|
55
|
-
@relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, order: @order).each do |relation|
|
56
|
+
@relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, cursor: @cursor, order: @order).each do |relation|
|
56
57
|
relation.records.each(&block)
|
57
58
|
end
|
58
59
|
end
|
@@ -77,13 +78,26 @@ module ActiveRecord
|
|
77
78
|
end
|
78
79
|
end
|
79
80
|
|
80
|
-
#
|
81
|
+
# Touches records in batches. Returns the total number of rows affected.
|
82
|
+
#
|
83
|
+
# Person.in_batches.touch_all
|
84
|
+
#
|
85
|
+
# See Relation#touch_all for details of how each batch is touched.
|
86
|
+
def touch_all(...)
|
87
|
+
sum do |relation|
|
88
|
+
relation.touch_all(...)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
# Destroys records in batches. Returns the total number of rows affected.
|
81
93
|
#
|
82
94
|
# Person.where("age < 10").in_batches.destroy_all
|
83
95
|
#
|
84
96
|
# See Relation#destroy_all for details of how each batch is destroyed.
|
85
97
|
def destroy_all
|
86
|
-
|
98
|
+
sum do |relation|
|
99
|
+
relation.destroy_all.count(&:destroyed?)
|
100
|
+
end
|
87
101
|
end
|
88
102
|
|
89
103
|
# Yields an ActiveRecord::Relation object for each batch of records.
|
@@ -92,7 +106,7 @@ module ActiveRecord
|
|
92
106
|
# relation.update_all(awesome: true)
|
93
107
|
# end
|
94
108
|
def each(&block)
|
95
|
-
enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, order: @order, use_ranges: @use_ranges)
|
109
|
+
enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, cursor: @cursor, order: @order, use_ranges: @use_ranges)
|
96
110
|
return enum.each(&block) if block_given?
|
97
111
|
enum
|
98
112
|
end
|
@@ -5,7 +5,7 @@ require "active_record/relation/batches/batch_enumerator"
|
|
5
5
|
module ActiveRecord
|
6
6
|
# = Active Record \Batches
|
7
7
|
module Batches
|
8
|
-
ORDER_IGNORE_MESSAGE = "Scoped order is ignored,
|
8
|
+
ORDER_IGNORE_MESSAGE = "Scoped order is ignored, use :cursor with :order to configure custom order."
|
9
9
|
DEFAULT_ORDER = :asc
|
10
10
|
|
11
11
|
# Looping through a collection of records from the database
|
@@ -35,11 +35,13 @@ module ActiveRecord
|
|
35
35
|
#
|
36
36
|
# ==== Options
|
37
37
|
# * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
|
38
|
-
# * <tt>:start</tt> - Specifies the
|
39
|
-
# * <tt>:finish</tt> - Specifies the
|
38
|
+
# * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
|
39
|
+
# * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
|
40
40
|
# * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
|
41
41
|
# an order is present in the relation.
|
42
|
-
# * <tt>:
|
42
|
+
# * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
|
43
|
+
# of column names). Defaults to primary key.
|
44
|
+
# * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
|
43
45
|
# of :asc or :desc). Defaults to +:asc+.
|
44
46
|
#
|
45
47
|
# class Order < ActiveRecord::Base
|
@@ -71,20 +73,25 @@ module ActiveRecord
|
|
71
73
|
#
|
72
74
|
# NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
|
73
75
|
# ascending on the primary key ("id ASC").
|
74
|
-
# This also means that this method only works when the
|
76
|
+
# This also means that this method only works when the cursor column is
|
75
77
|
# orderable (e.g. an integer or string).
|
76
78
|
#
|
79
|
+
# NOTE: When using custom columns for batching, they should include at least one unique column
|
80
|
+
# (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
|
81
|
+
# all columns should be static (unchangeable after it was set).
|
82
|
+
#
|
77
83
|
# NOTE: By its nature, batch processing is subject to race conditions if
|
78
84
|
# other processes are modifying the database.
|
79
|
-
def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER, &block)
|
85
|
+
def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER, &block)
|
80
86
|
if block_given?
|
81
|
-
find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do |records|
|
87
|
+
find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do |records|
|
82
88
|
records.each(&block)
|
83
89
|
end
|
84
90
|
else
|
85
|
-
enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
|
91
|
+
enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do
|
86
92
|
relation = self
|
87
|
-
|
93
|
+
cursor = Array(cursor)
|
94
|
+
apply_limits(relation, cursor, start, finish, build_batch_orders(cursor, order)).size
|
88
95
|
end
|
89
96
|
end
|
90
97
|
end
|
@@ -109,11 +116,13 @@ module ActiveRecord
|
|
109
116
|
#
|
110
117
|
# ==== Options
|
111
118
|
# * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
|
112
|
-
# * <tt>:start</tt> - Specifies the
|
113
|
-
# * <tt>:finish</tt> - Specifies the
|
119
|
+
# * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
|
120
|
+
# * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
|
114
121
|
# * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
|
115
122
|
# an order is present in the relation.
|
116
|
-
# * <tt>:
|
123
|
+
# * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
|
124
|
+
# of column names). Defaults to primary key.
|
125
|
+
# * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
|
117
126
|
# of :asc or :desc). Defaults to +:asc+.
|
118
127
|
#
|
119
128
|
# class Order < ActiveRecord::Base
|
@@ -140,21 +149,26 @@ module ActiveRecord
|
|
140
149
|
#
|
141
150
|
# NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
|
142
151
|
# ascending on the primary key ("id ASC").
|
143
|
-
# This also means that this method only works when the
|
152
|
+
# This also means that this method only works when the cursor column is
|
144
153
|
# orderable (e.g. an integer or string).
|
145
154
|
#
|
155
|
+
# NOTE: When using custom columns for batching, they should include at least one unique column
|
156
|
+
# (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
|
157
|
+
# all columns should be static (unchangeable after it was set).
|
158
|
+
#
|
146
159
|
# NOTE: By its nature, batch processing is subject to race conditions if
|
147
160
|
# other processes are modifying the database.
|
148
|
-
def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER)
|
161
|
+
def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER)
|
149
162
|
relation = self
|
150
163
|
unless block_given?
|
151
|
-
return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
|
152
|
-
|
164
|
+
return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do
|
165
|
+
cursor = Array(cursor)
|
166
|
+
total = apply_limits(relation, cursor, start, finish, build_batch_orders(cursor, order)).size
|
153
167
|
(total - 1).div(batch_size) + 1
|
154
168
|
end
|
155
169
|
end
|
156
170
|
|
157
|
-
in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, order: order) do |batch|
|
171
|
+
in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, cursor: cursor, order: order) do |batch|
|
158
172
|
yield batch.to_a
|
159
173
|
end
|
160
174
|
end
|
@@ -183,11 +197,13 @@ module ActiveRecord
|
|
183
197
|
# ==== Options
|
184
198
|
# * <tt>:of</tt> - Specifies the size of the batch. Defaults to 1000.
|
185
199
|
# * <tt>:load</tt> - Specifies if the relation should be loaded. Defaults to false.
|
186
|
-
# * <tt>:start</tt> - Specifies the
|
187
|
-
# * <tt>:finish</tt> - Specifies the
|
200
|
+
# * <tt>:start</tt> - Specifies the cursor column value to start from, inclusive of the value.
|
201
|
+
# * <tt>:finish</tt> - Specifies the cursor column value to end at, inclusive of the value.
|
188
202
|
# * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
|
189
203
|
# an order is present in the relation.
|
190
|
-
# * <tt>:
|
204
|
+
# * <tt>:cursor</tt> - Specifies the column to use for batching (can be a column name or an array
|
205
|
+
# of column names). Defaults to primary key.
|
206
|
+
# * <tt>:order</tt> - Specifies the cursor column order (can be +:asc+ or +:desc+ or an array consisting
|
191
207
|
# of :asc or :desc). Defaults to +:asc+.
|
192
208
|
#
|
193
209
|
# class Order < ActiveRecord::Base
|
@@ -231,24 +247,27 @@ module ActiveRecord
|
|
231
247
|
#
|
232
248
|
# NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
|
233
249
|
# ascending on the primary key ("id ASC").
|
234
|
-
# This also means that this method only works when the
|
250
|
+
# This also means that this method only works when the cursor column is
|
235
251
|
# orderable (e.g. an integer or string).
|
236
252
|
#
|
253
|
+
# NOTE: When using custom columns for batching, they should include at least one unique column
|
254
|
+
# (e.g. primary key) as a tiebreaker. Also, to reduce the likelihood of race conditions,
|
255
|
+
# all columns should be static (unchangeable after it was set).
|
256
|
+
#
|
237
257
|
# NOTE: By its nature, batch processing is subject to race conditions if
|
238
258
|
# other processes are modifying the database.
|
239
|
-
def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, order: DEFAULT_ORDER, use_ranges: nil, &block)
|
240
|
-
|
241
|
-
|
242
|
-
end
|
243
|
-
|
244
|
-
unless block
|
245
|
-
return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, order: order, use_ranges: use_ranges)
|
246
|
-
end
|
259
|
+
def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, cursor: primary_key, order: DEFAULT_ORDER, use_ranges: nil, &block)
|
260
|
+
cursor = Array(cursor).map(&:to_s)
|
261
|
+
ensure_valid_options_for_batching!(cursor, start, finish, order)
|
247
262
|
|
248
263
|
if arel.orders.present?
|
249
264
|
act_on_ignored_order(error_on_ignore)
|
250
265
|
end
|
251
266
|
|
267
|
+
unless block
|
268
|
+
return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, cursor: cursor, order: order, use_ranges: use_ranges)
|
269
|
+
end
|
270
|
+
|
252
271
|
batch_limit = of
|
253
272
|
|
254
273
|
if limit_value
|
@@ -261,6 +280,7 @@ module ActiveRecord
|
|
261
280
|
relation: self,
|
262
281
|
start: start,
|
263
282
|
finish: finish,
|
283
|
+
cursor: cursor,
|
264
284
|
order: order,
|
265
285
|
batch_limit: batch_limit,
|
266
286
|
&block
|
@@ -271,6 +291,7 @@ module ActiveRecord
|
|
271
291
|
start: start,
|
272
292
|
finish: finish,
|
273
293
|
load: load,
|
294
|
+
cursor: cursor,
|
274
295
|
order: order,
|
275
296
|
use_ranges: use_ranges,
|
276
297
|
remaining: remaining,
|
@@ -281,28 +302,51 @@ module ActiveRecord
|
|
281
302
|
end
|
282
303
|
|
283
304
|
private
|
284
|
-
def
|
285
|
-
|
286
|
-
|
305
|
+
def ensure_valid_options_for_batching!(cursor, start, finish, order)
|
306
|
+
if start && Array(start).size != cursor.size
|
307
|
+
raise ArgumentError, ":start must contain one value per cursor column"
|
308
|
+
end
|
309
|
+
|
310
|
+
if finish && Array(finish).size != cursor.size
|
311
|
+
raise ArgumentError, ":finish must contain one value per cursor column"
|
312
|
+
end
|
313
|
+
|
314
|
+
if (Array(primary_key) - cursor).any?
|
315
|
+
indexes = model.schema_cache.indexes(table_name)
|
316
|
+
unique_index = indexes.find { |index| index.unique && index.where.nil? && (Array(index.columns) - cursor).empty? }
|
317
|
+
|
318
|
+
unless unique_index
|
319
|
+
raise ArgumentError, ":cursor must include a primary key or other unique column(s)"
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
if (Array(order) - [:asc, :desc]).any?
|
324
|
+
raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
|
325
|
+
end
|
326
|
+
end
|
327
|
+
|
328
|
+
def apply_limits(relation, cursor, start, finish, batch_orders)
|
329
|
+
relation = apply_start_limit(relation, cursor, start, batch_orders) if start
|
330
|
+
relation = apply_finish_limit(relation, cursor, finish, batch_orders) if finish
|
287
331
|
relation
|
288
332
|
end
|
289
333
|
|
290
|
-
def apply_start_limit(relation, start, batch_orders)
|
334
|
+
def apply_start_limit(relation, cursor, start, batch_orders)
|
291
335
|
operators = batch_orders.map do |_column, order|
|
292
336
|
order == :desc ? :lteq : :gteq
|
293
337
|
end
|
294
|
-
batch_condition(relation,
|
338
|
+
batch_condition(relation, cursor, start, operators)
|
295
339
|
end
|
296
340
|
|
297
|
-
def apply_finish_limit(relation, finish, batch_orders)
|
341
|
+
def apply_finish_limit(relation, cursor, finish, batch_orders)
|
298
342
|
operators = batch_orders.map do |_column, order|
|
299
343
|
order == :desc ? :gteq : :lteq
|
300
344
|
end
|
301
|
-
batch_condition(relation,
|
345
|
+
batch_condition(relation, cursor, finish, operators)
|
302
346
|
end
|
303
347
|
|
304
|
-
def batch_condition(relation,
|
305
|
-
cursor_positions =
|
348
|
+
def batch_condition(relation, cursor, values, operators)
|
349
|
+
cursor_positions = cursor.zip(Array(values), operators)
|
306
350
|
|
307
351
|
first_clause_column, first_clause_value, operator = cursor_positions.pop
|
308
352
|
where_clause = predicate_builder[first_clause_column, first_clause_value, operator]
|
@@ -316,9 +360,9 @@ module ActiveRecord
|
|
316
360
|
relation.where(where_clause)
|
317
361
|
end
|
318
362
|
|
319
|
-
def build_batch_orders(order)
|
320
|
-
|
321
|
-
[column,
|
363
|
+
def build_batch_orders(cursor, order)
|
364
|
+
cursor.zip(Array(order)).map do |column, order_|
|
365
|
+
[column, order_ || DEFAULT_ORDER]
|
322
366
|
end
|
323
367
|
end
|
324
368
|
|
@@ -327,33 +371,33 @@ module ActiveRecord
|
|
327
371
|
|
328
372
|
if raise_error
|
329
373
|
raise ArgumentError.new(ORDER_IGNORE_MESSAGE)
|
330
|
-
elsif logger
|
331
|
-
logger.warn(ORDER_IGNORE_MESSAGE)
|
374
|
+
elsif model.logger
|
375
|
+
model.logger.warn(ORDER_IGNORE_MESSAGE)
|
332
376
|
end
|
333
377
|
end
|
334
378
|
|
335
|
-
def
|
336
|
-
Array(primary_key).zip(Array(order))
|
337
|
-
end
|
338
|
-
|
339
|
-
def batch_on_loaded_relation(relation:, start:, finish:, order:, batch_limit:)
|
379
|
+
def batch_on_loaded_relation(relation:, start:, finish:, cursor:, order:, batch_limit:)
|
340
380
|
records = relation.to_a
|
381
|
+
order = build_batch_orders(cursor, order).map(&:second)
|
341
382
|
|
342
383
|
if start || finish
|
343
384
|
records = records.filter do |record|
|
344
|
-
|
385
|
+
values = record_cursor_values(record, cursor)
|
386
|
+
|
387
|
+
(start.nil? || compare_values_for_order(values, Array(start), order) >= 0) &&
|
388
|
+
(finish.nil? || compare_values_for_order(values, Array(finish), order) <= 0)
|
345
389
|
end
|
346
390
|
end
|
347
391
|
|
348
|
-
records
|
349
|
-
|
350
|
-
|
351
|
-
|
392
|
+
records.sort! do |record1, record2|
|
393
|
+
values1 = record_cursor_values(record1, cursor)
|
394
|
+
values2 = record_cursor_values(record2, cursor)
|
395
|
+
compare_values_for_order(values1, values2, order)
|
352
396
|
end
|
353
397
|
|
354
|
-
|
398
|
+
records.each_slice(batch_limit) do |subrecords|
|
355
399
|
subrelation = relation.spawn
|
356
|
-
subrelation.load_records(
|
400
|
+
subrelation.load_records(subrecords)
|
357
401
|
|
358
402
|
yield subrelation
|
359
403
|
end
|
@@ -361,44 +405,65 @@ module ActiveRecord
|
|
361
405
|
nil
|
362
406
|
end
|
363
407
|
|
364
|
-
def
|
365
|
-
|
408
|
+
def record_cursor_values(record, cursor)
|
409
|
+
record.attributes.slice(*cursor).values
|
410
|
+
end
|
411
|
+
|
412
|
+
# This is a custom implementation of `<=>` operator,
|
413
|
+
# which also takes into account how the collection will be ordered.
|
414
|
+
def compare_values_for_order(values1, values2, order)
|
415
|
+
values1.each_with_index do |element1, index|
|
416
|
+
element2 = values2[index]
|
417
|
+
direction = order[index]
|
418
|
+
comparison = element1 <=> element2
|
419
|
+
comparison = -comparison if direction == :desc
|
420
|
+
return comparison if comparison != 0
|
421
|
+
end
|
422
|
+
|
423
|
+
0
|
424
|
+
end
|
425
|
+
|
426
|
+
def batch_on_unloaded_relation(relation:, start:, finish:, load:, cursor:, order:, use_ranges:, remaining:, batch_limit:)
|
427
|
+
batch_orders = build_batch_orders(cursor, order)
|
366
428
|
relation = relation.reorder(batch_orders.to_h).limit(batch_limit)
|
367
|
-
relation = apply_limits(relation, start, finish, batch_orders)
|
429
|
+
relation = apply_limits(relation, cursor, start, finish, batch_orders)
|
368
430
|
relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
|
369
431
|
batch_relation = relation
|
370
|
-
empty_scope = to_sql ==
|
432
|
+
empty_scope = to_sql == model.unscoped.all.to_sql
|
371
433
|
|
372
434
|
loop do
|
373
435
|
if load
|
374
436
|
records = batch_relation.records
|
375
|
-
|
376
|
-
yielded_relation = where(
|
437
|
+
values = records.pluck(*cursor)
|
438
|
+
yielded_relation = where(cursor => values)
|
377
439
|
yielded_relation.load_records(records)
|
378
440
|
elsif (empty_scope && use_ranges != false) || use_ranges
|
379
|
-
|
380
|
-
|
441
|
+
values = batch_relation.pluck(*cursor)
|
442
|
+
|
443
|
+
finish = values.last
|
381
444
|
if finish
|
382
|
-
yielded_relation = apply_finish_limit(batch_relation, finish, batch_orders)
|
445
|
+
yielded_relation = apply_finish_limit(batch_relation, cursor, finish, batch_orders)
|
383
446
|
yielded_relation = yielded_relation.except(:limit, :order)
|
384
447
|
yielded_relation.skip_query_cache!(false)
|
385
448
|
end
|
386
449
|
else
|
387
|
-
|
388
|
-
yielded_relation = where(
|
450
|
+
values = batch_relation.pluck(*cursor)
|
451
|
+
yielded_relation = where(cursor => values)
|
389
452
|
end
|
390
453
|
|
391
|
-
break if
|
454
|
+
break if values.empty?
|
392
455
|
|
393
|
-
|
394
|
-
|
456
|
+
if values.flatten.any?(nil)
|
457
|
+
raise ArgumentError, "Not all of the batch cursor columns were included in the custom select clause "\
|
458
|
+
"or some columns contain nil."
|
459
|
+
end
|
395
460
|
|
396
461
|
yield yielded_relation
|
397
462
|
|
398
|
-
break if
|
463
|
+
break if values.length < batch_limit
|
399
464
|
|
400
465
|
if limit_value
|
401
|
-
remaining -=
|
466
|
+
remaining -= values.length
|
402
467
|
|
403
468
|
if remaining == 0
|
404
469
|
# Saves a useless iteration when the limit is a multiple of the
|
@@ -416,7 +481,8 @@ module ActiveRecord
|
|
416
481
|
end
|
417
482
|
operators << (last_order == :desc ? :lt : :gt)
|
418
483
|
|
419
|
-
|
484
|
+
cursor_value = values.last
|
485
|
+
batch_relation = batch_condition(relation, cursor, cursor_value, operators)
|
420
486
|
end
|
421
487
|
|
422
488
|
nil
|