activerecord 4.2.11.3 → 5.0.0.beta1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of activerecord might be problematic. Click here for more details.

Files changed (229) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +1029 -1349
  3. data/MIT-LICENSE +1 -1
  4. data/README.rdoc +6 -7
  5. data/examples/performance.rb +2 -2
  6. data/lib/active_record.rb +7 -3
  7. data/lib/active_record/aggregations.rb +35 -25
  8. data/lib/active_record/association_relation.rb +2 -2
  9. data/lib/active_record/associations.rb +305 -204
  10. data/lib/active_record/associations/alias_tracker.rb +19 -16
  11. data/lib/active_record/associations/association.rb +10 -8
  12. data/lib/active_record/associations/association_scope.rb +73 -102
  13. data/lib/active_record/associations/belongs_to_association.rb +20 -32
  14. data/lib/active_record/associations/builder/association.rb +28 -34
  15. data/lib/active_record/associations/builder/belongs_to.rb +41 -18
  16. data/lib/active_record/associations/builder/collection_association.rb +8 -24
  17. data/lib/active_record/associations/builder/has_and_belongs_to_many.rb +11 -11
  18. data/lib/active_record/associations/builder/has_many.rb +4 -4
  19. data/lib/active_record/associations/builder/has_one.rb +10 -5
  20. data/lib/active_record/associations/builder/singular_association.rb +2 -9
  21. data/lib/active_record/associations/collection_association.rb +40 -43
  22. data/lib/active_record/associations/collection_proxy.rb +55 -29
  23. data/lib/active_record/associations/foreign_association.rb +1 -1
  24. data/lib/active_record/associations/has_many_association.rb +20 -71
  25. data/lib/active_record/associations/has_many_through_association.rb +8 -52
  26. data/lib/active_record/associations/has_one_association.rb +12 -5
  27. data/lib/active_record/associations/join_dependency.rb +28 -18
  28. data/lib/active_record/associations/join_dependency/join_association.rb +13 -12
  29. data/lib/active_record/associations/preloader.rb +13 -4
  30. data/lib/active_record/associations/preloader/association.rb +45 -51
  31. data/lib/active_record/associations/preloader/collection_association.rb +0 -6
  32. data/lib/active_record/associations/preloader/has_many_through.rb +1 -1
  33. data/lib/active_record/associations/preloader/has_one.rb +0 -8
  34. data/lib/active_record/associations/preloader/through_association.rb +5 -4
  35. data/lib/active_record/associations/singular_association.rb +6 -0
  36. data/lib/active_record/associations/through_association.rb +11 -3
  37. data/lib/active_record/attribute.rb +61 -17
  38. data/lib/active_record/attribute/user_provided_default.rb +23 -0
  39. data/lib/active_record/attribute_assignment.rb +27 -140
  40. data/lib/active_record/attribute_decorators.rb +6 -5
  41. data/lib/active_record/attribute_methods.rb +79 -26
  42. data/lib/active_record/attribute_methods/before_type_cast.rb +1 -1
  43. data/lib/active_record/attribute_methods/dirty.rb +46 -86
  44. data/lib/active_record/attribute_methods/primary_key.rb +2 -2
  45. data/lib/active_record/attribute_methods/query.rb +2 -2
  46. data/lib/active_record/attribute_methods/read.rb +26 -42
  47. data/lib/active_record/attribute_methods/serialization.rb +13 -16
  48. data/lib/active_record/attribute_methods/time_zone_conversion.rb +42 -9
  49. data/lib/active_record/attribute_methods/write.rb +13 -24
  50. data/lib/active_record/attribute_mutation_tracker.rb +70 -0
  51. data/lib/active_record/attribute_set.rb +30 -3
  52. data/lib/active_record/attribute_set/builder.rb +6 -4
  53. data/lib/active_record/attributes.rb +194 -81
  54. data/lib/active_record/autosave_association.rb +33 -15
  55. data/lib/active_record/base.rb +30 -18
  56. data/lib/active_record/callbacks.rb +36 -40
  57. data/lib/active_record/coders/yaml_column.rb +20 -8
  58. data/lib/active_record/collection_cache_key.rb +31 -0
  59. data/lib/active_record/connection_adapters/abstract/connection_pool.rb +431 -122
  60. data/lib/active_record/connection_adapters/abstract/database_limits.rb +3 -3
  61. data/lib/active_record/connection_adapters/abstract/database_statements.rb +40 -22
  62. data/lib/active_record/connection_adapters/abstract/quoting.rb +62 -8
  63. data/lib/active_record/connection_adapters/abstract/schema_creation.rb +46 -38
  64. data/lib/active_record/connection_adapters/abstract/schema_definitions.rb +229 -185
  65. data/lib/active_record/connection_adapters/abstract/schema_dumper.rb +52 -13
  66. data/lib/active_record/connection_adapters/abstract/schema_statements.rb +275 -115
  67. data/lib/active_record/connection_adapters/abstract/transaction.rb +32 -33
  68. data/lib/active_record/connection_adapters/abstract_adapter.rb +83 -32
  69. data/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +384 -221
  70. data/lib/active_record/connection_adapters/column.rb +27 -41
  71. data/lib/active_record/connection_adapters/connection_specification.rb +2 -21
  72. data/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb +22 -0
  73. data/lib/active_record/connection_adapters/mysql/schema_creation.rb +57 -0
  74. data/lib/active_record/connection_adapters/mysql/schema_definitions.rb +69 -0
  75. data/lib/active_record/connection_adapters/mysql/schema_dumper.rb +59 -0
  76. data/lib/active_record/connection_adapters/mysql2_adapter.rb +22 -101
  77. data/lib/active_record/connection_adapters/postgresql/column.rb +6 -10
  78. data/lib/active_record/connection_adapters/postgresql/database_statements.rb +3 -3
  79. data/lib/active_record/connection_adapters/postgresql/oid.rb +1 -6
  80. data/lib/active_record/connection_adapters/postgresql/oid/array.rb +23 -57
  81. data/lib/active_record/connection_adapters/postgresql/oid/bit.rb +2 -2
  82. data/lib/active_record/connection_adapters/postgresql/oid/bytea.rb +1 -1
  83. data/lib/active_record/connection_adapters/postgresql/oid/cidr.rb +1 -1
  84. data/lib/active_record/connection_adapters/postgresql/oid/date_time.rb +7 -22
  85. data/lib/active_record/connection_adapters/postgresql/oid/hstore.rb +3 -3
  86. data/lib/active_record/connection_adapters/postgresql/oid/json.rb +1 -26
  87. data/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb +2 -2
  88. data/lib/active_record/connection_adapters/postgresql/oid/money.rb +0 -2
  89. data/lib/active_record/connection_adapters/postgresql/oid/point.rb +4 -4
  90. data/lib/active_record/connection_adapters/postgresql/oid/rails_5_1_point.rb +50 -0
  91. data/lib/active_record/connection_adapters/postgresql/oid/range.rb +23 -16
  92. data/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb +0 -4
  93. data/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +2 -2
  94. data/lib/active_record/connection_adapters/postgresql/oid/vector.rb +1 -1
  95. data/lib/active_record/connection_adapters/postgresql/oid/xml.rb +1 -1
  96. data/lib/active_record/connection_adapters/postgresql/quoting.rb +18 -11
  97. data/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +29 -10
  98. data/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +107 -79
  99. data/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +54 -0
  100. data/lib/active_record/connection_adapters/postgresql/schema_statements.rb +174 -128
  101. data/lib/active_record/connection_adapters/postgresql/type_metadata.rb +35 -0
  102. data/lib/active_record/connection_adapters/postgresql_adapter.rb +184 -112
  103. data/lib/active_record/connection_adapters/schema_cache.rb +36 -23
  104. data/lib/active_record/connection_adapters/sql_type_metadata.rb +32 -0
  105. data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +15 -0
  106. data/lib/active_record/connection_adapters/sqlite3_adapter.rb +134 -110
  107. data/lib/active_record/connection_adapters/statement_pool.rb +28 -11
  108. data/lib/active_record/connection_handling.rb +5 -5
  109. data/lib/active_record/core.rb +72 -104
  110. data/lib/active_record/counter_cache.rb +9 -20
  111. data/lib/active_record/dynamic_matchers.rb +1 -20
  112. data/lib/active_record/enum.rb +110 -76
  113. data/lib/active_record/errors.rb +72 -47
  114. data/lib/active_record/explain_registry.rb +1 -1
  115. data/lib/active_record/explain_subscriber.rb +1 -1
  116. data/lib/active_record/fixture_set/file.rb +19 -4
  117. data/lib/active_record/fixtures.rb +76 -40
  118. data/lib/active_record/gem_version.rb +4 -4
  119. data/lib/active_record/inheritance.rb +27 -40
  120. data/lib/active_record/integration.rb +4 -4
  121. data/lib/active_record/legacy_yaml_adapter.rb +18 -2
  122. data/lib/active_record/locale/en.yml +3 -2
  123. data/lib/active_record/locking/optimistic.rb +10 -14
  124. data/lib/active_record/locking/pessimistic.rb +1 -1
  125. data/lib/active_record/log_subscriber.rb +40 -22
  126. data/lib/active_record/migration.rb +304 -133
  127. data/lib/active_record/migration/command_recorder.rb +59 -18
  128. data/lib/active_record/migration/compatibility.rb +90 -0
  129. data/lib/active_record/model_schema.rb +92 -40
  130. data/lib/active_record/nested_attributes.rb +45 -34
  131. data/lib/active_record/null_relation.rb +15 -7
  132. data/lib/active_record/persistence.rb +112 -72
  133. data/lib/active_record/querying.rb +6 -5
  134. data/lib/active_record/railtie.rb +20 -13
  135. data/lib/active_record/railties/controller_runtime.rb +1 -1
  136. data/lib/active_record/railties/databases.rake +47 -38
  137. data/lib/active_record/readonly_attributes.rb +1 -1
  138. data/lib/active_record/reflection.rb +182 -57
  139. data/lib/active_record/relation.rb +152 -100
  140. data/lib/active_record/relation/batches.rb +133 -33
  141. data/lib/active_record/relation/batches/batch_enumerator.rb +67 -0
  142. data/lib/active_record/relation/calculations.rb +80 -101
  143. data/lib/active_record/relation/delegation.rb +6 -19
  144. data/lib/active_record/relation/finder_methods.rb +58 -46
  145. data/lib/active_record/relation/from_clause.rb +32 -0
  146. data/lib/active_record/relation/merger.rb +13 -42
  147. data/lib/active_record/relation/predicate_builder.rb +99 -105
  148. data/lib/active_record/relation/predicate_builder/array_handler.rb +11 -16
  149. data/lib/active_record/relation/predicate_builder/association_query_handler.rb +78 -0
  150. data/lib/active_record/relation/predicate_builder/base_handler.rb +17 -0
  151. data/lib/active_record/relation/predicate_builder/basic_object_handler.rb +17 -0
  152. data/lib/active_record/relation/predicate_builder/class_handler.rb +27 -0
  153. data/lib/active_record/relation/predicate_builder/range_handler.rb +17 -0
  154. data/lib/active_record/relation/query_attribute.rb +19 -0
  155. data/lib/active_record/relation/query_methods.rb +274 -238
  156. data/lib/active_record/relation/record_fetch_warning.rb +51 -0
  157. data/lib/active_record/relation/spawn_methods.rb +3 -6
  158. data/lib/active_record/relation/where_clause.rb +173 -0
  159. data/lib/active_record/relation/where_clause_factory.rb +37 -0
  160. data/lib/active_record/result.rb +4 -3
  161. data/lib/active_record/runtime_registry.rb +1 -1
  162. data/lib/active_record/sanitization.rb +94 -65
  163. data/lib/active_record/schema.rb +23 -22
  164. data/lib/active_record/schema_dumper.rb +33 -22
  165. data/lib/active_record/schema_migration.rb +10 -4
  166. data/lib/active_record/scoping.rb +17 -6
  167. data/lib/active_record/scoping/default.rb +19 -6
  168. data/lib/active_record/scoping/named.rb +39 -28
  169. data/lib/active_record/secure_token.rb +38 -0
  170. data/lib/active_record/serialization.rb +2 -4
  171. data/lib/active_record/statement_cache.rb +15 -13
  172. data/lib/active_record/store.rb +8 -3
  173. data/lib/active_record/suppressor.rb +54 -0
  174. data/lib/active_record/table_metadata.rb +64 -0
  175. data/lib/active_record/tasks/database_tasks.rb +30 -40
  176. data/lib/active_record/tasks/mysql_database_tasks.rb +7 -15
  177. data/lib/active_record/tasks/postgresql_database_tasks.rb +11 -2
  178. data/lib/active_record/tasks/sqlite_database_tasks.rb +5 -1
  179. data/lib/active_record/timestamp.rb +16 -9
  180. data/lib/active_record/touch_later.rb +58 -0
  181. data/lib/active_record/transactions.rb +138 -56
  182. data/lib/active_record/type.rb +66 -17
  183. data/lib/active_record/type/adapter_specific_registry.rb +130 -0
  184. data/lib/active_record/type/date.rb +2 -45
  185. data/lib/active_record/type/date_time.rb +2 -49
  186. data/lib/active_record/type/internal/abstract_json.rb +33 -0
  187. data/lib/active_record/type/internal/timezone.rb +15 -0
  188. data/lib/active_record/type/serialized.rb +9 -14
  189. data/lib/active_record/type/time.rb +3 -21
  190. data/lib/active_record/type/type_map.rb +4 -4
  191. data/lib/active_record/type_caster.rb +7 -0
  192. data/lib/active_record/type_caster/connection.rb +29 -0
  193. data/lib/active_record/type_caster/map.rb +19 -0
  194. data/lib/active_record/validations.rb +33 -32
  195. data/lib/active_record/validations/absence.rb +24 -0
  196. data/lib/active_record/validations/associated.rb +10 -3
  197. data/lib/active_record/validations/length.rb +36 -0
  198. data/lib/active_record/validations/presence.rb +12 -12
  199. data/lib/active_record/validations/uniqueness.rb +24 -21
  200. data/lib/rails/generators/active_record/migration.rb +7 -0
  201. data/lib/rails/generators/active_record/migration/migration_generator.rb +7 -4
  202. data/lib/rails/generators/active_record/migration/templates/create_table_migration.rb +8 -3
  203. data/lib/rails/generators/active_record/migration/templates/migration.rb +4 -1
  204. data/lib/rails/generators/active_record/model/model_generator.rb +21 -15
  205. data/lib/rails/generators/active_record/model/templates/model.rb +3 -0
  206. metadata +50 -35
  207. data/lib/active_record/connection_adapters/mysql_adapter.rb +0 -498
  208. data/lib/active_record/connection_adapters/postgresql/array_parser.rb +0 -93
  209. data/lib/active_record/connection_adapters/postgresql/oid/date.rb +0 -11
  210. data/lib/active_record/connection_adapters/postgresql/oid/float.rb +0 -21
  211. data/lib/active_record/connection_adapters/postgresql/oid/infinity.rb +0 -13
  212. data/lib/active_record/connection_adapters/postgresql/oid/integer.rb +0 -11
  213. data/lib/active_record/connection_adapters/postgresql/oid/time.rb +0 -11
  214. data/lib/active_record/serializers/xml_serializer.rb +0 -193
  215. data/lib/active_record/type/big_integer.rb +0 -13
  216. data/lib/active_record/type/binary.rb +0 -50
  217. data/lib/active_record/type/boolean.rb +0 -31
  218. data/lib/active_record/type/decimal.rb +0 -64
  219. data/lib/active_record/type/decimal_without_scale.rb +0 -11
  220. data/lib/active_record/type/decorator.rb +0 -14
  221. data/lib/active_record/type/float.rb +0 -19
  222. data/lib/active_record/type/integer.rb +0 -59
  223. data/lib/active_record/type/mutable.rb +0 -16
  224. data/lib/active_record/type/numeric.rb +0 -36
  225. data/lib/active_record/type/string.rb +0 -40
  226. data/lib/active_record/type/text.rb +0 -11
  227. data/lib/active_record/type/time_value.rb +0 -38
  228. data/lib/active_record/type/unsigned_integer.rb +0 -15
  229. data/lib/active_record/type/value.rb +0 -110
@@ -1,8 +1,10 @@
1
+ require "active_record/relation/batches/batch_enumerator"
2
+
1
3
  module ActiveRecord
2
4
  module Batches
3
5
  # Looping through a collection of records from the database
4
- # (using the +all+ method, for example) is very inefficient
5
- # since it will try to instantiate all the objects at once.
6
+ # (using the Scoping::Named::ClassMethods.all method, for example)
7
+ # is very inefficient since it will try to instantiate all the objects at once.
6
8
  #
7
9
  # In that case, batch processing methods allow you to work
8
10
  # with the records in batches, thereby greatly reducing memory consumption.
@@ -27,37 +29,46 @@ module ActiveRecord
27
29
  #
28
30
  # ==== Options
29
31
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000.
30
- # * <tt>:start</tt> - Specifies the starting point for the batch processing.
32
+ # * <tt>:begin_at</tt> - Specifies the primary key value to start from, inclusive of the value.
33
+ # * <tt>:end_at</tt> - Specifies the primary key value to end at, inclusive of the value.
31
34
  # This is especially useful if you want multiple workers dealing with
32
35
  # the same processing queue. You can make worker 1 handle all the records
33
36
  # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond
34
- # (by setting the +:start+ option on that worker).
37
+ # (by setting the +:begin_at+ and +:end_at+ option on each worker).
35
38
  #
36
39
  # # Let's process for a batch of 2000 records, skipping the first 2000 rows
37
- # Person.find_each(start: 2000, batch_size: 2000) do |person|
40
+ # Person.find_each(begin_at: 2000, batch_size: 2000) do |person|
38
41
  # person.party_all_night!
39
42
  # end
40
43
  #
41
44
  # NOTE: It's not possible to set the order. That is automatically set to
42
45
  # ascending on the primary key ("id ASC") to make the batch ordering
43
- # work. This also means that this method only works with integer-based
44
- # primary keys.
46
+ # work. This also means that this method only works when the primary key is
47
+ # orderable (e.g. an integer or string).
45
48
  #
46
49
  # NOTE: You can't set the limit either, that's used to control
47
50
  # the batch sizes.
48
- def find_each(options = {})
51
+ def find_each(begin_at: nil, end_at: nil, batch_size: 1000, start: nil)
52
+ if start
53
+ begin_at = start
54
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
55
+ Passing `start` value to find_each is deprecated, and will be removed in Rails 5.1.
56
+ Please pass `begin_at` instead.
57
+ MSG
58
+ end
49
59
  if block_given?
50
- find_in_batches(options) do |records|
60
+ find_in_batches(begin_at: begin_at, end_at: end_at, batch_size: batch_size) do |records|
51
61
  records.each { |record| yield record }
52
62
  end
53
63
  else
54
- enum_for :find_each, options do
55
- options[:start] ? where(table[primary_key].gteq(options[:start])).size : size
64
+ enum_for(:find_each, begin_at: begin_at, end_at: end_at, batch_size: batch_size) do
65
+ relation = self
66
+ apply_limits(relation, begin_at, end_at).size
56
67
  end
57
68
  end
58
69
  end
59
70
 
60
- # Yields each batch of records that was found by the find +options+ as
71
+ # Yields each batch of records that was found by the find options as
61
72
  # an array.
62
73
  #
63
74
  # Person.where("age > 21").find_in_batches do |group|
@@ -77,60 +88,149 @@ module ActiveRecord
77
88
  #
78
89
  # ==== Options
79
90
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000.
80
- # * <tt>:start</tt> - Specifies the starting point for the batch processing.
91
+ # * <tt>:begin_at</tt> - Specifies the primary key value to start from, inclusive of the value.
92
+ # * <tt>:end_at</tt> - Specifies the primary key value to end at, inclusive of the value.
81
93
  # This is especially useful if you want multiple workers dealing with
82
94
  # the same processing queue. You can make worker 1 handle all the records
83
95
  # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond
84
- # (by setting the +:start+ option on that worker).
96
+ # (by setting the +:begin_at+ and +:end_at+ option on each worker).
85
97
  #
86
98
  # # Let's process the next 2000 records
87
- # Person.find_in_batches(start: 2000, batch_size: 2000) do |group|
99
+ # Person.find_in_batches(begin_at: 2000, batch_size: 2000) do |group|
88
100
  # group.each { |person| person.party_all_night! }
89
101
  # end
90
102
  #
91
103
  # NOTE: It's not possible to set the order. That is automatically set to
92
104
  # ascending on the primary key ("id ASC") to make the batch ordering
93
- # work. This also means that this method only works with integer-based
94
- # primary keys.
105
+ # work. This also means that this method only works when the primary key is
106
+ # orderable (e.g. an integer or string).
95
107
  #
96
108
  # NOTE: You can't set the limit either, that's used to control
97
109
  # the batch sizes.
98
- def find_in_batches(options = {})
99
- options.assert_valid_keys(:start, :batch_size)
110
+ def find_in_batches(begin_at: nil, end_at: nil, batch_size: 1000, start: nil)
111
+ if start
112
+ begin_at = start
113
+ ActiveSupport::Deprecation.warn(<<-MSG.squish)
114
+ Passing `start` value to find_in_batches is deprecated, and will be removed in Rails 5.1.
115
+ Please pass `begin_at` instead.
116
+ MSG
117
+ end
100
118
 
101
119
  relation = self
102
- start = options[:start]
103
- batch_size = options[:batch_size] || 1000
104
-
105
120
  unless block_given?
106
- return to_enum(:find_in_batches, options) do
107
- total = start ? where(table[primary_key].gteq(start)).size : size
121
+ return to_enum(:find_in_batches, begin_at: begin_at, end_at: end_at, batch_size: batch_size) do
122
+ total = apply_limits(relation, begin_at, end_at).size
108
123
  (total - 1).div(batch_size) + 1
109
124
  end
110
125
  end
111
126
 
127
+ in_batches(of: batch_size, begin_at: begin_at, end_at: end_at, load: true) do |batch|
128
+ yield batch.to_a
129
+ end
130
+ end
131
+
132
+ # Yields ActiveRecord::Relation objects to work with a batch of records.
133
+ #
134
+ # Person.where("age > 21").in_batches do |relation|
135
+ # relation.delete_all
136
+ # sleep(10) # Throttle the delete queries
137
+ # end
138
+ #
139
+ # If you do not provide a block to #in_batches, it will return a
140
+ # BatchEnumerator which is enumerable.
141
+ #
142
+ # Person.in_batches.with_index do |relation, batch_index|
143
+ # puts "Processing relation ##{batch_index}"
144
+ # relation.each { |relation| relation.delete_all }
145
+ # end
146
+ #
147
+ # Examples of calling methods on the returned BatchEnumerator object:
148
+ #
149
+ # Person.in_batches.delete_all
150
+ # Person.in_batches.update_all(awesome: true)
151
+ # Person.in_batches.each_record(&:party_all_night!)
152
+ #
153
+ # ==== Options
154
+ # * <tt>:of</tt> - Specifies the size of the batch. Default to 1000.
155
+ # * <tt>:load</tt> - Specifies if the relation should be loaded. Default to false.
156
+ # * <tt>:begin_at</tt> - Specifies the primary key value to start from, inclusive of the value.
157
+ # * <tt>:end_at</tt> - Specifies the primary key value to end at, inclusive of the value.
158
+ #
159
+ # This is especially useful if you want to work with the
160
+ # ActiveRecord::Relation object instead of the array of records, or if
161
+ # you want multiple workers dealing with the same processing queue. You can
162
+ # make worker 1 handle all the records between id 0 and 10,000 and worker 2
163
+ # handle from 10,000 and beyond (by setting the +:begin_at+ and +:end_at+
164
+ # option on each worker).
165
+ #
166
+ # # Let's process the next 2000 records
167
+ # Person.in_batches(of: 2000, begin_at: 2000).update_all(awesome: true)
168
+ #
169
+ # An example of calling where query method on the relation:
170
+ #
171
+ # Person.in_batches.each do |relation|
172
+ # relation.update_all('age = age + 1')
173
+ # relation.where('age > 21').update_all(should_party: true)
174
+ # relation.where('age <= 21').delete_all
175
+ # end
176
+ #
177
+ # NOTE: If you are going to iterate through each record, you should call
178
+ # #each_record on the yielded BatchEnumerator:
179
+ #
180
+ # Person.in_batches.each_record(&:party_all_night!)
181
+ #
182
+ # NOTE: It's not possible to set the order. That is automatically set to
183
+ # ascending on the primary key ("id ASC") to make the batch ordering
184
+ # consistent. Therefore the primary key must be orderable, e.g an integer
185
+ # or a string.
186
+ #
187
+ # NOTE: You can't set the limit either, that's used to control the batch
188
+ # sizes.
189
+ def in_batches(of: 1000, begin_at: nil, end_at: nil, load: false)
190
+ relation = self
191
+ unless block_given?
192
+ return BatchEnumerator.new(of: of, begin_at: begin_at, end_at: end_at, relation: self)
193
+ end
194
+
112
195
  if logger && (arel.orders.present? || arel.taken.present?)
113
196
  logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
114
197
  end
115
198
 
116
- relation = relation.reorder(batch_order).limit(batch_size)
117
- records = start ? relation.where(table[primary_key].gteq(start)).to_a : relation.to_a
199
+ relation = relation.reorder(batch_order).limit(of)
200
+ relation = apply_limits(relation, begin_at, end_at)
201
+ batch_relation = relation
118
202
 
119
- while records.any?
120
- records_size = records.size
121
- primary_key_offset = records.last.id
122
- raise "Primary key not included in the custom select clause" unless primary_key_offset
203
+ loop do
204
+ if load
205
+ records = batch_relation.to_a
206
+ ids = records.map(&:id)
207
+ yielded_relation = self.where(primary_key => ids)
208
+ yielded_relation.load_records(records)
209
+ else
210
+ ids = batch_relation.pluck(primary_key)
211
+ yielded_relation = self.where(primary_key => ids)
212
+ end
213
+
214
+ break if ids.empty?
123
215
 
124
- yield records
216
+ primary_key_offset = ids.last
217
+ raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
125
218
 
126
- break if records_size < batch_size
219
+ yield yielded_relation
127
220
 
128
- records = relation.where(table[primary_key].gt(primary_key_offset)).to_a
221
+ break if ids.length < of
222
+ batch_relation = relation.where(table[primary_key].gt(primary_key_offset))
129
223
  end
130
224
  end
131
225
 
132
226
  private
133
227
 
228
+ def apply_limits(relation, begin_at, end_at)
229
+ relation = relation.where(table[primary_key].gteq(begin_at)) if begin_at
230
+ relation = relation.where(table[primary_key].lteq(end_at)) if end_at
231
+ relation
232
+ end
233
+
134
234
  def batch_order
135
235
  "#{quoted_table_name}.#{quoted_primary_key} ASC"
136
236
  end
@@ -0,0 +1,67 @@
1
+ module ActiveRecord
2
+ module Batches
3
+ class BatchEnumerator
4
+ include Enumerable
5
+
6
+ def initialize(of: 1000, begin_at: nil, end_at: nil, relation:) #:nodoc:
7
+ @of = of
8
+ @relation = relation
9
+ @begin_at = begin_at
10
+ @end_at = end_at
11
+ end
12
+
13
+ # Looping through a collection of records from the database (using the
14
+ # +all+ method, for example) is very inefficient since it will try to
15
+ # instantiate all the objects at once.
16
+ #
17
+ # In that case, batch processing methods allow you to work with the
18
+ # records in batches, thereby greatly reducing memory consumption.
19
+ #
20
+ # Person.in_batches.each_record do |person|
21
+ # person.do_awesome_stuff
22
+ # end
23
+ #
24
+ # Person.where("age > 21").in_batches(of: 10).each_record do |person|
25
+ # person.party_all_night!
26
+ # end
27
+ #
28
+ # If you do not provide a block to #each_record, it will return an Enumerator
29
+ # for chaining with other methods:
30
+ #
31
+ # Person.in_batches.each_record.with_index do |person, index|
32
+ # person.award_trophy(index + 1)
33
+ # end
34
+ def each_record
35
+ return to_enum(:each_record) unless block_given?
36
+
37
+ @relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: true).each do |relation|
38
+ relation.to_a.each { |record| yield record }
39
+ end
40
+ end
41
+
42
+ # Delegates #delete_all, #update_all, #destroy_all methods to each batch.
43
+ #
44
+ # People.in_batches.delete_all
45
+ # People.in_batches.destroy_all('age < 10')
46
+ # People.in_batches.update_all('age = age + 1')
47
+ [:delete_all, :update_all, :destroy_all].each do |method|
48
+ define_method(method) do |*args, &block|
49
+ @relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: false).each do |relation|
50
+ relation.send(method, *args, &block)
51
+ end
52
+ end
53
+ end
54
+
55
+ # Yields an ActiveRecord::Relation object for each batch of records.
56
+ #
57
+ # Person.in_batches.each do |relation|
58
+ # relation.update_all(awesome: true)
59
+ # end
60
+ def each
61
+ enum = @relation.to_enum(:in_batches, of: @of, begin_at: @begin_at, end_at: @end_at, load: false)
62
+ return enum.each { |relation| yield relation } if block_given?
63
+ enum
64
+ end
65
+ end
66
+ end
67
+ end
@@ -14,127 +14,112 @@ module ActiveRecord
14
14
  # Person.distinct.count(:age)
15
15
  # # => counts the number of different age values
16
16
  #
17
- # If +count+ is used with +group+, it returns a Hash whose keys represent the aggregated column,
17
+ # If #count is used with {Relation#group}[rdoc-ref:QueryMethods#group],
18
+ # it returns a Hash whose keys represent the aggregated column,
18
19
  # and the values are the respective amounts:
19
20
  #
20
21
  # Person.group(:city).count
21
22
  # # => { 'Rome' => 5, 'Paris' => 3 }
22
23
  #
23
- # If +count+ is used with +group+ for multiple columns, it returns a Hash whose
24
+ # If #count is used with {Relation#group}[rdoc-ref:QueryMethods#group] for multiple columns, it returns a Hash whose
24
25
  # keys are an array containing the individual values of each column and the value
25
- # of each key would be the +count+.
26
+ # of each key would be the #count.
26
27
  #
27
28
  # Article.group(:status, :category).count
28
29
  # # => {["draft", "business"]=>10, ["draft", "technology"]=>4,
29
30
  # ["published", "business"]=>0, ["published", "technology"]=>2}
30
31
  #
31
- # If +count+ is used with +select+, it will count the selected columns:
32
+ # If #count is used with {Relation#select}[rdoc-ref:QueryMethods#select], it will count the selected columns:
32
33
  #
33
34
  # Person.select(:age).count
34
35
  # # => counts the number of different age values
35
36
  #
36
- # Note: not all valid +select+ expressions are valid +count+ expressions. The specifics differ
37
+ # Note: not all valid {Relation#select}[rdoc-ref:QueryMethods#select] expressions are valid #count expressions. The specifics differ
37
38
  # between databases. In invalid cases, an error from the database is thrown.
38
- def count(column_name = nil, options = {})
39
- if options.present? && !ActiveRecord.const_defined?(:DeprecatedFinders)
40
- raise ArgumentError, "Relation#count does not support finder options anymore. " \
41
- "Please build a scope and then call count on it or use the " \
42
- "activerecord-deprecated_finders gem to enable this functionality."
43
-
44
- end
45
-
46
- # TODO: Remove options argument as soon we remove support to
47
- # activerecord-deprecated_finders.
48
- calculate(:count, column_name, options)
39
+ def count(column_name = nil)
40
+ calculate(:count, column_name)
49
41
  end
50
42
 
51
43
  # Calculates the average value on a given column. Returns +nil+ if there's
52
- # no row. See +calculate+ for examples with options.
44
+ # no row. See #calculate for examples with options.
53
45
  #
54
46
  # Person.average(:age) # => 35.8
55
- def average(column_name, options = {})
56
- # TODO: Remove options argument as soon we remove support to
57
- # activerecord-deprecated_finders.
58
- calculate(:average, column_name, options)
47
+ def average(column_name)
48
+ calculate(:average, column_name)
59
49
  end
60
50
 
61
51
  # Calculates the minimum value on a given column. The value is returned
62
52
  # with the same data type of the column, or +nil+ if there's no row. See
63
- # +calculate+ for examples with options.
53
+ # #calculate for examples with options.
64
54
  #
65
55
  # Person.minimum(:age) # => 7
66
- def minimum(column_name, options = {})
67
- # TODO: Remove options argument as soon we remove support to
68
- # activerecord-deprecated_finders.
69
- calculate(:minimum, column_name, options)
56
+ def minimum(column_name)
57
+ calculate(:minimum, column_name)
70
58
  end
71
59
 
72
60
  # Calculates the maximum value on a given column. The value is returned
73
61
  # with the same data type of the column, or +nil+ if there's no row. See
74
- # +calculate+ for examples with options.
62
+ # #calculate for examples with options.
75
63
  #
76
64
  # Person.maximum(:age) # => 93
77
- def maximum(column_name, options = {})
78
- # TODO: Remove options argument as soon we remove support to
79
- # activerecord-deprecated_finders.
80
- calculate(:maximum, column_name, options)
65
+ def maximum(column_name)
66
+ calculate(:maximum, column_name)
81
67
  end
82
68
 
83
69
  # Calculates the sum of values on a given column. The value is returned
84
- # with the same data type of the column, 0 if there's no row. See
85
- # +calculate+ for examples with options.
70
+ # with the same data type of the column, +0+ if there's no row. See
71
+ # #calculate for examples with options.
86
72
  #
87
73
  # Person.sum(:age) # => 4562
88
- def sum(*args)
89
- calculate(:sum, *args)
74
+ def sum(column_name = nil, &block)
75
+ return super(&block) if block_given?
76
+ calculate(:sum, column_name)
90
77
  end
91
78
 
92
- # This calculates aggregate values in the given column. Methods for count, sum, average,
93
- # minimum, and maximum have been added as shortcuts.
79
+ # This calculates aggregate values in the given column. Methods for #count, #sum, #average,
80
+ # #minimum, and #maximum have been added as shortcuts.
94
81
  #
95
- # There are two basic forms of output:
82
+ # Person.calculate(:count, :all) # The same as Person.count
83
+ # Person.average(:age) # SELECT AVG(age) FROM people...
96
84
  #
97
- # * Single aggregate value: The single value is type cast to Integer for COUNT, Float
98
- # for AVG, and the given column's type for everything else.
85
+ # # Selects the minimum age for any family without any minors
86
+ # Person.group(:last_name).having("min(age) > 17").minimum(:age)
99
87
  #
100
- # * Grouped values: This returns an ordered hash of the values and groups them. It
101
- # takes either a column name, or the name of a belongs_to association.
88
+ # Person.sum("2 * age")
102
89
  #
103
- # values = Person.group('last_name').maximum(:age)
104
- # puts values["Drake"]
105
- # # => 43
90
+ # There are two basic forms of output:
106
91
  #
107
- # drake = Family.find_by(last_name: 'Drake')
108
- # values = Person.group(:family).maximum(:age) # Person belongs_to :family
109
- # puts values[drake]
110
- # # => 43
92
+ # * Single aggregate value: The single value is type cast to Fixnum for COUNT, Float
93
+ # for AVG, and the given column's type for everything else.
111
94
  #
112
- # values.each do |family, max_age|
113
- # ...
114
- # end
95
+ # * Grouped values: This returns an ordered hash of the values and groups them. It
96
+ # takes either a column name, or the name of a belongs_to association.
115
97
  #
116
- # Person.calculate(:count, :all) # The same as Person.count
117
- # Person.average(:age) # SELECT AVG(age) FROM people...
98
+ # values = Person.group('last_name').maximum(:age)
99
+ # puts values["Drake"]
100
+ # # => 43
118
101
  #
119
- # # Selects the minimum age for any family without any minors
120
- # Person.group(:last_name).having("min(age) > 17").minimum(:age)
102
+ # drake = Family.find_by(last_name: 'Drake')
103
+ # values = Person.group(:family).maximum(:age) # Person belongs_to :family
104
+ # puts values[drake]
105
+ # # => 43
121
106
  #
122
- # Person.sum("2 * age")
123
- def calculate(operation, column_name, options = {})
124
- # TODO: Remove options argument as soon we remove support to
125
- # activerecord-deprecated_finders.
107
+ # values.each do |family, max_age|
108
+ # ...
109
+ # end
110
+ def calculate(operation, column_name)
126
111
  if column_name.is_a?(Symbol) && attribute_alias?(column_name)
127
112
  column_name = attribute_alias(column_name)
128
113
  end
129
114
 
130
115
  if has_include?(column_name)
131
- construct_relation_for_association_calculations.calculate(operation, column_name, options)
116
+ construct_relation_for_association_calculations.calculate(operation, column_name)
132
117
  else
133
- perform_calculation(operation, column_name, options)
118
+ perform_calculation(operation, column_name)
134
119
  end
135
120
  end
136
121
 
137
- # Use <tt>pluck</tt> as a shortcut to select one or more attributes without
122
+ # Use #pluck as a shortcut to select one or more attributes without
138
123
  # loading a bunch of records just to grab the attributes you want.
139
124
  #
140
125
  # Person.pluck(:name)
@@ -143,19 +128,19 @@ module ActiveRecord
143
128
  #
144
129
  # Person.all.map(&:name)
145
130
  #
146
- # Pluck returns an <tt>Array</tt> of attribute values type-casted to match
131
+ # Pluck returns an Array of attribute values type-casted to match
147
132
  # the plucked column names, if they can be deduced. Plucking an SQL fragment
148
133
  # returns String values by default.
149
134
  #
150
- # Person.pluck(:id)
151
- # # SELECT people.id FROM people
152
- # # => [1, 2, 3]
135
+ # Person.pluck(:name)
136
+ # # SELECT people.name FROM people
137
+ # # => ['David', 'Jeremy', 'Jose']
153
138
  #
154
139
  # Person.pluck(:id, :name)
155
140
  # # SELECT people.id, people.name FROM people
156
141
  # # => [[1, 'David'], [2, 'Jeremy'], [3, 'Jose']]
157
142
  #
158
- # Person.pluck('DISTINCT role')
143
+ # Person.distinct.pluck(:role)
159
144
  # # SELECT DISTINCT role FROM people
160
145
  # # => ['admin', 'member', 'guest']
161
146
  #
@@ -167,6 +152,8 @@ module ActiveRecord
167
152
  # # SELECT DATEDIFF(updated_at, created_at) FROM people
168
153
  # # => ['0', '27761', '173']
169
154
  #
155
+ # See also #ids.
156
+ #
170
157
  def pluck(*column_names)
171
158
  column_names.map! do |column_name|
172
159
  if column_name.is_a?(Symbol) && attribute_alias?(column_name)
@@ -176,6 +163,10 @@ module ActiveRecord
176
163
  end
177
164
  end
178
165
 
166
+ if loaded? && (column_names - @klass.column_names).empty?
167
+ return @records.pluck(*column_names)
168
+ end
169
+
179
170
  if has_include?(column_names.first)
180
171
  construct_relation_for_association_calculations.pluck(*column_names)
181
172
  else
@@ -183,8 +174,8 @@ module ActiveRecord
183
174
  relation.select_values = column_names.map { |cn|
184
175
  columns_hash.key?(cn) ? arel_table[cn] : cn
185
176
  }
186
- result = klass.connection.select_all(relation.arel, nil, relation.arel.bind_values + bind_values)
187
- result.cast_values(klass.column_types)
177
+ result = klass.connection.select_all(relation.arel, nil, bound_attributes)
178
+ result.cast_values(klass.attribute_types)
188
179
  end
189
180
  end
190
181
 
@@ -199,15 +190,14 @@ module ActiveRecord
199
190
  private
200
191
 
201
192
  def has_include?(column_name)
202
- eager_loading? || (includes_values.present? && ((column_name && column_name != :all) || references_eager_loaded_tables?))
193
+ eager_loading? || (includes_values.present? && column_name && column_name != :all)
203
194
  end
204
195
 
205
- def perform_calculation(operation, column_name, options = {})
206
- # TODO: Remove options argument as soon we remove support to
207
- # activerecord-deprecated_finders.
196
+ def perform_calculation(operation, column_name)
208
197
  operation = operation.to_s.downcase
209
198
 
210
- # If #count is used with #distinct / #uniq it is considered distinct. (eg. relation.distinct.count)
199
+ # If #count is used with #distinct (i.e. `relation.distinct.count`) it is
200
+ # considered distinct.
211
201
  distinct = self.distinct_value
212
202
 
213
203
  if operation == "count"
@@ -229,6 +219,8 @@ module ActiveRecord
229
219
  end
230
220
 
231
221
  def aggregate_column(column_name)
222
+ return column_name if Arel::Expressions === column_name
223
+
232
224
  if @klass.column_names.include?(column_name.to_s)
233
225
  Arel::Attribute.new(@klass.unscoped.table, column_name)
234
226
  else
@@ -241,19 +233,16 @@ module ActiveRecord
241
233
  end
242
234
 
243
235
  def execute_simple_calculation(operation, column_name, distinct) #:nodoc:
244
- # Postgresql doesn't like ORDER BY when there are no GROUP BY
236
+ # PostgreSQL doesn't like ORDER BY when there are no GROUP BY
245
237
  relation = unscope(:order)
246
238
 
247
239
  column_alias = column_name
248
240
 
249
- bind_values = nil
250
-
251
241
  if operation == "count" && (relation.limit_value || relation.offset_value)
252
242
  # Shortcut when limit is zero.
253
243
  return 0 if relation.limit_value == 0
254
244
 
255
245
  query_builder = build_count_subquery(relation, column_name, distinct)
256
- bind_values = query_builder.bind_values + relation.bind_values
257
246
  else
258
247
  column = aggregate_column(column_name)
259
248
 
@@ -264,10 +253,9 @@ module ActiveRecord
264
253
  relation.select_values = [select_value]
265
254
 
266
255
  query_builder = relation.arel
267
- bind_values = query_builder.bind_values + relation.bind_values
268
256
  end
269
257
 
270
- result = @klass.connection.select_all(query_builder, nil, bind_values)
258
+ result = @klass.connection.select_all(query_builder, nil, bound_attributes)
271
259
  row = result.first
272
260
  value = row && row.values.first
273
261
  column = result.column_types.fetch(column_alias) do
@@ -289,14 +277,8 @@ module ActiveRecord
289
277
  end
290
278
  group_fields = arel_columns(group_fields)
291
279
 
292
- group_aliases = group_fields.map { |field|
293
- column_alias_for(field)
294
- }
295
- group_columns = group_aliases.zip(group_fields).map { |aliaz,field|
296
- [aliaz, field]
297
- }
298
-
299
- group = group_fields
280
+ group_aliases = group_fields.map { |field| column_alias_for(field) }
281
+ group_columns = group_aliases.zip(group_fields)
300
282
 
301
283
  if operation == 'count' && column_name == :all
302
284
  aggregate_alias = 'count_all'
@@ -310,9 +292,9 @@ module ActiveRecord
310
292
  operation,
311
293
  distinct).as(aggregate_alias)
312
294
  ]
313
- select_values += self.select_values unless having_values.empty?
295
+ select_values += select_values unless having_clause.empty?
314
296
 
315
- select_values.concat group_fields.zip(group_aliases).map { |field,aliaz|
297
+ select_values.concat group_columns.map { |aliaz, field|
316
298
  if field.respond_to?(:as)
317
299
  field.as(aliaz)
318
300
  else
@@ -321,14 +303,14 @@ module ActiveRecord
321
303
  }
322
304
 
323
305
  relation = except(:group)
324
- relation.group_values = group
306
+ relation.group_values = group_fields
325
307
  relation.select_values = select_values
326
308
 
327
- calculated_data = @klass.connection.select_all(relation, nil, relation.arel.bind_values + bind_values)
309
+ calculated_data = @klass.connection.select_all(relation, nil, relation.bound_attributes)
328
310
 
329
311
  if association
330
312
  key_ids = calculated_data.collect { |row| row[group_aliases.first] }
331
- key_records = association.klass.base_class.find(key_ids)
313
+ key_records = association.klass.base_class.where(association.klass.base_class.primary_key => key_ids)
332
314
  key_records = Hash[key_records.map { |r| [r.id, r] }]
333
315
  end
334
316
 
@@ -354,7 +336,6 @@ module ActiveRecord
354
336
  # column_alias_for("sum(id)") # => "sum_id"
355
337
  # column_alias_for("count(distinct users.id)") # => "count_distinct_users_id"
356
338
  # column_alias_for("count(*)") # => "count_all"
357
- # column_alias_for("count", "id") # => "count_id"
358
339
  def column_alias_for(keys)
359
340
  if keys.respond_to? :name
360
341
  keys = "#{keys.relation.name}.#{keys.name}"
@@ -377,15 +358,15 @@ module ActiveRecord
377
358
  def type_cast_calculated_value(value, type, operation = nil)
378
359
  case operation
379
360
  when 'count' then value.to_i
380
- when 'sum' then type.type_cast_from_database(value || 0)
361
+ when 'sum' then type.deserialize(value || 0)
381
362
  when 'average' then value.respond_to?(:to_d) ? value.to_d : value
382
- else type.type_cast_from_database(value)
363
+ else type.deserialize(value)
383
364
  end
384
365
  end
385
366
 
386
- # TODO: refactor to allow non-string `select_values` (eg. Arel nodes).
387
367
  def select_for_count
388
368
  if select_values.present?
369
+ return select_values.first if select_values.one?
389
370
  select_values.join(", ")
390
371
  else
391
372
  :all
@@ -398,11 +379,9 @@ module ActiveRecord
398
379
 
399
380
  aliased_column = aggregate_column(column_name == :all ? 1 : column_name).as(column_alias)
400
381
  relation.select_values = [aliased_column]
401
- arel = relation.arel
402
- subquery = arel.as(subquery_alias)
382
+ subquery = relation.arel.as(subquery_alias)
403
383
 
404
384
  sm = Arel::SelectManager.new relation.engine
405
- sm.bind_values = arel.bind_values
406
385
  select_value = operation_over_aggregate_column(column_alias, 'count', distinct)
407
386
  sm.project(select_value).from(subquery)
408
387
  end