activerecord 4.2.6 → 5.0.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of activerecord might be problematic. Click here for more details.

Files changed (246) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1307 -1105
  3. data/MIT-LICENSE +2 -2
  4. data/README.rdoc +7 -8
  5. data/examples/performance.rb +2 -3
  6. data/examples/simple.rb +0 -1
  7. data/lib/active_record/aggregations.rb +37 -23
  8. data/lib/active_record/association_relation.rb +3 -3
  9. data/lib/active_record/associations/alias_tracker.rb +19 -16
  10. data/lib/active_record/associations/association.rb +11 -9
  11. data/lib/active_record/associations/association_scope.rb +73 -102
  12. data/lib/active_record/associations/belongs_to_association.rb +21 -32
  13. data/lib/active_record/associations/builder/association.rb +28 -34
  14. data/lib/active_record/associations/builder/belongs_to.rb +43 -18
  15. data/lib/active_record/associations/builder/collection_association.rb +7 -19
  16. data/lib/active_record/associations/builder/has_and_belongs_to_many.rb +14 -11
  17. data/lib/active_record/associations/builder/has_many.rb +4 -4
  18. data/lib/active_record/associations/builder/has_one.rb +11 -6
  19. data/lib/active_record/associations/builder/singular_association.rb +3 -10
  20. data/lib/active_record/associations/collection_association.rb +50 -31
  21. data/lib/active_record/associations/collection_proxy.rb +69 -29
  22. data/lib/active_record/associations/foreign_association.rb +1 -1
  23. data/lib/active_record/associations/has_many_association.rb +20 -71
  24. data/lib/active_record/associations/has_many_through_association.rb +8 -47
  25. data/lib/active_record/associations/has_one_association.rb +12 -5
  26. data/lib/active_record/associations/join_dependency/join_association.rb +20 -8
  27. data/lib/active_record/associations/join_dependency.rb +29 -19
  28. data/lib/active_record/associations/preloader/association.rb +46 -52
  29. data/lib/active_record/associations/preloader/collection_association.rb +0 -6
  30. data/lib/active_record/associations/preloader/has_many_through.rb +1 -1
  31. data/lib/active_record/associations/preloader/has_one.rb +0 -8
  32. data/lib/active_record/associations/preloader/through_association.rb +27 -14
  33. data/lib/active_record/associations/preloader.rb +14 -4
  34. data/lib/active_record/associations/singular_association.rb +7 -1
  35. data/lib/active_record/associations/through_association.rb +11 -3
  36. data/lib/active_record/associations.rb +317 -209
  37. data/lib/active_record/attribute/user_provided_default.rb +28 -0
  38. data/lib/active_record/attribute.rb +68 -18
  39. data/lib/active_record/attribute_assignment.rb +20 -141
  40. data/lib/active_record/attribute_decorators.rb +6 -5
  41. data/lib/active_record/attribute_methods/before_type_cast.rb +1 -1
  42. data/lib/active_record/attribute_methods/dirty.rb +46 -86
  43. data/lib/active_record/attribute_methods/primary_key.rb +2 -2
  44. data/lib/active_record/attribute_methods/query.rb +2 -2
  45. data/lib/active_record/attribute_methods/read.rb +31 -59
  46. data/lib/active_record/attribute_methods/serialization.rb +13 -16
  47. data/lib/active_record/attribute_methods/time_zone_conversion.rb +61 -14
  48. data/lib/active_record/attribute_methods/write.rb +14 -38
  49. data/lib/active_record/attribute_methods.rb +70 -45
  50. data/lib/active_record/attribute_mutation_tracker.rb +70 -0
  51. data/lib/active_record/attribute_set/builder.rb +6 -4
  52. data/lib/active_record/attribute_set.rb +30 -3
  53. data/lib/active_record/attributes.rb +199 -80
  54. data/lib/active_record/autosave_association.rb +49 -16
  55. data/lib/active_record/base.rb +32 -23
  56. data/lib/active_record/callbacks.rb +39 -43
  57. data/lib/active_record/coders/json.rb +1 -1
  58. data/lib/active_record/coders/yaml_column.rb +20 -8
  59. data/lib/active_record/collection_cache_key.rb +40 -0
  60. data/lib/active_record/connection_adapters/abstract/connection_pool.rb +452 -182
  61. data/lib/active_record/connection_adapters/abstract/database_limits.rb +3 -3
  62. data/lib/active_record/connection_adapters/abstract/database_statements.rb +65 -61
  63. data/lib/active_record/connection_adapters/abstract/query_cache.rb +2 -2
  64. data/lib/active_record/connection_adapters/abstract/quoting.rb +74 -9
  65. data/lib/active_record/connection_adapters/abstract/savepoints.rb +3 -3
  66. data/lib/active_record/connection_adapters/abstract/schema_creation.rb +61 -39
  67. data/lib/active_record/connection_adapters/abstract/schema_definitions.rb +236 -185
  68. data/lib/active_record/connection_adapters/abstract/schema_dumper.rb +72 -17
  69. data/lib/active_record/connection_adapters/abstract/schema_statements.rb +378 -140
  70. data/lib/active_record/connection_adapters/abstract/transaction.rb +51 -34
  71. data/lib/active_record/connection_adapters/abstract_adapter.rb +153 -59
  72. data/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +405 -362
  73. data/lib/active_record/connection_adapters/column.rb +28 -43
  74. data/lib/active_record/connection_adapters/connection_specification.rb +15 -27
  75. data/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb +22 -0
  76. data/lib/active_record/connection_adapters/mysql/column.rb +50 -0
  77. data/lib/active_record/connection_adapters/mysql/database_statements.rb +125 -0
  78. data/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb +70 -0
  79. data/lib/active_record/connection_adapters/mysql/quoting.rb +51 -0
  80. data/lib/active_record/connection_adapters/mysql/schema_creation.rb +67 -0
  81. data/lib/active_record/connection_adapters/mysql/schema_definitions.rb +93 -0
  82. data/lib/active_record/connection_adapters/mysql/schema_dumper.rb +54 -0
  83. data/lib/active_record/connection_adapters/mysql/type_metadata.rb +32 -0
  84. data/lib/active_record/connection_adapters/mysql2_adapter.rb +25 -176
  85. data/lib/active_record/connection_adapters/postgresql/column.rb +5 -10
  86. data/lib/active_record/connection_adapters/postgresql/database_statements.rb +10 -72
  87. data/lib/active_record/connection_adapters/postgresql/explain_pretty_printer.rb +42 -0
  88. data/lib/active_record/connection_adapters/postgresql/oid/array.rb +27 -56
  89. data/lib/active_record/connection_adapters/postgresql/oid/bit.rb +2 -2
  90. data/lib/active_record/connection_adapters/postgresql/oid/bytea.rb +1 -1
  91. data/lib/active_record/connection_adapters/postgresql/oid/cidr.rb +3 -1
  92. data/lib/active_record/connection_adapters/postgresql/oid/date_time.rb +7 -22
  93. data/lib/active_record/connection_adapters/postgresql/oid/hstore.rb +3 -3
  94. data/lib/active_record/connection_adapters/postgresql/oid/json.rb +1 -26
  95. data/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb +2 -2
  96. data/lib/active_record/connection_adapters/postgresql/oid/money.rb +0 -4
  97. data/lib/active_record/connection_adapters/postgresql/oid/point.rb +4 -4
  98. data/lib/active_record/connection_adapters/postgresql/oid/rails_5_1_point.rb +50 -0
  99. data/lib/active_record/connection_adapters/postgresql/oid/range.rb +31 -17
  100. data/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb +0 -4
  101. data/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +2 -2
  102. data/lib/active_record/connection_adapters/postgresql/oid/vector.rb +1 -1
  103. data/lib/active_record/connection_adapters/postgresql/oid/xml.rb +1 -1
  104. data/lib/active_record/connection_adapters/postgresql/oid.rb +1 -6
  105. data/lib/active_record/connection_adapters/postgresql/quoting.rb +26 -18
  106. data/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +29 -10
  107. data/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +107 -79
  108. data/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +47 -0
  109. data/lib/active_record/connection_adapters/postgresql/schema_statements.rb +234 -148
  110. data/lib/active_record/connection_adapters/postgresql/type_metadata.rb +35 -0
  111. data/lib/active_record/connection_adapters/postgresql_adapter.rb +248 -160
  112. data/lib/active_record/connection_adapters/schema_cache.rb +36 -23
  113. data/lib/active_record/connection_adapters/sql_type_metadata.rb +32 -0
  114. data/lib/active_record/connection_adapters/sqlite3/explain_pretty_printer.rb +19 -0
  115. data/lib/active_record/connection_adapters/sqlite3/quoting.rb +48 -0
  116. data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +22 -0
  117. data/lib/active_record/connection_adapters/sqlite3_adapter.rb +148 -203
  118. data/lib/active_record/connection_adapters/statement_pool.rb +31 -12
  119. data/lib/active_record/connection_handling.rb +37 -14
  120. data/lib/active_record/core.rb +89 -107
  121. data/lib/active_record/counter_cache.rb +13 -24
  122. data/lib/active_record/dynamic_matchers.rb +1 -20
  123. data/lib/active_record/enum.rb +113 -76
  124. data/lib/active_record/errors.rb +87 -48
  125. data/lib/active_record/explain_registry.rb +1 -1
  126. data/lib/active_record/explain_subscriber.rb +1 -1
  127. data/lib/active_record/fixture_set/file.rb +26 -5
  128. data/lib/active_record/fixtures.rb +76 -40
  129. data/lib/active_record/gem_version.rb +3 -3
  130. data/lib/active_record/inheritance.rb +32 -40
  131. data/lib/active_record/integration.rb +4 -4
  132. data/lib/active_record/internal_metadata.rb +56 -0
  133. data/lib/active_record/legacy_yaml_adapter.rb +18 -2
  134. data/lib/active_record/locale/en.yml +3 -2
  135. data/lib/active_record/locking/optimistic.rb +15 -15
  136. data/lib/active_record/locking/pessimistic.rb +1 -1
  137. data/lib/active_record/log_subscriber.rb +43 -21
  138. data/lib/active_record/migration/command_recorder.rb +59 -18
  139. data/lib/active_record/migration/compatibility.rb +126 -0
  140. data/lib/active_record/migration.rb +364 -109
  141. data/lib/active_record/model_schema.rb +128 -38
  142. data/lib/active_record/nested_attributes.rb +58 -29
  143. data/lib/active_record/null_relation.rb +16 -8
  144. data/lib/active_record/persistence.rb +121 -80
  145. data/lib/active_record/query_cache.rb +15 -18
  146. data/lib/active_record/querying.rb +10 -9
  147. data/lib/active_record/railtie.rb +27 -18
  148. data/lib/active_record/railties/controller_runtime.rb +1 -1
  149. data/lib/active_record/railties/databases.rake +58 -45
  150. data/lib/active_record/readonly_attributes.rb +1 -1
  151. data/lib/active_record/reflection.rb +282 -115
  152. data/lib/active_record/relation/batches/batch_enumerator.rb +67 -0
  153. data/lib/active_record/relation/batches.rb +139 -34
  154. data/lib/active_record/relation/calculations.rb +80 -102
  155. data/lib/active_record/relation/delegation.rb +7 -20
  156. data/lib/active_record/relation/finder_methods.rb +163 -81
  157. data/lib/active_record/relation/from_clause.rb +32 -0
  158. data/lib/active_record/relation/merger.rb +16 -42
  159. data/lib/active_record/relation/predicate_builder/array_handler.rb +11 -15
  160. data/lib/active_record/relation/predicate_builder/association_query_handler.rb +88 -0
  161. data/lib/active_record/relation/predicate_builder/base_handler.rb +17 -0
  162. data/lib/active_record/relation/predicate_builder/basic_object_handler.rb +17 -0
  163. data/lib/active_record/relation/predicate_builder/class_handler.rb +27 -0
  164. data/lib/active_record/relation/predicate_builder/polymorphic_array_handler.rb +57 -0
  165. data/lib/active_record/relation/predicate_builder/range_handler.rb +33 -0
  166. data/lib/active_record/relation/predicate_builder/relation_handler.rb +1 -1
  167. data/lib/active_record/relation/predicate_builder.rb +120 -107
  168. data/lib/active_record/relation/query_attribute.rb +19 -0
  169. data/lib/active_record/relation/query_methods.rb +308 -244
  170. data/lib/active_record/relation/record_fetch_warning.rb +49 -0
  171. data/lib/active_record/relation/spawn_methods.rb +4 -7
  172. data/lib/active_record/relation/where_clause.rb +174 -0
  173. data/lib/active_record/relation/where_clause_factory.rb +38 -0
  174. data/lib/active_record/relation.rb +176 -116
  175. data/lib/active_record/result.rb +4 -3
  176. data/lib/active_record/runtime_registry.rb +1 -1
  177. data/lib/active_record/sanitization.rb +95 -66
  178. data/lib/active_record/schema.rb +26 -22
  179. data/lib/active_record/schema_dumper.rb +62 -38
  180. data/lib/active_record/schema_migration.rb +11 -17
  181. data/lib/active_record/scoping/default.rb +23 -9
  182. data/lib/active_record/scoping/named.rb +49 -28
  183. data/lib/active_record/scoping.rb +32 -15
  184. data/lib/active_record/secure_token.rb +38 -0
  185. data/lib/active_record/serialization.rb +2 -4
  186. data/lib/active_record/statement_cache.rb +16 -14
  187. data/lib/active_record/store.rb +8 -3
  188. data/lib/active_record/suppressor.rb +58 -0
  189. data/lib/active_record/table_metadata.rb +68 -0
  190. data/lib/active_record/tasks/database_tasks.rb +58 -41
  191. data/lib/active_record/tasks/mysql_database_tasks.rb +16 -20
  192. data/lib/active_record/tasks/postgresql_database_tasks.rb +11 -2
  193. data/lib/active_record/tasks/sqlite_database_tasks.rb +5 -1
  194. data/lib/active_record/timestamp.rb +20 -9
  195. data/lib/active_record/touch_later.rb +58 -0
  196. data/lib/active_record/transactions.rb +138 -56
  197. data/lib/active_record/type/adapter_specific_registry.rb +130 -0
  198. data/lib/active_record/type/date.rb +2 -41
  199. data/lib/active_record/type/date_time.rb +2 -49
  200. data/lib/active_record/type/internal/abstract_json.rb +29 -0
  201. data/lib/active_record/type/internal/timezone.rb +15 -0
  202. data/lib/active_record/type/serialized.rb +15 -14
  203. data/lib/active_record/type/time.rb +10 -16
  204. data/lib/active_record/type/type_map.rb +4 -4
  205. data/lib/active_record/type.rb +66 -17
  206. data/lib/active_record/type_caster/connection.rb +29 -0
  207. data/lib/active_record/type_caster/map.rb +19 -0
  208. data/lib/active_record/type_caster.rb +7 -0
  209. data/lib/active_record/validations/absence.rb +23 -0
  210. data/lib/active_record/validations/associated.rb +10 -3
  211. data/lib/active_record/validations/length.rb +24 -0
  212. data/lib/active_record/validations/presence.rb +11 -12
  213. data/lib/active_record/validations/uniqueness.rb +30 -29
  214. data/lib/active_record/validations.rb +33 -32
  215. data/lib/active_record.rb +7 -2
  216. data/lib/rails/generators/active_record/migration/migration_generator.rb +7 -4
  217. data/lib/rails/generators/active_record/migration/templates/create_table_migration.rb +8 -3
  218. data/lib/rails/generators/active_record/migration/templates/migration.rb +8 -1
  219. data/lib/rails/generators/active_record/migration.rb +7 -0
  220. data/lib/rails/generators/active_record/model/model_generator.rb +32 -15
  221. data/lib/rails/generators/active_record/model/templates/application_record.rb +5 -0
  222. data/lib/rails/generators/active_record/model/templates/model.rb +3 -0
  223. metadata +58 -34
  224. data/lib/active_record/connection_adapters/mysql_adapter.rb +0 -491
  225. data/lib/active_record/connection_adapters/postgresql/array_parser.rb +0 -93
  226. data/lib/active_record/connection_adapters/postgresql/oid/date.rb +0 -11
  227. data/lib/active_record/connection_adapters/postgresql/oid/float.rb +0 -21
  228. data/lib/active_record/connection_adapters/postgresql/oid/infinity.rb +0 -13
  229. data/lib/active_record/connection_adapters/postgresql/oid/integer.rb +0 -11
  230. data/lib/active_record/connection_adapters/postgresql/oid/time.rb +0 -11
  231. data/lib/active_record/serializers/xml_serializer.rb +0 -193
  232. data/lib/active_record/type/big_integer.rb +0 -13
  233. data/lib/active_record/type/binary.rb +0 -50
  234. data/lib/active_record/type/boolean.rb +0 -31
  235. data/lib/active_record/type/decimal.rb +0 -50
  236. data/lib/active_record/type/decimal_without_scale.rb +0 -11
  237. data/lib/active_record/type/decorator.rb +0 -14
  238. data/lib/active_record/type/float.rb +0 -19
  239. data/lib/active_record/type/integer.rb +0 -59
  240. data/lib/active_record/type/mutable.rb +0 -16
  241. data/lib/active_record/type/numeric.rb +0 -36
  242. data/lib/active_record/type/string.rb +0 -40
  243. data/lib/active_record/type/text.rb +0 -11
  244. data/lib/active_record/type/time_value.rb +0 -38
  245. data/lib/active_record/type/unsigned_integer.rb +0 -15
  246. data/lib/active_record/type/value.rb +0 -105
@@ -1,8 +1,12 @@
1
+ require "active_record/relation/batches/batch_enumerator"
2
+
1
3
  module ActiveRecord
2
4
  module Batches
5
+ ORDER_OR_LIMIT_IGNORED_MESSAGE = "Scoped order and limit are ignored, it's forced to be batch order and batch size."
6
+
3
7
  # Looping through a collection of records from the database
4
- # (using the +all+ method, for example) is very inefficient
5
- # since it will try to instantiate all the objects at once.
8
+ # (using the Scoping::Named::ClassMethods.all method, for example)
9
+ # is very inefficient since it will try to instantiate all the objects at once.
6
10
  #
7
11
  # In that case, batch processing methods allow you to work
8
12
  # with the records in batches, thereby greatly reducing memory consumption.
@@ -27,11 +31,15 @@ module ActiveRecord
27
31
  #
28
32
  # ==== Options
29
33
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000.
30
- # * <tt>:start</tt> - Specifies the starting point for the batch processing.
34
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
35
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
36
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
37
+ # the order and limit have to be ignored due to batching.
38
+ #
31
39
  # This is especially useful if you want multiple workers dealing with
32
40
  # the same processing queue. You can make worker 1 handle all the records
33
41
  # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond
34
- # (by setting the +:start+ option on that worker).
42
+ # (by setting the +:start+ and +:finish+ option on each worker).
35
43
  #
36
44
  # # Let's process for a batch of 2000 records, skipping the first 2000 rows
37
45
  # Person.find_each(start: 2000, batch_size: 2000) do |person|
@@ -40,24 +48,25 @@ module ActiveRecord
40
48
  #
41
49
  # NOTE: It's not possible to set the order. That is automatically set to
42
50
  # ascending on the primary key ("id ASC") to make the batch ordering
43
- # work. This also means that this method only works with integer-based
44
- # primary keys.
51
+ # work. This also means that this method only works when the primary key is
52
+ # orderable (e.g. an integer or string).
45
53
  #
46
54
  # NOTE: You can't set the limit either, that's used to control
47
55
  # the batch sizes.
48
- def find_each(options = {})
56
+ def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
49
57
  if block_given?
50
- find_in_batches(options) do |records|
58
+ find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do |records|
51
59
  records.each { |record| yield record }
52
60
  end
53
61
  else
54
- enum_for :find_each, options do
55
- options[:start] ? where(table[primary_key].gteq(options[:start])).size : size
62
+ enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
63
+ relation = self
64
+ apply_limits(relation, start, finish).size
56
65
  end
57
66
  end
58
67
  end
59
68
 
60
- # Yields each batch of records that was found by the find +options+ as
69
+ # Yields each batch of records that was found by the find options as
61
70
  # an array.
62
71
  #
63
72
  # Person.where("age > 21").find_in_batches do |group|
@@ -77,11 +86,15 @@ module ActiveRecord
77
86
  #
78
87
  # ==== Options
79
88
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000.
80
- # * <tt>:start</tt> - Specifies the starting point for the batch processing.
89
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
90
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
91
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
92
+ # the order and limit have to be ignored due to batching.
93
+ #
81
94
  # This is especially useful if you want multiple workers dealing with
82
95
  # the same processing queue. You can make worker 1 handle all the records
83
96
  # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond
84
- # (by setting the +:start+ option on that worker).
97
+ # (by setting the +:start+ and +:finish+ option on each worker).
85
98
  #
86
99
  # # Let's process the next 2000 records
87
100
  # Person.find_in_batches(start: 2000, batch_size: 2000) do |group|
@@ -90,49 +103,141 @@ module ActiveRecord
90
103
  #
91
104
  # NOTE: It's not possible to set the order. That is automatically set to
92
105
  # ascending on the primary key ("id ASC") to make the batch ordering
93
- # work. This also means that this method only works with integer-based
94
- # primary keys.
106
+ # work. This also means that this method only works when the primary key is
107
+ # orderable (e.g. an integer or string).
95
108
  #
96
109
  # NOTE: You can't set the limit either, that's used to control
97
110
  # the batch sizes.
98
- def find_in_batches(options = {})
99
- options.assert_valid_keys(:start, :batch_size)
100
-
111
+ def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
101
112
  relation = self
102
- start = options[:start]
103
- batch_size = options[:batch_size] || 1000
104
-
105
113
  unless block_given?
106
- return to_enum(:find_in_batches, options) do
107
- total = start ? where(table[primary_key].gteq(start)).size : size
114
+ return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
115
+ total = apply_limits(relation, start, finish).size
108
116
  (total - 1).div(batch_size) + 1
109
117
  end
110
118
  end
111
119
 
112
- if logger && (arel.orders.present? || arel.taken.present?)
113
- logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
120
+ in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore) do |batch|
121
+ yield batch.to_a
114
122
  end
123
+ end
115
124
 
116
- relation = relation.reorder(batch_order).limit(batch_size)
117
- records = start ? relation.where(table[primary_key].gteq(start)).to_a : relation.to_a
125
+ # Yields ActiveRecord::Relation objects to work with a batch of records.
126
+ #
127
+ # Person.where("age > 21").in_batches do |relation|
128
+ # relation.delete_all
129
+ # sleep(10) # Throttle the delete queries
130
+ # end
131
+ #
132
+ # If you do not provide a block to #in_batches, it will return a
133
+ # BatchEnumerator which is enumerable.
134
+ #
135
+ # Person.in_batches.with_index do |relation, batch_index|
136
+ # puts "Processing relation ##{batch_index}"
137
+ # relation.each { |relation| relation.delete_all }
138
+ # end
139
+ #
140
+ # Examples of calling methods on the returned BatchEnumerator object:
141
+ #
142
+ # Person.in_batches.delete_all
143
+ # Person.in_batches.update_all(awesome: true)
144
+ # Person.in_batches.each_record(&:party_all_night!)
145
+ #
146
+ # ==== Options
147
+ # * <tt>:of</tt> - Specifies the size of the batch. Default to 1000.
148
+ # * <tt>:load</tt> - Specifies if the relation should be loaded. Default to false.
149
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
150
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
151
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
152
+ # the order and limit have to be ignored due to batching.
153
+ #
154
+ # This is especially useful if you want to work with the
155
+ # ActiveRecord::Relation object instead of the array of records, or if
156
+ # you want multiple workers dealing with the same processing queue. You can
157
+ # make worker 1 handle all the records between id 0 and 10,000 and worker 2
158
+ # handle from 10,000 and beyond (by setting the +:start+ and +:finish+
159
+ # option on each worker).
160
+ #
161
+ # # Let's process the next 2000 records
162
+ # Person.in_batches(of: 2000, start: 2000).update_all(awesome: true)
163
+ #
164
+ # An example of calling where query method on the relation:
165
+ #
166
+ # Person.in_batches.each do |relation|
167
+ # relation.update_all('age = age + 1')
168
+ # relation.where('age > 21').update_all(should_party: true)
169
+ # relation.where('age <= 21').delete_all
170
+ # end
171
+ #
172
+ # NOTE: If you are going to iterate through each record, you should call
173
+ # #each_record on the yielded BatchEnumerator:
174
+ #
175
+ # Person.in_batches.each_record(&:party_all_night!)
176
+ #
177
+ # NOTE: It's not possible to set the order. That is automatically set to
178
+ # ascending on the primary key ("id ASC") to make the batch ordering
179
+ # consistent. Therefore the primary key must be orderable, e.g an integer
180
+ # or a string.
181
+ #
182
+ # NOTE: You can't set the limit either, that's used to control the batch
183
+ # sizes.
184
+ def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil)
185
+ relation = self
186
+ unless block_given?
187
+ return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self)
188
+ end
189
+
190
+ if arel.orders.present? || arel.taken.present?
191
+ act_on_order_or_limit_ignored(error_on_ignore)
192
+ end
193
+
194
+ relation = relation.reorder(batch_order).limit(of)
195
+ relation = apply_limits(relation, start, finish)
196
+ batch_relation = relation
118
197
 
119
- while records.any?
120
- records_size = records.size
121
- primary_key_offset = records.last.id
122
- raise "Primary key not included in the custom select clause" unless primary_key_offset
198
+ loop do
199
+ if load
200
+ records = batch_relation.records
201
+ ids = records.map(&:id)
202
+ yielded_relation = self.where(primary_key => ids)
203
+ yielded_relation.load_records(records)
204
+ else
205
+ ids = batch_relation.pluck(primary_key)
206
+ yielded_relation = self.where(primary_key => ids)
207
+ end
123
208
 
124
- yield records
209
+ break if ids.empty?
125
210
 
126
- break if records_size < batch_size
211
+ primary_key_offset = ids.last
212
+ raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
127
213
 
128
- records = relation.where(table[primary_key].gt(primary_key_offset)).to_a
214
+ yield yielded_relation
215
+
216
+ break if ids.length < of
217
+ batch_relation = relation.where(arel_attribute(primary_key).gt(primary_key_offset))
129
218
  end
130
219
  end
131
220
 
132
221
  private
133
222
 
223
+ def apply_limits(relation, start, finish)
224
+ relation = relation.where(arel_attribute(primary_key).gteq(start)) if start
225
+ relation = relation.where(arel_attribute(primary_key).lteq(finish)) if finish
226
+ relation
227
+ end
228
+
134
229
  def batch_order
135
230
  "#{quoted_table_name}.#{quoted_primary_key} ASC"
136
231
  end
232
+
233
+ def act_on_order_or_limit_ignored(error_on_ignore)
234
+ raise_error = (error_on_ignore.nil? ? self.klass.error_on_ignored_order_or_limit : error_on_ignore)
235
+
236
+ if raise_error
237
+ raise ArgumentError.new(ORDER_OR_LIMIT_IGNORED_MESSAGE)
238
+ elsif logger
239
+ logger.warn(ORDER_OR_LIMIT_IGNORED_MESSAGE)
240
+ end
241
+ end
137
242
  end
138
243
  end
@@ -14,121 +14,112 @@ module ActiveRecord
14
14
  # Person.distinct.count(:age)
15
15
  # # => counts the number of different age values
16
16
  #
17
- # If +count+ is used with +group+, it returns a Hash whose keys represent the aggregated column,
17
+ # If #count is used with {Relation#group}[rdoc-ref:QueryMethods#group],
18
+ # it returns a Hash whose keys represent the aggregated column,
18
19
  # and the values are the respective amounts:
19
20
  #
20
21
  # Person.group(:city).count
21
22
  # # => { 'Rome' => 5, 'Paris' => 3 }
22
23
  #
23
- # If +count+ is used with +group+ for multiple columns, it returns a Hash whose
24
+ # If #count is used with {Relation#group}[rdoc-ref:QueryMethods#group] for multiple columns, it returns a Hash whose
24
25
  # keys are an array containing the individual values of each column and the value
25
- # of each key would be the +count+.
26
+ # of each key would be the #count.
26
27
  #
27
28
  # Article.group(:status, :category).count
28
29
  # # => {["draft", "business"]=>10, ["draft", "technology"]=>4,
29
30
  # ["published", "business"]=>0, ["published", "technology"]=>2}
30
31
  #
31
- # If +count+ is used with +select+, it will count the selected columns:
32
+ # If #count is used with {Relation#select}[rdoc-ref:QueryMethods#select], it will count the selected columns:
32
33
  #
33
34
  # Person.select(:age).count
34
35
  # # => counts the number of different age values
35
36
  #
36
- # Note: not all valid +select+ expressions are valid +count+ expressions. The specifics differ
37
+ # Note: not all valid {Relation#select}[rdoc-ref:QueryMethods#select] expressions are valid #count expressions. The specifics differ
37
38
  # between databases. In invalid cases, an error from the database is thrown.
38
- def count(column_name = nil, options = {})
39
- # TODO: Remove options argument as soon we remove support to
40
- # activerecord-deprecated_finders.
41
- column_name, options = nil, column_name if column_name.is_a?(Hash)
42
- calculate(:count, column_name, options)
39
+ def count(column_name = nil)
40
+ calculate(:count, column_name)
43
41
  end
44
42
 
45
43
  # Calculates the average value on a given column. Returns +nil+ if there's
46
- # no row. See +calculate+ for examples with options.
44
+ # no row. See #calculate for examples with options.
47
45
  #
48
46
  # Person.average(:age) # => 35.8
49
- def average(column_name, options = {})
50
- # TODO: Remove options argument as soon we remove support to
51
- # activerecord-deprecated_finders.
52
- calculate(:average, column_name, options)
47
+ def average(column_name)
48
+ calculate(:average, column_name)
53
49
  end
54
50
 
55
51
  # Calculates the minimum value on a given column. The value is returned
56
52
  # with the same data type of the column, or +nil+ if there's no row. See
57
- # +calculate+ for examples with options.
53
+ # #calculate for examples with options.
58
54
  #
59
55
  # Person.minimum(:age) # => 7
60
- def minimum(column_name, options = {})
61
- # TODO: Remove options argument as soon we remove support to
62
- # activerecord-deprecated_finders.
63
- calculate(:minimum, column_name, options)
56
+ def minimum(column_name)
57
+ calculate(:minimum, column_name)
64
58
  end
65
59
 
66
60
  # Calculates the maximum value on a given column. The value is returned
67
61
  # with the same data type of the column, or +nil+ if there's no row. See
68
- # +calculate+ for examples with options.
62
+ # #calculate for examples with options.
69
63
  #
70
64
  # Person.maximum(:age) # => 93
71
- def maximum(column_name, options = {})
72
- # TODO: Remove options argument as soon we remove support to
73
- # activerecord-deprecated_finders.
74
- calculate(:maximum, column_name, options)
65
+ def maximum(column_name)
66
+ calculate(:maximum, column_name)
75
67
  end
76
68
 
77
69
  # Calculates the sum of values on a given column. The value is returned
78
- # with the same data type of the column, 0 if there's no row. See
79
- # +calculate+ for examples with options.
70
+ # with the same data type of the column, +0+ if there's no row. See
71
+ # #calculate for examples with options.
80
72
  #
81
73
  # Person.sum(:age) # => 4562
82
- def sum(*args)
83
- calculate(:sum, *args)
74
+ def sum(column_name = nil, &block)
75
+ return super(&block) if block_given?
76
+ calculate(:sum, column_name)
84
77
  end
85
78
 
86
- # This calculates aggregate values in the given column. Methods for count, sum, average,
87
- # minimum, and maximum have been added as shortcuts.
79
+ # This calculates aggregate values in the given column. Methods for #count, #sum, #average,
80
+ # #minimum, and #maximum have been added as shortcuts.
88
81
  #
89
- # There are two basic forms of output:
82
+ # Person.calculate(:count, :all) # The same as Person.count
83
+ # Person.average(:age) # SELECT AVG(age) FROM people...
90
84
  #
91
- # * Single aggregate value: The single value is type cast to Fixnum for COUNT, Float
92
- # for AVG, and the given column's type for everything else.
85
+ # # Selects the minimum age for any family without any minors
86
+ # Person.group(:last_name).having("min(age) > 17").minimum(:age)
93
87
  #
94
- # * Grouped values: This returns an ordered hash of the values and groups them. It
95
- # takes either a column name, or the name of a belongs_to association.
88
+ # Person.sum("2 * age")
96
89
  #
97
- # values = Person.group('last_name').maximum(:age)
98
- # puts values["Drake"]
99
- # # => 43
90
+ # There are two basic forms of output:
100
91
  #
101
- # drake = Family.find_by(last_name: 'Drake')
102
- # values = Person.group(:family).maximum(:age) # Person belongs_to :family
103
- # puts values[drake]
104
- # # => 43
92
+ # * Single aggregate value: The single value is type cast to Integer for COUNT, Float
93
+ # for AVG, and the given column's type for everything else.
105
94
  #
106
- # values.each do |family, max_age|
107
- # ...
108
- # end
95
+ # * Grouped values: This returns an ordered hash of the values and groups them. It
96
+ # takes either a column name, or the name of a belongs_to association.
109
97
  #
110
- # Person.calculate(:count, :all) # The same as Person.count
111
- # Person.average(:age) # SELECT AVG(age) FROM people...
98
+ # values = Person.group('last_name').maximum(:age)
99
+ # puts values["Drake"]
100
+ # # => 43
112
101
  #
113
- # # Selects the minimum age for any family without any minors
114
- # Person.group(:last_name).having("min(age) > 17").minimum(:age)
102
+ # drake = Family.find_by(last_name: 'Drake')
103
+ # values = Person.group(:family).maximum(:age) # Person belongs_to :family
104
+ # puts values[drake]
105
+ # # => 43
115
106
  #
116
- # Person.sum("2 * age")
117
- def calculate(operation, column_name, options = {})
118
- # TODO: Remove options argument as soon we remove support to
119
- # activerecord-deprecated_finders.
107
+ # values.each do |family, max_age|
108
+ # ...
109
+ # end
110
+ def calculate(operation, column_name)
120
111
  if column_name.is_a?(Symbol) && attribute_alias?(column_name)
121
112
  column_name = attribute_alias(column_name)
122
113
  end
123
114
 
124
115
  if has_include?(column_name)
125
- construct_relation_for_association_calculations.calculate(operation, column_name, options)
116
+ construct_relation_for_association_calculations.calculate(operation, column_name)
126
117
  else
127
- perform_calculation(operation, column_name, options)
118
+ perform_calculation(operation, column_name)
128
119
  end
129
120
  end
130
121
 
131
- # Use <tt>pluck</tt> as a shortcut to select one or more attributes without
122
+ # Use #pluck as a shortcut to select one or more attributes without
132
123
  # loading a bunch of records just to grab the attributes you want.
133
124
  #
134
125
  # Person.pluck(:name)
@@ -137,19 +128,19 @@ module ActiveRecord
137
128
  #
138
129
  # Person.all.map(&:name)
139
130
  #
140
- # Pluck returns an <tt>Array</tt> of attribute values type-casted to match
131
+ # Pluck returns an Array of attribute values type-casted to match
141
132
  # the plucked column names, if they can be deduced. Plucking an SQL fragment
142
133
  # returns String values by default.
143
134
  #
144
- # Person.pluck(:id)
145
- # # SELECT people.id FROM people
146
- # # => [1, 2, 3]
135
+ # Person.pluck(:name)
136
+ # # SELECT people.name FROM people
137
+ # # => ['David', 'Jeremy', 'Jose']
147
138
  #
148
139
  # Person.pluck(:id, :name)
149
140
  # # SELECT people.id, people.name FROM people
150
141
  # # => [[1, 'David'], [2, 'Jeremy'], [3, 'Jose']]
151
142
  #
152
- # Person.pluck('DISTINCT role')
143
+ # Person.distinct.pluck(:role)
153
144
  # # SELECT DISTINCT role FROM people
154
145
  # # => ['admin', 'member', 'guest']
155
146
  #
@@ -161,13 +152,11 @@ module ActiveRecord
161
152
  # # SELECT DATEDIFF(updated_at, created_at) FROM people
162
153
  # # => ['0', '27761', '173']
163
154
  #
155
+ # See also #ids.
156
+ #
164
157
  def pluck(*column_names)
165
- column_names.map! do |column_name|
166
- if column_name.is_a?(Symbol) && attribute_alias?(column_name)
167
- attribute_alias(column_name)
168
- else
169
- column_name.to_s
170
- end
158
+ if loaded? && (column_names.map(&:to_s) - @klass.attribute_names - @klass.attribute_aliases.keys).empty?
159
+ return @records.pluck(*column_names)
171
160
  end
172
161
 
173
162
  if has_include?(column_names.first)
@@ -175,10 +164,10 @@ module ActiveRecord
175
164
  else
176
165
  relation = spawn
177
166
  relation.select_values = column_names.map { |cn|
178
- columns_hash.key?(cn) ? arel_table[cn] : cn
167
+ @klass.has_attribute?(cn) || @klass.attribute_alias?(cn) ? arel_attribute(cn) : cn
179
168
  }
180
- result = klass.connection.select_all(relation.arel, nil, relation.arel.bind_values + bind_values)
181
- result.cast_values(klass.column_types)
169
+ result = klass.connection.select_all(relation.arel, nil, bound_attributes)
170
+ result.cast_values(klass.attribute_types)
182
171
  end
183
172
  end
184
173
 
@@ -193,15 +182,14 @@ module ActiveRecord
193
182
  private
194
183
 
195
184
  def has_include?(column_name)
196
- eager_loading? || (includes_values.present? && ((column_name && column_name != :all) || references_eager_loaded_tables?))
185
+ eager_loading? || (includes_values.present? && column_name && column_name != :all)
197
186
  end
198
187
 
199
- def perform_calculation(operation, column_name, options = {})
200
- # TODO: Remove options argument as soon we remove support to
201
- # activerecord-deprecated_finders.
188
+ def perform_calculation(operation, column_name)
202
189
  operation = operation.to_s.downcase
203
190
 
204
- # If #count is used with #distinct / #uniq it is considered distinct. (eg. relation.distinct.count)
191
+ # If #count is used with #distinct (i.e. `relation.distinct.count`) it is
192
+ # considered distinct.
205
193
  distinct = self.distinct_value
206
194
 
207
195
  if operation == "count"
@@ -223,6 +211,8 @@ module ActiveRecord
223
211
  end
224
212
 
225
213
  def aggregate_column(column_name)
214
+ return column_name if Arel::Expressions === column_name
215
+
226
216
  if @klass.column_names.include?(column_name.to_s)
227
217
  Arel::Attribute.new(@klass.unscoped.table, column_name)
228
218
  else
@@ -235,19 +225,16 @@ module ActiveRecord
235
225
  end
236
226
 
237
227
  def execute_simple_calculation(operation, column_name, distinct) #:nodoc:
238
- # Postgresql doesn't like ORDER BY when there are no GROUP BY
228
+ # PostgreSQL doesn't like ORDER BY when there are no GROUP BY
239
229
  relation = unscope(:order)
240
230
 
241
231
  column_alias = column_name
242
232
 
243
- bind_values = nil
244
-
245
233
  if operation == "count" && (relation.limit_value || relation.offset_value)
246
234
  # Shortcut when limit is zero.
247
235
  return 0 if relation.limit_value == 0
248
236
 
249
237
  query_builder = build_count_subquery(relation, column_name, distinct)
250
- bind_values = query_builder.bind_values + relation.bind_values
251
238
  else
252
239
  column = aggregate_column(column_name)
253
240
 
@@ -258,10 +245,9 @@ module ActiveRecord
258
245
  relation.select_values = [select_value]
259
246
 
260
247
  query_builder = relation.arel
261
- bind_values = query_builder.bind_values + relation.bind_values
262
248
  end
263
249
 
264
- result = @klass.connection.select_all(query_builder, nil, bind_values)
250
+ result = @klass.connection.select_all(query_builder, nil, bound_attributes)
265
251
  row = result.first
266
252
  value = row && row.values.first
267
253
  column = result.column_types.fetch(column_alias) do
@@ -281,15 +267,10 @@ module ActiveRecord
281
267
  else
282
268
  group_fields = group_attrs
283
269
  end
270
+ group_fields = arel_columns(group_fields)
284
271
 
285
- group_aliases = group_fields.map { |field|
286
- column_alias_for(field)
287
- }
288
- group_columns = group_aliases.zip(group_fields).map { |aliaz,field|
289
- [aliaz, field]
290
- }
291
-
292
- group = group_fields
272
+ group_aliases = group_fields.map { |field| column_alias_for(field) }
273
+ group_columns = group_aliases.zip(group_fields)
293
274
 
294
275
  if operation == 'count' && column_name == :all
295
276
  aggregate_alias = 'count_all'
@@ -303,9 +284,9 @@ module ActiveRecord
303
284
  operation,
304
285
  distinct).as(aggregate_alias)
305
286
  ]
306
- select_values += select_values unless having_values.empty?
287
+ select_values += select_values unless having_clause.empty?
307
288
 
308
- select_values.concat group_fields.zip(group_aliases).map { |field,aliaz|
289
+ select_values.concat group_columns.map { |aliaz, field|
309
290
  if field.respond_to?(:as)
310
291
  field.as(aliaz)
311
292
  else
@@ -314,14 +295,14 @@ module ActiveRecord
314
295
  }
315
296
 
316
297
  relation = except(:group)
317
- relation.group_values = group
298
+ relation.group_values = group_fields
318
299
  relation.select_values = select_values
319
300
 
320
- calculated_data = @klass.connection.select_all(relation, nil, relation.arel.bind_values + bind_values)
301
+ calculated_data = @klass.connection.select_all(relation, nil, relation.bound_attributes)
321
302
 
322
303
  if association
323
304
  key_ids = calculated_data.collect { |row| row[group_aliases.first] }
324
- key_records = association.klass.base_class.find(key_ids)
305
+ key_records = association.klass.base_class.where(association.klass.base_class.primary_key => key_ids)
325
306
  key_records = Hash[key_records.map { |r| [r.id, r] }]
326
307
  end
327
308
 
@@ -347,7 +328,6 @@ module ActiveRecord
347
328
  # column_alias_for("sum(id)") # => "sum_id"
348
329
  # column_alias_for("count(distinct users.id)") # => "count_distinct_users_id"
349
330
  # column_alias_for("count(*)") # => "count_all"
350
- # column_alias_for("count", "id") # => "count_id"
351
331
  def column_alias_for(keys)
352
332
  if keys.respond_to? :name
353
333
  keys = "#{keys.relation.name}.#{keys.name}"
@@ -370,15 +350,15 @@ module ActiveRecord
370
350
  def type_cast_calculated_value(value, type, operation = nil)
371
351
  case operation
372
352
  when 'count' then value.to_i
373
- when 'sum' then type.type_cast_from_database(value || 0)
353
+ when 'sum' then type.deserialize(value || 0)
374
354
  when 'average' then value.respond_to?(:to_d) ? value.to_d : value
375
- else type.type_cast_from_database(value)
355
+ else type.deserialize(value)
376
356
  end
377
357
  end
378
358
 
379
- # TODO: refactor to allow non-string `select_values` (eg. Arel nodes).
380
359
  def select_for_count
381
360
  if select_values.present?
361
+ return select_values.first if select_values.one?
382
362
  select_values.join(", ")
383
363
  else
384
364
  :all
@@ -391,11 +371,9 @@ module ActiveRecord
391
371
 
392
372
  aliased_column = aggregate_column(column_name == :all ? 1 : column_name).as(column_alias)
393
373
  relation.select_values = [aliased_column]
394
- arel = relation.arel
395
- subquery = arel.as(subquery_alias)
374
+ subquery = relation.arel.as(subquery_alias)
396
375
 
397
376
  sm = Arel::SelectManager.new relation.engine
398
- sm.bind_values = arel.bind_values
399
377
  select_value = operation_over_aggregate_column(column_alias, 'count', distinct)
400
378
  sm.project(select_value).from(subquery)
401
379
  end