activerecord 4.2.11.3 → 5.0.0.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of activerecord might be problematic. Click here for more details.

Files changed (246) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +1281 -1204
  3. data/MIT-LICENSE +2 -2
  4. data/README.rdoc +7 -8
  5. data/examples/performance.rb +2 -3
  6. data/examples/simple.rb +0 -1
  7. data/lib/active_record/aggregations.rb +35 -24
  8. data/lib/active_record/association_relation.rb +3 -3
  9. data/lib/active_record/associations/alias_tracker.rb +19 -16
  10. data/lib/active_record/associations/association.rb +11 -9
  11. data/lib/active_record/associations/association_scope.rb +73 -102
  12. data/lib/active_record/associations/belongs_to_association.rb +21 -32
  13. data/lib/active_record/associations/builder/association.rb +28 -34
  14. data/lib/active_record/associations/builder/belongs_to.rb +43 -18
  15. data/lib/active_record/associations/builder/collection_association.rb +7 -19
  16. data/lib/active_record/associations/builder/has_and_belongs_to_many.rb +14 -11
  17. data/lib/active_record/associations/builder/has_many.rb +4 -4
  18. data/lib/active_record/associations/builder/has_one.rb +11 -6
  19. data/lib/active_record/associations/builder/singular_association.rb +3 -10
  20. data/lib/active_record/associations/collection_association.rb +49 -41
  21. data/lib/active_record/associations/collection_proxy.rb +67 -27
  22. data/lib/active_record/associations/foreign_association.rb +1 -1
  23. data/lib/active_record/associations/has_many_association.rb +20 -71
  24. data/lib/active_record/associations/has_many_through_association.rb +8 -47
  25. data/lib/active_record/associations/has_one_association.rb +12 -5
  26. data/lib/active_record/associations/join_dependency/join_association.rb +16 -10
  27. data/lib/active_record/associations/join_dependency.rb +29 -19
  28. data/lib/active_record/associations/preloader/association.rb +46 -52
  29. data/lib/active_record/associations/preloader/collection_association.rb +0 -6
  30. data/lib/active_record/associations/preloader/has_many_through.rb +1 -1
  31. data/lib/active_record/associations/preloader/has_one.rb +0 -8
  32. data/lib/active_record/associations/preloader/through_association.rb +27 -14
  33. data/lib/active_record/associations/preloader.rb +14 -4
  34. data/lib/active_record/associations/singular_association.rb +7 -1
  35. data/lib/active_record/associations/through_association.rb +11 -3
  36. data/lib/active_record/associations.rb +317 -209
  37. data/lib/active_record/attribute/user_provided_default.rb +28 -0
  38. data/lib/active_record/attribute.rb +68 -18
  39. data/lib/active_record/attribute_assignment.rb +19 -140
  40. data/lib/active_record/attribute_decorators.rb +6 -5
  41. data/lib/active_record/attribute_methods/before_type_cast.rb +1 -1
  42. data/lib/active_record/attribute_methods/dirty.rb +46 -86
  43. data/lib/active_record/attribute_methods/primary_key.rb +2 -2
  44. data/lib/active_record/attribute_methods/query.rb +2 -2
  45. data/lib/active_record/attribute_methods/read.rb +31 -59
  46. data/lib/active_record/attribute_methods/serialization.rb +13 -16
  47. data/lib/active_record/attribute_methods/time_zone_conversion.rb +61 -14
  48. data/lib/active_record/attribute_methods/write.rb +13 -37
  49. data/lib/active_record/attribute_methods.rb +76 -47
  50. data/lib/active_record/attribute_mutation_tracker.rb +70 -0
  51. data/lib/active_record/attribute_set/builder.rb +6 -4
  52. data/lib/active_record/attribute_set.rb +30 -3
  53. data/lib/active_record/attributes.rb +199 -81
  54. data/lib/active_record/autosave_association.rb +49 -16
  55. data/lib/active_record/base.rb +32 -23
  56. data/lib/active_record/callbacks.rb +39 -43
  57. data/lib/active_record/coders/json.rb +1 -1
  58. data/lib/active_record/coders/yaml_column.rb +20 -8
  59. data/lib/active_record/collection_cache_key.rb +40 -0
  60. data/lib/active_record/connection_adapters/abstract/connection_pool.rb +452 -182
  61. data/lib/active_record/connection_adapters/abstract/database_limits.rb +3 -3
  62. data/lib/active_record/connection_adapters/abstract/database_statements.rb +65 -61
  63. data/lib/active_record/connection_adapters/abstract/query_cache.rb +2 -2
  64. data/lib/active_record/connection_adapters/abstract/quoting.rb +74 -10
  65. data/lib/active_record/connection_adapters/abstract/savepoints.rb +3 -3
  66. data/lib/active_record/connection_adapters/abstract/schema_creation.rb +61 -39
  67. data/lib/active_record/connection_adapters/abstract/schema_definitions.rb +236 -185
  68. data/lib/active_record/connection_adapters/abstract/schema_dumper.rb +72 -17
  69. data/lib/active_record/connection_adapters/abstract/schema_statements.rb +380 -141
  70. data/lib/active_record/connection_adapters/abstract/transaction.rb +51 -34
  71. data/lib/active_record/connection_adapters/abstract_adapter.rb +141 -59
  72. data/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +401 -370
  73. data/lib/active_record/connection_adapters/column.rb +28 -43
  74. data/lib/active_record/connection_adapters/connection_specification.rb +15 -27
  75. data/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb +22 -0
  76. data/lib/active_record/connection_adapters/mysql/column.rb +50 -0
  77. data/lib/active_record/connection_adapters/mysql/database_statements.rb +125 -0
  78. data/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb +70 -0
  79. data/lib/active_record/connection_adapters/mysql/quoting.rb +51 -0
  80. data/lib/active_record/connection_adapters/mysql/schema_creation.rb +67 -0
  81. data/lib/active_record/connection_adapters/mysql/schema_definitions.rb +93 -0
  82. data/lib/active_record/connection_adapters/mysql/schema_dumper.rb +54 -0
  83. data/lib/active_record/connection_adapters/mysql/type_metadata.rb +32 -0
  84. data/lib/active_record/connection_adapters/mysql2_adapter.rb +29 -166
  85. data/lib/active_record/connection_adapters/postgresql/column.rb +5 -10
  86. data/lib/active_record/connection_adapters/postgresql/database_statements.rb +10 -72
  87. data/lib/active_record/connection_adapters/postgresql/explain_pretty_printer.rb +42 -0
  88. data/lib/active_record/connection_adapters/postgresql/oid/array.rb +27 -57
  89. data/lib/active_record/connection_adapters/postgresql/oid/bit.rb +2 -2
  90. data/lib/active_record/connection_adapters/postgresql/oid/bytea.rb +1 -1
  91. data/lib/active_record/connection_adapters/postgresql/oid/cidr.rb +3 -1
  92. data/lib/active_record/connection_adapters/postgresql/oid/date_time.rb +7 -22
  93. data/lib/active_record/connection_adapters/postgresql/oid/hstore.rb +3 -3
  94. data/lib/active_record/connection_adapters/postgresql/oid/json.rb +1 -26
  95. data/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb +2 -2
  96. data/lib/active_record/connection_adapters/postgresql/oid/money.rb +0 -4
  97. data/lib/active_record/connection_adapters/postgresql/oid/point.rb +4 -4
  98. data/lib/active_record/connection_adapters/postgresql/oid/rails_5_1_point.rb +50 -0
  99. data/lib/active_record/connection_adapters/postgresql/oid/range.rb +31 -17
  100. data/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb +0 -4
  101. data/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +2 -2
  102. data/lib/active_record/connection_adapters/postgresql/oid/vector.rb +1 -1
  103. data/lib/active_record/connection_adapters/postgresql/oid/xml.rb +1 -1
  104. data/lib/active_record/connection_adapters/postgresql/oid.rb +1 -6
  105. data/lib/active_record/connection_adapters/postgresql/quoting.rb +26 -18
  106. data/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +29 -10
  107. data/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +107 -79
  108. data/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +47 -0
  109. data/lib/active_record/connection_adapters/postgresql/schema_statements.rb +234 -148
  110. data/lib/active_record/connection_adapters/postgresql/type_metadata.rb +35 -0
  111. data/lib/active_record/connection_adapters/postgresql_adapter.rb +248 -160
  112. data/lib/active_record/connection_adapters/schema_cache.rb +36 -23
  113. data/lib/active_record/connection_adapters/sql_type_metadata.rb +32 -0
  114. data/lib/active_record/connection_adapters/sqlite3/explain_pretty_printer.rb +19 -0
  115. data/lib/active_record/connection_adapters/sqlite3/quoting.rb +48 -0
  116. data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +22 -0
  117. data/lib/active_record/connection_adapters/sqlite3_adapter.rb +149 -192
  118. data/lib/active_record/connection_adapters/statement_pool.rb +31 -12
  119. data/lib/active_record/connection_handling.rb +37 -14
  120. data/lib/active_record/core.rb +89 -107
  121. data/lib/active_record/counter_cache.rb +13 -24
  122. data/lib/active_record/dynamic_matchers.rb +1 -20
  123. data/lib/active_record/enum.rb +113 -76
  124. data/lib/active_record/errors.rb +87 -48
  125. data/lib/active_record/explain_registry.rb +1 -1
  126. data/lib/active_record/explain_subscriber.rb +1 -1
  127. data/lib/active_record/fixture_set/file.rb +26 -5
  128. data/lib/active_record/fixtures.rb +76 -40
  129. data/lib/active_record/gem_version.rb +4 -4
  130. data/lib/active_record/inheritance.rb +32 -40
  131. data/lib/active_record/integration.rb +4 -4
  132. data/lib/active_record/internal_metadata.rb +56 -0
  133. data/lib/active_record/legacy_yaml_adapter.rb +18 -2
  134. data/lib/active_record/locale/en.yml +3 -2
  135. data/lib/active_record/locking/optimistic.rb +15 -15
  136. data/lib/active_record/locking/pessimistic.rb +1 -1
  137. data/lib/active_record/log_subscriber.rb +43 -21
  138. data/lib/active_record/migration/command_recorder.rb +59 -18
  139. data/lib/active_record/migration/compatibility.rb +126 -0
  140. data/lib/active_record/migration.rb +363 -133
  141. data/lib/active_record/model_schema.rb +129 -41
  142. data/lib/active_record/nested_attributes.rb +58 -29
  143. data/lib/active_record/null_relation.rb +16 -8
  144. data/lib/active_record/persistence.rb +121 -80
  145. data/lib/active_record/query_cache.rb +15 -18
  146. data/lib/active_record/querying.rb +10 -9
  147. data/lib/active_record/railtie.rb +23 -16
  148. data/lib/active_record/railties/controller_runtime.rb +1 -1
  149. data/lib/active_record/railties/databases.rake +69 -46
  150. data/lib/active_record/readonly_attributes.rb +1 -1
  151. data/lib/active_record/reflection.rb +282 -115
  152. data/lib/active_record/relation/batches/batch_enumerator.rb +67 -0
  153. data/lib/active_record/relation/batches.rb +139 -34
  154. data/lib/active_record/relation/calculations.rb +79 -108
  155. data/lib/active_record/relation/delegation.rb +7 -20
  156. data/lib/active_record/relation/finder_methods.rb +163 -81
  157. data/lib/active_record/relation/from_clause.rb +32 -0
  158. data/lib/active_record/relation/merger.rb +16 -42
  159. data/lib/active_record/relation/predicate_builder/array_handler.rb +11 -16
  160. data/lib/active_record/relation/predicate_builder/association_query_handler.rb +88 -0
  161. data/lib/active_record/relation/predicate_builder/base_handler.rb +17 -0
  162. data/lib/active_record/relation/predicate_builder/basic_object_handler.rb +17 -0
  163. data/lib/active_record/relation/predicate_builder/class_handler.rb +27 -0
  164. data/lib/active_record/relation/predicate_builder/polymorphic_array_handler.rb +57 -0
  165. data/lib/active_record/relation/predicate_builder/range_handler.rb +33 -0
  166. data/lib/active_record/relation/predicate_builder/relation_handler.rb +1 -1
  167. data/lib/active_record/relation/predicate_builder.rb +120 -107
  168. data/lib/active_record/relation/query_attribute.rb +19 -0
  169. data/lib/active_record/relation/query_methods.rb +308 -244
  170. data/lib/active_record/relation/record_fetch_warning.rb +49 -0
  171. data/lib/active_record/relation/spawn_methods.rb +4 -7
  172. data/lib/active_record/relation/where_clause.rb +174 -0
  173. data/lib/active_record/relation/where_clause_factory.rb +38 -0
  174. data/lib/active_record/relation.rb +176 -116
  175. data/lib/active_record/result.rb +4 -3
  176. data/lib/active_record/runtime_registry.rb +1 -1
  177. data/lib/active_record/sanitization.rb +95 -66
  178. data/lib/active_record/schema.rb +26 -22
  179. data/lib/active_record/schema_dumper.rb +62 -38
  180. data/lib/active_record/schema_migration.rb +11 -14
  181. data/lib/active_record/scoping/default.rb +23 -9
  182. data/lib/active_record/scoping/named.rb +49 -28
  183. data/lib/active_record/scoping.rb +32 -15
  184. data/lib/active_record/secure_token.rb +38 -0
  185. data/lib/active_record/serialization.rb +2 -4
  186. data/lib/active_record/statement_cache.rb +16 -14
  187. data/lib/active_record/store.rb +8 -3
  188. data/lib/active_record/suppressor.rb +58 -0
  189. data/lib/active_record/table_metadata.rb +68 -0
  190. data/lib/active_record/tasks/database_tasks.rb +57 -43
  191. data/lib/active_record/tasks/mysql_database_tasks.rb +6 -14
  192. data/lib/active_record/tasks/postgresql_database_tasks.rb +11 -2
  193. data/lib/active_record/tasks/sqlite_database_tasks.rb +5 -1
  194. data/lib/active_record/timestamp.rb +20 -9
  195. data/lib/active_record/touch_later.rb +58 -0
  196. data/lib/active_record/transactions.rb +138 -56
  197. data/lib/active_record/type/adapter_specific_registry.rb +130 -0
  198. data/lib/active_record/type/date.rb +2 -45
  199. data/lib/active_record/type/date_time.rb +2 -49
  200. data/lib/active_record/type/internal/abstract_json.rb +29 -0
  201. data/lib/active_record/type/internal/timezone.rb +15 -0
  202. data/lib/active_record/type/serialized.rb +15 -14
  203. data/lib/active_record/type/time.rb +10 -16
  204. data/lib/active_record/type/type_map.rb +4 -4
  205. data/lib/active_record/type.rb +66 -17
  206. data/lib/active_record/type_caster/connection.rb +29 -0
  207. data/lib/active_record/type_caster/map.rb +19 -0
  208. data/lib/active_record/type_caster.rb +7 -0
  209. data/lib/active_record/validations/absence.rb +23 -0
  210. data/lib/active_record/validations/associated.rb +10 -3
  211. data/lib/active_record/validations/length.rb +24 -0
  212. data/lib/active_record/validations/presence.rb +11 -12
  213. data/lib/active_record/validations/uniqueness.rb +30 -29
  214. data/lib/active_record/validations.rb +33 -32
  215. data/lib/active_record.rb +8 -4
  216. data/lib/rails/generators/active_record/migration/migration_generator.rb +7 -4
  217. data/lib/rails/generators/active_record/migration/templates/create_table_migration.rb +8 -3
  218. data/lib/rails/generators/active_record/migration/templates/migration.rb +8 -1
  219. data/lib/rails/generators/active_record/migration.rb +7 -0
  220. data/lib/rails/generators/active_record/model/model_generator.rb +32 -15
  221. data/lib/rails/generators/active_record/model/templates/application_record.rb +5 -0
  222. data/lib/rails/generators/active_record/model/templates/model.rb +3 -0
  223. metadata +60 -34
  224. data/lib/active_record/connection_adapters/mysql_adapter.rb +0 -498
  225. data/lib/active_record/connection_adapters/postgresql/array_parser.rb +0 -93
  226. data/lib/active_record/connection_adapters/postgresql/oid/date.rb +0 -11
  227. data/lib/active_record/connection_adapters/postgresql/oid/float.rb +0 -21
  228. data/lib/active_record/connection_adapters/postgresql/oid/infinity.rb +0 -13
  229. data/lib/active_record/connection_adapters/postgresql/oid/integer.rb +0 -11
  230. data/lib/active_record/connection_adapters/postgresql/oid/time.rb +0 -11
  231. data/lib/active_record/serializers/xml_serializer.rb +0 -193
  232. data/lib/active_record/type/big_integer.rb +0 -13
  233. data/lib/active_record/type/binary.rb +0 -50
  234. data/lib/active_record/type/boolean.rb +0 -31
  235. data/lib/active_record/type/decimal.rb +0 -64
  236. data/lib/active_record/type/decimal_without_scale.rb +0 -11
  237. data/lib/active_record/type/decorator.rb +0 -14
  238. data/lib/active_record/type/float.rb +0 -19
  239. data/lib/active_record/type/integer.rb +0 -59
  240. data/lib/active_record/type/mutable.rb +0 -16
  241. data/lib/active_record/type/numeric.rb +0 -36
  242. data/lib/active_record/type/string.rb +0 -40
  243. data/lib/active_record/type/text.rb +0 -11
  244. data/lib/active_record/type/time_value.rb +0 -38
  245. data/lib/active_record/type/unsigned_integer.rb +0 -15
  246. data/lib/active_record/type/value.rb +0 -110
@@ -1,8 +1,12 @@
1
+ require "active_record/relation/batches/batch_enumerator"
2
+
1
3
  module ActiveRecord
2
4
  module Batches
5
+ ORDER_OR_LIMIT_IGNORED_MESSAGE = "Scoped order and limit are ignored, it's forced to be batch order and batch size."
6
+
3
7
  # Looping through a collection of records from the database
4
- # (using the +all+ method, for example) is very inefficient
5
- # since it will try to instantiate all the objects at once.
8
+ # (using the Scoping::Named::ClassMethods.all method, for example)
9
+ # is very inefficient since it will try to instantiate all the objects at once.
6
10
  #
7
11
  # In that case, batch processing methods allow you to work
8
12
  # with the records in batches, thereby greatly reducing memory consumption.
@@ -27,11 +31,15 @@ module ActiveRecord
27
31
  #
28
32
  # ==== Options
29
33
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000.
30
- # * <tt>:start</tt> - Specifies the starting point for the batch processing.
34
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
35
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
36
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
37
+ # the order and limit have to be ignored due to batching.
38
+ #
31
39
  # This is especially useful if you want multiple workers dealing with
32
40
  # the same processing queue. You can make worker 1 handle all the records
33
41
  # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond
34
- # (by setting the +:start+ option on that worker).
42
+ # (by setting the +:start+ and +:finish+ option on each worker).
35
43
  #
36
44
  # # Let's process for a batch of 2000 records, skipping the first 2000 rows
37
45
  # Person.find_each(start: 2000, batch_size: 2000) do |person|
@@ -40,24 +48,25 @@ module ActiveRecord
40
48
  #
41
49
  # NOTE: It's not possible to set the order. That is automatically set to
42
50
  # ascending on the primary key ("id ASC") to make the batch ordering
43
- # work. This also means that this method only works with integer-based
44
- # primary keys.
51
+ # work. This also means that this method only works when the primary key is
52
+ # orderable (e.g. an integer or string).
45
53
  #
46
54
  # NOTE: You can't set the limit either, that's used to control
47
55
  # the batch sizes.
48
- def find_each(options = {})
56
+ def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
49
57
  if block_given?
50
- find_in_batches(options) do |records|
58
+ find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do |records|
51
59
  records.each { |record| yield record }
52
60
  end
53
61
  else
54
- enum_for :find_each, options do
55
- options[:start] ? where(table[primary_key].gteq(options[:start])).size : size
62
+ enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
63
+ relation = self
64
+ apply_limits(relation, start, finish).size
56
65
  end
57
66
  end
58
67
  end
59
68
 
60
- # Yields each batch of records that was found by the find +options+ as
69
+ # Yields each batch of records that was found by the find options as
61
70
  # an array.
62
71
  #
63
72
  # Person.where("age > 21").find_in_batches do |group|
@@ -77,11 +86,15 @@ module ActiveRecord
77
86
  #
78
87
  # ==== Options
79
88
  # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000.
80
- # * <tt>:start</tt> - Specifies the starting point for the batch processing.
89
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
90
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
91
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
92
+ # the order and limit have to be ignored due to batching.
93
+ #
81
94
  # This is especially useful if you want multiple workers dealing with
82
95
  # the same processing queue. You can make worker 1 handle all the records
83
96
  # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond
84
- # (by setting the +:start+ option on that worker).
97
+ # (by setting the +:start+ and +:finish+ option on each worker).
85
98
  #
86
99
  # # Let's process the next 2000 records
87
100
  # Person.find_in_batches(start: 2000, batch_size: 2000) do |group|
@@ -90,49 +103,141 @@ module ActiveRecord
90
103
  #
91
104
  # NOTE: It's not possible to set the order. That is automatically set to
92
105
  # ascending on the primary key ("id ASC") to make the batch ordering
93
- # work. This also means that this method only works with integer-based
94
- # primary keys.
106
+ # work. This also means that this method only works when the primary key is
107
+ # orderable (e.g. an integer or string).
95
108
  #
96
109
  # NOTE: You can't set the limit either, that's used to control
97
110
  # the batch sizes.
98
- def find_in_batches(options = {})
99
- options.assert_valid_keys(:start, :batch_size)
100
-
111
+ def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
101
112
  relation = self
102
- start = options[:start]
103
- batch_size = options[:batch_size] || 1000
104
-
105
113
  unless block_given?
106
- return to_enum(:find_in_batches, options) do
107
- total = start ? where(table[primary_key].gteq(start)).size : size
114
+ return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
115
+ total = apply_limits(relation, start, finish).size
108
116
  (total - 1).div(batch_size) + 1
109
117
  end
110
118
  end
111
119
 
112
- if logger && (arel.orders.present? || arel.taken.present?)
113
- logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
120
+ in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore) do |batch|
121
+ yield batch.to_a
114
122
  end
123
+ end
115
124
 
116
- relation = relation.reorder(batch_order).limit(batch_size)
117
- records = start ? relation.where(table[primary_key].gteq(start)).to_a : relation.to_a
125
+ # Yields ActiveRecord::Relation objects to work with a batch of records.
126
+ #
127
+ # Person.where("age > 21").in_batches do |relation|
128
+ # relation.delete_all
129
+ # sleep(10) # Throttle the delete queries
130
+ # end
131
+ #
132
+ # If you do not provide a block to #in_batches, it will return a
133
+ # BatchEnumerator which is enumerable.
134
+ #
135
+ # Person.in_batches.with_index do |relation, batch_index|
136
+ # puts "Processing relation ##{batch_index}"
137
+ # relation.each { |relation| relation.delete_all }
138
+ # end
139
+ #
140
+ # Examples of calling methods on the returned BatchEnumerator object:
141
+ #
142
+ # Person.in_batches.delete_all
143
+ # Person.in_batches.update_all(awesome: true)
144
+ # Person.in_batches.each_record(&:party_all_night!)
145
+ #
146
+ # ==== Options
147
+ # * <tt>:of</tt> - Specifies the size of the batch. Default to 1000.
148
+ # * <tt>:load</tt> - Specifies if the relation should be loaded. Default to false.
149
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
150
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
151
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
152
+ # the order and limit have to be ignored due to batching.
153
+ #
154
+ # This is especially useful if you want to work with the
155
+ # ActiveRecord::Relation object instead of the array of records, or if
156
+ # you want multiple workers dealing with the same processing queue. You can
157
+ # make worker 1 handle all the records between id 0 and 10,000 and worker 2
158
+ # handle from 10,000 and beyond (by setting the +:start+ and +:finish+
159
+ # option on each worker).
160
+ #
161
+ # # Let's process the next 2000 records
162
+ # Person.in_batches(of: 2000, start: 2000).update_all(awesome: true)
163
+ #
164
+ # An example of calling where query method on the relation:
165
+ #
166
+ # Person.in_batches.each do |relation|
167
+ # relation.update_all('age = age + 1')
168
+ # relation.where('age > 21').update_all(should_party: true)
169
+ # relation.where('age <= 21').delete_all
170
+ # end
171
+ #
172
+ # NOTE: If you are going to iterate through each record, you should call
173
+ # #each_record on the yielded BatchEnumerator:
174
+ #
175
+ # Person.in_batches.each_record(&:party_all_night!)
176
+ #
177
+ # NOTE: It's not possible to set the order. That is automatically set to
178
+ # ascending on the primary key ("id ASC") to make the batch ordering
179
+ # consistent. Therefore the primary key must be orderable, e.g an integer
180
+ # or a string.
181
+ #
182
+ # NOTE: You can't set the limit either, that's used to control the batch
183
+ # sizes.
184
+ def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil)
185
+ relation = self
186
+ unless block_given?
187
+ return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self)
188
+ end
189
+
190
+ if arel.orders.present? || arel.taken.present?
191
+ act_on_order_or_limit_ignored(error_on_ignore)
192
+ end
193
+
194
+ relation = relation.reorder(batch_order).limit(of)
195
+ relation = apply_limits(relation, start, finish)
196
+ batch_relation = relation
118
197
 
119
- while records.any?
120
- records_size = records.size
121
- primary_key_offset = records.last.id
122
- raise "Primary key not included in the custom select clause" unless primary_key_offset
198
+ loop do
199
+ if load
200
+ records = batch_relation.records
201
+ ids = records.map(&:id)
202
+ yielded_relation = self.where(primary_key => ids)
203
+ yielded_relation.load_records(records)
204
+ else
205
+ ids = batch_relation.pluck(primary_key)
206
+ yielded_relation = self.where(primary_key => ids)
207
+ end
123
208
 
124
- yield records
209
+ break if ids.empty?
125
210
 
126
- break if records_size < batch_size
211
+ primary_key_offset = ids.last
212
+ raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
127
213
 
128
- records = relation.where(table[primary_key].gt(primary_key_offset)).to_a
214
+ yield yielded_relation
215
+
216
+ break if ids.length < of
217
+ batch_relation = relation.where(arel_attribute(primary_key).gt(primary_key_offset))
129
218
  end
130
219
  end
131
220
 
132
221
  private
133
222
 
223
+ def apply_limits(relation, start, finish)
224
+ relation = relation.where(arel_attribute(primary_key).gteq(start)) if start
225
+ relation = relation.where(arel_attribute(primary_key).lteq(finish)) if finish
226
+ relation
227
+ end
228
+
134
229
  def batch_order
135
230
  "#{quoted_table_name}.#{quoted_primary_key} ASC"
136
231
  end
232
+
233
+ def act_on_order_or_limit_ignored(error_on_ignore)
234
+ raise_error = (error_on_ignore.nil? ? self.klass.error_on_ignored_order_or_limit : error_on_ignore)
235
+
236
+ if raise_error
237
+ raise ArgumentError.new(ORDER_OR_LIMIT_IGNORED_MESSAGE)
238
+ elsif logger
239
+ logger.warn(ORDER_OR_LIMIT_IGNORED_MESSAGE)
240
+ end
241
+ end
137
242
  end
138
243
  end
@@ -14,127 +14,112 @@ module ActiveRecord
14
14
  # Person.distinct.count(:age)
15
15
  # # => counts the number of different age values
16
16
  #
17
- # If +count+ is used with +group+, it returns a Hash whose keys represent the aggregated column,
17
+ # If #count is used with {Relation#group}[rdoc-ref:QueryMethods#group],
18
+ # it returns a Hash whose keys represent the aggregated column,
18
19
  # and the values are the respective amounts:
19
20
  #
20
21
  # Person.group(:city).count
21
22
  # # => { 'Rome' => 5, 'Paris' => 3 }
22
23
  #
23
- # If +count+ is used with +group+ for multiple columns, it returns a Hash whose
24
+ # If #count is used with {Relation#group}[rdoc-ref:QueryMethods#group] for multiple columns, it returns a Hash whose
24
25
  # keys are an array containing the individual values of each column and the value
25
- # of each key would be the +count+.
26
+ # of each key would be the #count.
26
27
  #
27
28
  # Article.group(:status, :category).count
28
29
  # # => {["draft", "business"]=>10, ["draft", "technology"]=>4,
29
30
  # ["published", "business"]=>0, ["published", "technology"]=>2}
30
31
  #
31
- # If +count+ is used with +select+, it will count the selected columns:
32
+ # If #count is used with {Relation#select}[rdoc-ref:QueryMethods#select], it will count the selected columns:
32
33
  #
33
34
  # Person.select(:age).count
34
35
  # # => counts the number of different age values
35
36
  #
36
- # Note: not all valid +select+ expressions are valid +count+ expressions. The specifics differ
37
+ # Note: not all valid {Relation#select}[rdoc-ref:QueryMethods#select] expressions are valid #count expressions. The specifics differ
37
38
  # between databases. In invalid cases, an error from the database is thrown.
38
- def count(column_name = nil, options = {})
39
- if options.present? && !ActiveRecord.const_defined?(:DeprecatedFinders)
40
- raise ArgumentError, "Relation#count does not support finder options anymore. " \
41
- "Please build a scope and then call count on it or use the " \
42
- "activerecord-deprecated_finders gem to enable this functionality."
43
-
44
- end
45
-
46
- # TODO: Remove options argument as soon we remove support to
47
- # activerecord-deprecated_finders.
48
- calculate(:count, column_name, options)
39
+ def count(column_name = nil)
40
+ calculate(:count, column_name)
49
41
  end
50
42
 
51
43
  # Calculates the average value on a given column. Returns +nil+ if there's
52
- # no row. See +calculate+ for examples with options.
44
+ # no row. See #calculate for examples with options.
53
45
  #
54
46
  # Person.average(:age) # => 35.8
55
- def average(column_name, options = {})
56
- # TODO: Remove options argument as soon we remove support to
57
- # activerecord-deprecated_finders.
58
- calculate(:average, column_name, options)
47
+ def average(column_name)
48
+ calculate(:average, column_name)
59
49
  end
60
50
 
61
51
  # Calculates the minimum value on a given column. The value is returned
62
52
  # with the same data type of the column, or +nil+ if there's no row. See
63
- # +calculate+ for examples with options.
53
+ # #calculate for examples with options.
64
54
  #
65
55
  # Person.minimum(:age) # => 7
66
- def minimum(column_name, options = {})
67
- # TODO: Remove options argument as soon we remove support to
68
- # activerecord-deprecated_finders.
69
- calculate(:minimum, column_name, options)
56
+ def minimum(column_name)
57
+ calculate(:minimum, column_name)
70
58
  end
71
59
 
72
60
  # Calculates the maximum value on a given column. The value is returned
73
61
  # with the same data type of the column, or +nil+ if there's no row. See
74
- # +calculate+ for examples with options.
62
+ # #calculate for examples with options.
75
63
  #
76
64
  # Person.maximum(:age) # => 93
77
- def maximum(column_name, options = {})
78
- # TODO: Remove options argument as soon we remove support to
79
- # activerecord-deprecated_finders.
80
- calculate(:maximum, column_name, options)
65
+ def maximum(column_name)
66
+ calculate(:maximum, column_name)
81
67
  end
82
68
 
83
69
  # Calculates the sum of values on a given column. The value is returned
84
- # with the same data type of the column, 0 if there's no row. See
85
- # +calculate+ for examples with options.
70
+ # with the same data type of the column, +0+ if there's no row. See
71
+ # #calculate for examples with options.
86
72
  #
87
73
  # Person.sum(:age) # => 4562
88
- def sum(*args)
89
- calculate(:sum, *args)
74
+ def sum(column_name = nil, &block)
75
+ return super(&block) if block_given?
76
+ calculate(:sum, column_name)
90
77
  end
91
78
 
92
- # This calculates aggregate values in the given column. Methods for count, sum, average,
93
- # minimum, and maximum have been added as shortcuts.
79
+ # This calculates aggregate values in the given column. Methods for #count, #sum, #average,
80
+ # #minimum, and #maximum have been added as shortcuts.
94
81
  #
95
- # There are two basic forms of output:
82
+ # Person.calculate(:count, :all) # The same as Person.count
83
+ # Person.average(:age) # SELECT AVG(age) FROM people...
96
84
  #
97
- # * Single aggregate value: The single value is type cast to Integer for COUNT, Float
98
- # for AVG, and the given column's type for everything else.
85
+ # # Selects the minimum age for any family without any minors
86
+ # Person.group(:last_name).having("min(age) > 17").minimum(:age)
99
87
  #
100
- # * Grouped values: This returns an ordered hash of the values and groups them. It
101
- # takes either a column name, or the name of a belongs_to association.
88
+ # Person.sum("2 * age")
102
89
  #
103
- # values = Person.group('last_name').maximum(:age)
104
- # puts values["Drake"]
105
- # # => 43
90
+ # There are two basic forms of output:
106
91
  #
107
- # drake = Family.find_by(last_name: 'Drake')
108
- # values = Person.group(:family).maximum(:age) # Person belongs_to :family
109
- # puts values[drake]
110
- # # => 43
92
+ # * Single aggregate value: The single value is type cast to Integer for COUNT, Float
93
+ # for AVG, and the given column's type for everything else.
111
94
  #
112
- # values.each do |family, max_age|
113
- # ...
114
- # end
95
+ # * Grouped values: This returns an ordered hash of the values and groups them. It
96
+ # takes either a column name, or the name of a belongs_to association.
115
97
  #
116
- # Person.calculate(:count, :all) # The same as Person.count
117
- # Person.average(:age) # SELECT AVG(age) FROM people...
98
+ # values = Person.group('last_name').maximum(:age)
99
+ # puts values["Drake"]
100
+ # # => 43
118
101
  #
119
- # # Selects the minimum age for any family without any minors
120
- # Person.group(:last_name).having("min(age) > 17").minimum(:age)
102
+ # drake = Family.find_by(last_name: 'Drake')
103
+ # values = Person.group(:family).maximum(:age) # Person belongs_to :family
104
+ # puts values[drake]
105
+ # # => 43
121
106
  #
122
- # Person.sum("2 * age")
123
- def calculate(operation, column_name, options = {})
124
- # TODO: Remove options argument as soon we remove support to
125
- # activerecord-deprecated_finders.
107
+ # values.each do |family, max_age|
108
+ # ...
109
+ # end
110
+ def calculate(operation, column_name)
126
111
  if column_name.is_a?(Symbol) && attribute_alias?(column_name)
127
112
  column_name = attribute_alias(column_name)
128
113
  end
129
114
 
130
115
  if has_include?(column_name)
131
- construct_relation_for_association_calculations.calculate(operation, column_name, options)
116
+ construct_relation_for_association_calculations.calculate(operation, column_name)
132
117
  else
133
- perform_calculation(operation, column_name, options)
118
+ perform_calculation(operation, column_name)
134
119
  end
135
120
  end
136
121
 
137
- # Use <tt>pluck</tt> as a shortcut to select one or more attributes without
122
+ # Use #pluck as a shortcut to select one or more attributes without
138
123
  # loading a bunch of records just to grab the attributes you want.
139
124
  #
140
125
  # Person.pluck(:name)
@@ -143,19 +128,19 @@ module ActiveRecord
143
128
  #
144
129
  # Person.all.map(&:name)
145
130
  #
146
- # Pluck returns an <tt>Array</tt> of attribute values type-casted to match
131
+ # Pluck returns an Array of attribute values type-casted to match
147
132
  # the plucked column names, if they can be deduced. Plucking an SQL fragment
148
133
  # returns String values by default.
149
134
  #
150
- # Person.pluck(:id)
151
- # # SELECT people.id FROM people
152
- # # => [1, 2, 3]
135
+ # Person.pluck(:name)
136
+ # # SELECT people.name FROM people
137
+ # # => ['David', 'Jeremy', 'Jose']
153
138
  #
154
139
  # Person.pluck(:id, :name)
155
140
  # # SELECT people.id, people.name FROM people
156
141
  # # => [[1, 'David'], [2, 'Jeremy'], [3, 'Jose']]
157
142
  #
158
- # Person.pluck('DISTINCT role')
143
+ # Person.distinct.pluck(:role)
159
144
  # # SELECT DISTINCT role FROM people
160
145
  # # => ['admin', 'member', 'guest']
161
146
  #
@@ -167,13 +152,11 @@ module ActiveRecord
167
152
  # # SELECT DATEDIFF(updated_at, created_at) FROM people
168
153
  # # => ['0', '27761', '173']
169
154
  #
155
+ # See also #ids.
156
+ #
170
157
  def pluck(*column_names)
171
- column_names.map! do |column_name|
172
- if column_name.is_a?(Symbol) && attribute_alias?(column_name)
173
- attribute_alias(column_name)
174
- else
175
- column_name.to_s
176
- end
158
+ if loaded? && (column_names.map(&:to_s) - @klass.attribute_names - @klass.attribute_aliases.keys).empty?
159
+ return @records.pluck(*column_names)
177
160
  end
178
161
 
179
162
  if has_include?(column_names.first)
@@ -181,10 +164,10 @@ module ActiveRecord
181
164
  else
182
165
  relation = spawn
183
166
  relation.select_values = column_names.map { |cn|
184
- columns_hash.key?(cn) ? arel_table[cn] : cn
167
+ @klass.has_attribute?(cn) || @klass.attribute_alias?(cn) ? arel_attribute(cn) : cn
185
168
  }
186
- result = klass.connection.select_all(relation.arel, nil, relation.arel.bind_values + bind_values)
187
- result.cast_values(klass.column_types)
169
+ result = klass.connection.select_all(relation.arel, nil, bound_attributes)
170
+ result.cast_values(klass.attribute_types)
188
171
  end
189
172
  end
190
173
 
@@ -199,15 +182,14 @@ module ActiveRecord
199
182
  private
200
183
 
201
184
  def has_include?(column_name)
202
- eager_loading? || (includes_values.present? && ((column_name && column_name != :all) || references_eager_loaded_tables?))
185
+ eager_loading? || (includes_values.present? && column_name && column_name != :all)
203
186
  end
204
187
 
205
- def perform_calculation(operation, column_name, options = {})
206
- # TODO: Remove options argument as soon we remove support to
207
- # activerecord-deprecated_finders.
188
+ def perform_calculation(operation, column_name)
208
189
  operation = operation.to_s.downcase
209
190
 
210
- # If #count is used with #distinct / #uniq it is considered distinct. (eg. relation.distinct.count)
191
+ # If #count is used with #distinct (i.e. `relation.distinct.count`) it is
192
+ # considered distinct.
211
193
  distinct = self.distinct_value
212
194
 
213
195
  if operation == "count"
@@ -229,6 +211,8 @@ module ActiveRecord
229
211
  end
230
212
 
231
213
  def aggregate_column(column_name)
214
+ return column_name if Arel::Expressions === column_name
215
+
232
216
  if @klass.column_names.include?(column_name.to_s)
233
217
  Arel::Attribute.new(@klass.unscoped.table, column_name)
234
218
  else
@@ -241,19 +225,16 @@ module ActiveRecord
241
225
  end
242
226
 
243
227
  def execute_simple_calculation(operation, column_name, distinct) #:nodoc:
244
- # Postgresql doesn't like ORDER BY when there are no GROUP BY
228
+ # PostgreSQL doesn't like ORDER BY when there are no GROUP BY
245
229
  relation = unscope(:order)
246
230
 
247
231
  column_alias = column_name
248
232
 
249
- bind_values = nil
250
-
251
233
  if operation == "count" && (relation.limit_value || relation.offset_value)
252
234
  # Shortcut when limit is zero.
253
235
  return 0 if relation.limit_value == 0
254
236
 
255
237
  query_builder = build_count_subquery(relation, column_name, distinct)
256
- bind_values = query_builder.bind_values + relation.bind_values
257
238
  else
258
239
  column = aggregate_column(column_name)
259
240
 
@@ -264,10 +245,9 @@ module ActiveRecord
264
245
  relation.select_values = [select_value]
265
246
 
266
247
  query_builder = relation.arel
267
- bind_values = query_builder.bind_values + relation.bind_values
268
248
  end
269
249
 
270
- result = @klass.connection.select_all(query_builder, nil, bind_values)
250
+ result = @klass.connection.select_all(query_builder, nil, bound_attributes)
271
251
  row = result.first
272
252
  value = row && row.values.first
273
253
  column = result.column_types.fetch(column_alias) do
@@ -289,14 +269,8 @@ module ActiveRecord
289
269
  end
290
270
  group_fields = arel_columns(group_fields)
291
271
 
292
- group_aliases = group_fields.map { |field|
293
- column_alias_for(field)
294
- }
295
- group_columns = group_aliases.zip(group_fields).map { |aliaz,field|
296
- [aliaz, field]
297
- }
298
-
299
- group = group_fields
272
+ group_aliases = group_fields.map { |field| column_alias_for(field) }
273
+ group_columns = group_aliases.zip(group_fields)
300
274
 
301
275
  if operation == 'count' && column_name == :all
302
276
  aggregate_alias = 'count_all'
@@ -310,9 +284,9 @@ module ActiveRecord
310
284
  operation,
311
285
  distinct).as(aggregate_alias)
312
286
  ]
313
- select_values += self.select_values unless having_values.empty?
287
+ select_values += select_values unless having_clause.empty?
314
288
 
315
- select_values.concat group_fields.zip(group_aliases).map { |field,aliaz|
289
+ select_values.concat group_columns.map { |aliaz, field|
316
290
  if field.respond_to?(:as)
317
291
  field.as(aliaz)
318
292
  else
@@ -321,14 +295,14 @@ module ActiveRecord
321
295
  }
322
296
 
323
297
  relation = except(:group)
324
- relation.group_values = group
298
+ relation.group_values = group_fields
325
299
  relation.select_values = select_values
326
300
 
327
- calculated_data = @klass.connection.select_all(relation, nil, relation.arel.bind_values + bind_values)
301
+ calculated_data = @klass.connection.select_all(relation, nil, relation.bound_attributes)
328
302
 
329
303
  if association
330
304
  key_ids = calculated_data.collect { |row| row[group_aliases.first] }
331
- key_records = association.klass.base_class.find(key_ids)
305
+ key_records = association.klass.base_class.where(association.klass.base_class.primary_key => key_ids)
332
306
  key_records = Hash[key_records.map { |r| [r.id, r] }]
333
307
  end
334
308
 
@@ -354,7 +328,6 @@ module ActiveRecord
354
328
  # column_alias_for("sum(id)") # => "sum_id"
355
329
  # column_alias_for("count(distinct users.id)") # => "count_distinct_users_id"
356
330
  # column_alias_for("count(*)") # => "count_all"
357
- # column_alias_for("count", "id") # => "count_id"
358
331
  def column_alias_for(keys)
359
332
  if keys.respond_to? :name
360
333
  keys = "#{keys.relation.name}.#{keys.name}"
@@ -377,15 +350,15 @@ module ActiveRecord
377
350
  def type_cast_calculated_value(value, type, operation = nil)
378
351
  case operation
379
352
  when 'count' then value.to_i
380
- when 'sum' then type.type_cast_from_database(value || 0)
353
+ when 'sum' then type.deserialize(value || 0)
381
354
  when 'average' then value.respond_to?(:to_d) ? value.to_d : value
382
- else type.type_cast_from_database(value)
355
+ else type.deserialize(value)
383
356
  end
384
357
  end
385
358
 
386
- # TODO: refactor to allow non-string `select_values` (eg. Arel nodes).
387
359
  def select_for_count
388
360
  if select_values.present?
361
+ return select_values.first if select_values.one?
389
362
  select_values.join(", ")
390
363
  else
391
364
  :all
@@ -398,11 +371,9 @@ module ActiveRecord
398
371
 
399
372
  aliased_column = aggregate_column(column_name == :all ? 1 : column_name).as(column_alias)
400
373
  relation.select_values = [aliased_column]
401
- arel = relation.arel
402
- subquery = arel.as(subquery_alias)
374
+ subquery = relation.arel.as(subquery_alias)
403
375
 
404
376
  sm = Arel::SelectManager.new relation.engine
405
- sm.bind_values = arel.bind_values
406
377
  select_value = operation_over_aggregate_column(column_alias, 'count', distinct)
407
378
  sm.project(select_value).from(subquery)
408
379
  end