activerecord 4.2.0 → 5.2.8.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of activerecord might be problematic. Click here for more details.

Files changed (274) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +640 -928
  3. data/MIT-LICENSE +2 -2
  4. data/README.rdoc +10 -11
  5. data/examples/performance.rb +32 -31
  6. data/examples/simple.rb +5 -4
  7. data/lib/active_record/aggregations.rb +264 -247
  8. data/lib/active_record/association_relation.rb +24 -6
  9. data/lib/active_record/associations/alias_tracker.rb +29 -35
  10. data/lib/active_record/associations/association.rb +87 -41
  11. data/lib/active_record/associations/association_scope.rb +106 -132
  12. data/lib/active_record/associations/belongs_to_association.rb +55 -36
  13. data/lib/active_record/associations/belongs_to_polymorphic_association.rb +8 -8
  14. data/lib/active_record/associations/builder/association.rb +29 -38
  15. data/lib/active_record/associations/builder/belongs_to.rb +77 -30
  16. data/lib/active_record/associations/builder/collection_association.rb +14 -23
  17. data/lib/active_record/associations/builder/has_and_belongs_to_many.rb +50 -39
  18. data/lib/active_record/associations/builder/has_many.rb +6 -4
  19. data/lib/active_record/associations/builder/has_one.rb +13 -6
  20. data/lib/active_record/associations/builder/singular_association.rb +15 -11
  21. data/lib/active_record/associations/collection_association.rb +145 -266
  22. data/lib/active_record/associations/collection_proxy.rb +242 -138
  23. data/lib/active_record/associations/foreign_association.rb +13 -0
  24. data/lib/active_record/associations/has_many_association.rb +35 -75
  25. data/lib/active_record/associations/has_many_through_association.rb +51 -69
  26. data/lib/active_record/associations/has_one_association.rb +39 -24
  27. data/lib/active_record/associations/has_one_through_association.rb +18 -9
  28. data/lib/active_record/associations/join_dependency/join_association.rb +40 -81
  29. data/lib/active_record/associations/join_dependency/join_base.rb +10 -9
  30. data/lib/active_record/associations/join_dependency/join_part.rb +12 -12
  31. data/lib/active_record/associations/join_dependency.rb +134 -154
  32. data/lib/active_record/associations/preloader/association.rb +85 -116
  33. data/lib/active_record/associations/preloader/through_association.rb +85 -74
  34. data/lib/active_record/associations/preloader.rb +83 -93
  35. data/lib/active_record/associations/singular_association.rb +27 -40
  36. data/lib/active_record/associations/through_association.rb +48 -23
  37. data/lib/active_record/associations.rb +1732 -1596
  38. data/lib/active_record/attribute_assignment.rb +58 -182
  39. data/lib/active_record/attribute_decorators.rb +39 -15
  40. data/lib/active_record/attribute_methods/before_type_cast.rb +12 -5
  41. data/lib/active_record/attribute_methods/dirty.rb +94 -125
  42. data/lib/active_record/attribute_methods/primary_key.rb +86 -71
  43. data/lib/active_record/attribute_methods/query.rb +4 -2
  44. data/lib/active_record/attribute_methods/read.rb +45 -63
  45. data/lib/active_record/attribute_methods/serialization.rb +40 -20
  46. data/lib/active_record/attribute_methods/time_zone_conversion.rb +62 -36
  47. data/lib/active_record/attribute_methods/write.rb +31 -46
  48. data/lib/active_record/attribute_methods.rb +170 -117
  49. data/lib/active_record/attributes.rb +201 -74
  50. data/lib/active_record/autosave_association.rb +118 -45
  51. data/lib/active_record/base.rb +60 -48
  52. data/lib/active_record/callbacks.rb +97 -57
  53. data/lib/active_record/coders/json.rb +3 -1
  54. data/lib/active_record/coders/yaml_column.rb +37 -13
  55. data/lib/active_record/collection_cache_key.rb +53 -0
  56. data/lib/active_record/connection_adapters/abstract/connection_pool.rb +712 -284
  57. data/lib/active_record/connection_adapters/abstract/database_limits.rb +10 -5
  58. data/lib/active_record/connection_adapters/abstract/database_statements.rb +254 -87
  59. data/lib/active_record/connection_adapters/abstract/query_cache.rb +72 -22
  60. data/lib/active_record/connection_adapters/abstract/quoting.rb +119 -52
  61. data/lib/active_record/connection_adapters/abstract/savepoints.rb +6 -4
  62. data/lib/active_record/connection_adapters/abstract/schema_creation.rb +67 -46
  63. data/lib/active_record/connection_adapters/abstract/schema_definitions.rb +328 -217
  64. data/lib/active_record/connection_adapters/abstract/schema_dumper.rb +81 -36
  65. data/lib/active_record/connection_adapters/abstract/schema_statements.rb +617 -212
  66. data/lib/active_record/connection_adapters/abstract/transaction.rb +139 -75
  67. data/lib/active_record/connection_adapters/abstract_adapter.rb +332 -191
  68. data/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +567 -563
  69. data/lib/active_record/connection_adapters/column.rb +50 -41
  70. data/lib/active_record/connection_adapters/connection_specification.rb +147 -135
  71. data/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb +33 -0
  72. data/lib/active_record/connection_adapters/mysql/column.rb +27 -0
  73. data/lib/active_record/connection_adapters/mysql/database_statements.rb +140 -0
  74. data/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb +72 -0
  75. data/lib/active_record/connection_adapters/mysql/quoting.rb +44 -0
  76. data/lib/active_record/connection_adapters/mysql/schema_creation.rb +73 -0
  77. data/lib/active_record/connection_adapters/mysql/schema_definitions.rb +87 -0
  78. data/lib/active_record/connection_adapters/mysql/schema_dumper.rb +80 -0
  79. data/lib/active_record/connection_adapters/mysql/schema_statements.rb +148 -0
  80. data/lib/active_record/connection_adapters/mysql/type_metadata.rb +35 -0
  81. data/lib/active_record/connection_adapters/mysql2_adapter.rb +42 -195
  82. data/lib/active_record/connection_adapters/postgresql/column.rb +35 -11
  83. data/lib/active_record/connection_adapters/postgresql/database_statements.rb +46 -115
  84. data/lib/active_record/connection_adapters/postgresql/explain_pretty_printer.rb +44 -0
  85. data/lib/active_record/connection_adapters/postgresql/oid/array.rb +50 -57
  86. data/lib/active_record/connection_adapters/postgresql/oid/bit.rb +10 -6
  87. data/lib/active_record/connection_adapters/postgresql/oid/bit_varying.rb +2 -0
  88. data/lib/active_record/connection_adapters/postgresql/oid/bytea.rb +5 -2
  89. data/lib/active_record/connection_adapters/postgresql/oid/cidr.rb +5 -1
  90. data/lib/active_record/connection_adapters/postgresql/oid/date.rb +13 -1
  91. data/lib/active_record/connection_adapters/postgresql/oid/date_time.rb +9 -13
  92. data/lib/active_record/connection_adapters/postgresql/oid/decimal.rb +3 -1
  93. data/lib/active_record/connection_adapters/postgresql/oid/enum.rb +7 -3
  94. data/lib/active_record/connection_adapters/postgresql/oid/hstore.rb +31 -19
  95. data/lib/active_record/connection_adapters/postgresql/oid/inet.rb +2 -0
  96. data/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb +3 -11
  97. data/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb +45 -0
  98. data/lib/active_record/connection_adapters/postgresql/oid/money.rb +7 -9
  99. data/lib/active_record/connection_adapters/postgresql/oid/{integer.rb → oid.rb} +6 -2
  100. data/lib/active_record/connection_adapters/postgresql/oid/point.rb +33 -11
  101. data/lib/active_record/connection_adapters/postgresql/oid/range.rb +52 -34
  102. data/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb +4 -1
  103. data/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb +65 -51
  104. data/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +5 -3
  105. data/lib/active_record/connection_adapters/postgresql/oid/vector.rb +3 -1
  106. data/lib/active_record/connection_adapters/postgresql/oid/xml.rb +3 -1
  107. data/lib/active_record/connection_adapters/postgresql/oid.rb +23 -25
  108. data/lib/active_record/connection_adapters/postgresql/quoting.rb +107 -47
  109. data/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +27 -14
  110. data/lib/active_record/connection_adapters/postgresql/schema_creation.rb +65 -0
  111. data/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +144 -90
  112. data/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +50 -0
  113. data/lib/active_record/connection_adapters/postgresql/schema_statements.rb +466 -280
  114. data/lib/active_record/connection_adapters/postgresql/type_metadata.rb +39 -0
  115. data/lib/active_record/connection_adapters/postgresql/utils.rb +12 -8
  116. data/lib/active_record/connection_adapters/postgresql_adapter.rb +439 -330
  117. data/lib/active_record/connection_adapters/schema_cache.rb +48 -24
  118. data/lib/active_record/connection_adapters/sql_type_metadata.rb +34 -0
  119. data/lib/active_record/connection_adapters/sqlite3/explain_pretty_printer.rb +21 -0
  120. data/lib/active_record/connection_adapters/sqlite3/quoting.rb +67 -0
  121. data/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +17 -0
  122. data/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb +19 -0
  123. data/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb +18 -0
  124. data/lib/active_record/connection_adapters/sqlite3/schema_statements.rb +106 -0
  125. data/lib/active_record/connection_adapters/sqlite3_adapter.rb +269 -324
  126. data/lib/active_record/connection_adapters/statement_pool.rb +34 -13
  127. data/lib/active_record/connection_handling.rb +40 -27
  128. data/lib/active_record/core.rb +205 -202
  129. data/lib/active_record/counter_cache.rb +80 -37
  130. data/lib/active_record/define_callbacks.rb +22 -0
  131. data/lib/active_record/dynamic_matchers.rb +87 -105
  132. data/lib/active_record/enum.rb +136 -90
  133. data/lib/active_record/errors.rb +180 -52
  134. data/lib/active_record/explain.rb +23 -11
  135. data/lib/active_record/explain_registry.rb +4 -2
  136. data/lib/active_record/explain_subscriber.rb +11 -6
  137. data/lib/active_record/fixture_set/file.rb +35 -9
  138. data/lib/active_record/fixtures.rb +193 -135
  139. data/lib/active_record/gem_version.rb +5 -3
  140. data/lib/active_record/inheritance.rb +148 -112
  141. data/lib/active_record/integration.rb +70 -28
  142. data/lib/active_record/internal_metadata.rb +45 -0
  143. data/lib/active_record/legacy_yaml_adapter.rb +48 -0
  144. data/lib/active_record/locale/en.yml +3 -2
  145. data/lib/active_record/locking/optimistic.rb +92 -98
  146. data/lib/active_record/locking/pessimistic.rb +15 -3
  147. data/lib/active_record/log_subscriber.rb +95 -33
  148. data/lib/active_record/migration/command_recorder.rb +133 -90
  149. data/lib/active_record/migration/compatibility.rb +217 -0
  150. data/lib/active_record/migration/join_table.rb +8 -6
  151. data/lib/active_record/migration.rb +594 -267
  152. data/lib/active_record/model_schema.rb +292 -111
  153. data/lib/active_record/nested_attributes.rb +266 -214
  154. data/lib/active_record/no_touching.rb +8 -2
  155. data/lib/active_record/null_relation.rb +24 -37
  156. data/lib/active_record/persistence.rb +350 -119
  157. data/lib/active_record/query_cache.rb +13 -24
  158. data/lib/active_record/querying.rb +19 -17
  159. data/lib/active_record/railtie.rb +117 -35
  160. data/lib/active_record/railties/console_sandbox.rb +2 -0
  161. data/lib/active_record/railties/controller_runtime.rb +9 -3
  162. data/lib/active_record/railties/databases.rake +160 -174
  163. data/lib/active_record/readonly_attributes.rb +5 -4
  164. data/lib/active_record/reflection.rb +447 -288
  165. data/lib/active_record/relation/batches/batch_enumerator.rb +69 -0
  166. data/lib/active_record/relation/batches.rb +204 -55
  167. data/lib/active_record/relation/calculations.rb +259 -244
  168. data/lib/active_record/relation/delegation.rb +67 -60
  169. data/lib/active_record/relation/finder_methods.rb +290 -253
  170. data/lib/active_record/relation/from_clause.rb +26 -0
  171. data/lib/active_record/relation/merger.rb +91 -68
  172. data/lib/active_record/relation/predicate_builder/array_handler.rb +24 -23
  173. data/lib/active_record/relation/predicate_builder/association_query_value.rb +46 -0
  174. data/lib/active_record/relation/predicate_builder/base_handler.rb +19 -0
  175. data/lib/active_record/relation/predicate_builder/basic_object_handler.rb +20 -0
  176. data/lib/active_record/relation/predicate_builder/polymorphic_array_value.rb +56 -0
  177. data/lib/active_record/relation/predicate_builder/range_handler.rb +42 -0
  178. data/lib/active_record/relation/predicate_builder/relation_handler.rb +7 -1
  179. data/lib/active_record/relation/predicate_builder.rb +118 -92
  180. data/lib/active_record/relation/query_attribute.rb +45 -0
  181. data/lib/active_record/relation/query_methods.rb +446 -389
  182. data/lib/active_record/relation/record_fetch_warning.rb +51 -0
  183. data/lib/active_record/relation/spawn_methods.rb +18 -16
  184. data/lib/active_record/relation/where_clause.rb +186 -0
  185. data/lib/active_record/relation/where_clause_factory.rb +34 -0
  186. data/lib/active_record/relation.rb +287 -339
  187. data/lib/active_record/result.rb +54 -36
  188. data/lib/active_record/runtime_registry.rb +6 -4
  189. data/lib/active_record/sanitization.rb +155 -124
  190. data/lib/active_record/schema.rb +30 -24
  191. data/lib/active_record/schema_dumper.rb +91 -87
  192. data/lib/active_record/schema_migration.rb +19 -19
  193. data/lib/active_record/scoping/default.rb +102 -84
  194. data/lib/active_record/scoping/named.rb +81 -32
  195. data/lib/active_record/scoping.rb +45 -26
  196. data/lib/active_record/secure_token.rb +40 -0
  197. data/lib/active_record/serialization.rb +5 -5
  198. data/lib/active_record/statement_cache.rb +45 -35
  199. data/lib/active_record/store.rb +42 -36
  200. data/lib/active_record/suppressor.rb +61 -0
  201. data/lib/active_record/table_metadata.rb +82 -0
  202. data/lib/active_record/tasks/database_tasks.rb +136 -95
  203. data/lib/active_record/tasks/mysql_database_tasks.rb +59 -89
  204. data/lib/active_record/tasks/postgresql_database_tasks.rb +84 -31
  205. data/lib/active_record/tasks/sqlite_database_tasks.rb +44 -16
  206. data/lib/active_record/timestamp.rb +70 -38
  207. data/lib/active_record/touch_later.rb +64 -0
  208. data/lib/active_record/transactions.rb +208 -123
  209. data/lib/active_record/translation.rb +2 -0
  210. data/lib/active_record/type/adapter_specific_registry.rb +136 -0
  211. data/lib/active_record/type/date.rb +4 -41
  212. data/lib/active_record/type/date_time.rb +4 -38
  213. data/lib/active_record/type/decimal_without_scale.rb +6 -2
  214. data/lib/active_record/type/hash_lookup_type_map.rb +13 -5
  215. data/lib/active_record/type/internal/timezone.rb +17 -0
  216. data/lib/active_record/type/json.rb +30 -0
  217. data/lib/active_record/type/serialized.rb +30 -15
  218. data/lib/active_record/type/text.rb +2 -2
  219. data/lib/active_record/type/time.rb +11 -16
  220. data/lib/active_record/type/type_map.rb +15 -17
  221. data/lib/active_record/type/unsigned_integer.rb +9 -7
  222. data/lib/active_record/type.rb +79 -23
  223. data/lib/active_record/type_caster/connection.rb +33 -0
  224. data/lib/active_record/type_caster/map.rb +23 -0
  225. data/lib/active_record/type_caster.rb +9 -0
  226. data/lib/active_record/validations/absence.rb +25 -0
  227. data/lib/active_record/validations/associated.rb +13 -4
  228. data/lib/active_record/validations/length.rb +26 -0
  229. data/lib/active_record/validations/presence.rb +14 -13
  230. data/lib/active_record/validations/uniqueness.rb +41 -32
  231. data/lib/active_record/validations.rb +38 -35
  232. data/lib/active_record/version.rb +3 -1
  233. data/lib/active_record.rb +36 -21
  234. data/lib/rails/generators/active_record/application_record/application_record_generator.rb +27 -0
  235. data/lib/rails/generators/active_record/application_record/templates/application_record.rb.tt +5 -0
  236. data/lib/rails/generators/active_record/migration/migration_generator.rb +43 -35
  237. data/lib/rails/generators/active_record/migration/templates/{create_table_migration.rb → create_table_migration.rb.tt} +8 -6
  238. data/lib/rails/generators/active_record/migration/templates/{migration.rb → migration.rb.tt} +8 -7
  239. data/lib/rails/generators/active_record/migration.rb +18 -1
  240. data/lib/rails/generators/active_record/model/model_generator.rb +18 -22
  241. data/lib/rails/generators/active_record/model/templates/{model.rb → model.rb.tt} +3 -0
  242. data/lib/rails/generators/active_record.rb +7 -5
  243. metadata +77 -53
  244. data/lib/active_record/associations/preloader/belongs_to.rb +0 -17
  245. data/lib/active_record/associations/preloader/collection_association.rb +0 -24
  246. data/lib/active_record/associations/preloader/has_many.rb +0 -17
  247. data/lib/active_record/associations/preloader/has_many_through.rb +0 -19
  248. data/lib/active_record/associations/preloader/has_one.rb +0 -23
  249. data/lib/active_record/associations/preloader/has_one_through.rb +0 -9
  250. data/lib/active_record/associations/preloader/singular_association.rb +0 -21
  251. data/lib/active_record/attribute.rb +0 -149
  252. data/lib/active_record/attribute_set/builder.rb +0 -86
  253. data/lib/active_record/attribute_set.rb +0 -77
  254. data/lib/active_record/connection_adapters/mysql_adapter.rb +0 -491
  255. data/lib/active_record/connection_adapters/postgresql/array_parser.rb +0 -93
  256. data/lib/active_record/connection_adapters/postgresql/oid/float.rb +0 -21
  257. data/lib/active_record/connection_adapters/postgresql/oid/infinity.rb +0 -13
  258. data/lib/active_record/connection_adapters/postgresql/oid/json.rb +0 -35
  259. data/lib/active_record/connection_adapters/postgresql/oid/time.rb +0 -11
  260. data/lib/active_record/railties/jdbcmysql_error.rb +0 -16
  261. data/lib/active_record/serializers/xml_serializer.rb +0 -193
  262. data/lib/active_record/type/big_integer.rb +0 -13
  263. data/lib/active_record/type/binary.rb +0 -50
  264. data/lib/active_record/type/boolean.rb +0 -30
  265. data/lib/active_record/type/decimal.rb +0 -40
  266. data/lib/active_record/type/decorator.rb +0 -14
  267. data/lib/active_record/type/float.rb +0 -19
  268. data/lib/active_record/type/integer.rb +0 -55
  269. data/lib/active_record/type/mutable.rb +0 -16
  270. data/lib/active_record/type/numeric.rb +0 -36
  271. data/lib/active_record/type/string.rb +0 -36
  272. data/lib/active_record/type/time_value.rb +0 -38
  273. data/lib/active_record/type/value.rb +0 -101
  274. /data/lib/rails/generators/active_record/model/templates/{module.rb → module.rb.tt} +0 -0
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecord
4
+ module Batches
5
+ class BatchEnumerator
6
+ include Enumerable
7
+
8
+ def initialize(of: 1000, start: nil, finish: nil, relation:) #:nodoc:
9
+ @of = of
10
+ @relation = relation
11
+ @start = start
12
+ @finish = finish
13
+ end
14
+
15
+ # Looping through a collection of records from the database (using the
16
+ # +all+ method, for example) is very inefficient since it will try to
17
+ # instantiate all the objects at once.
18
+ #
19
+ # In that case, batch processing methods allow you to work with the
20
+ # records in batches, thereby greatly reducing memory consumption.
21
+ #
22
+ # Person.in_batches.each_record do |person|
23
+ # person.do_awesome_stuff
24
+ # end
25
+ #
26
+ # Person.where("age > 21").in_batches(of: 10).each_record do |person|
27
+ # person.party_all_night!
28
+ # end
29
+ #
30
+ # If you do not provide a block to #each_record, it will return an Enumerator
31
+ # for chaining with other methods:
32
+ #
33
+ # Person.in_batches.each_record.with_index do |person, index|
34
+ # person.award_trophy(index + 1)
35
+ # end
36
+ def each_record
37
+ return to_enum(:each_record) unless block_given?
38
+
39
+ @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true).each do |relation|
40
+ relation.records.each { |record| yield record }
41
+ end
42
+ end
43
+
44
+ # Delegates #delete_all, #update_all, #destroy_all methods to each batch.
45
+ #
46
+ # People.in_batches.delete_all
47
+ # People.where('age < 10').in_batches.destroy_all
48
+ # People.in_batches.update_all('age = age + 1')
49
+ [:delete_all, :update_all, :destroy_all].each do |method|
50
+ define_method(method) do |*args, &block|
51
+ @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false).each do |relation|
52
+ relation.send(method, *args, &block)
53
+ end
54
+ end
55
+ end
56
+
57
+ # Yields an ActiveRecord::Relation object for each batch of records.
58
+ #
59
+ # Person.in_batches.each do |relation|
60
+ # relation.update_all(awesome: true)
61
+ # end
62
+ def each
63
+ enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false)
64
+ return enum.each { |relation| yield relation } if block_given?
65
+ enum
66
+ end
67
+ end
68
+ end
69
+ end
@@ -1,8 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record/relation/batches/batch_enumerator"
4
+
1
5
  module ActiveRecord
2
6
  module Batches
7
+ ORDER_IGNORE_MESSAGE = "Scoped order is ignored, it's forced to be batch order."
8
+
3
9
  # Looping through a collection of records from the database
4
- # (using the +all+ method, for example) is very inefficient
5
- # since it will try to instantiate all the objects at once.
10
+ # (using the Scoping::Named::ClassMethods.all method, for example)
11
+ # is very inefficient since it will try to instantiate all the objects at once.
6
12
  #
7
13
  # In that case, batch processing methods allow you to work
8
14
  # with the records in batches, thereby greatly reducing memory consumption.
@@ -26,38 +32,52 @@ module ActiveRecord
26
32
  # end
27
33
  #
28
34
  # ==== Options
29
- # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000.
30
- # * <tt>:start</tt> - Specifies the starting point for the batch processing.
31
- # This is especially useful if you want multiple workers dealing with
32
- # the same processing queue. You can make worker 1 handle all the records
33
- # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond
34
- # (by setting the +:start+ option on that worker).
35
- #
36
- # # Let's process for a batch of 2000 records, skipping the first 2000 rows
37
- # Person.find_each(start: 2000, batch_size: 2000) do |person|
35
+ # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
36
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
37
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
38
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
39
+ # an order is present in the relation.
40
+ #
41
+ # Limits are honored, and if present there is no requirement for the batch
42
+ # size: it can be less than, equal to, or greater than the limit.
43
+ #
44
+ # The options +start+ and +finish+ are especially useful if you want
45
+ # multiple workers dealing with the same processing queue. You can make
46
+ # worker 1 handle all the records between id 1 and 9999 and worker 2
47
+ # handle from 10000 and beyond by setting the +:start+ and +:finish+
48
+ # option on each worker.
49
+ #
50
+ # # In worker 1, let's process until 9999 records.
51
+ # Person.find_each(finish: 9_999) do |person|
52
+ # person.party_all_night!
53
+ # end
54
+ #
55
+ # # In worker 2, let's process from record 10_000 and onwards.
56
+ # Person.find_each(start: 10_000) do |person|
38
57
  # person.party_all_night!
39
58
  # end
40
59
  #
41
60
  # NOTE: It's not possible to set the order. That is automatically set to
42
61
  # ascending on the primary key ("id ASC") to make the batch ordering
43
- # work. This also means that this method only works with integer-based
44
- # primary keys.
62
+ # work. This also means that this method only works when the primary key is
63
+ # orderable (e.g. an integer or string).
45
64
  #
46
- # NOTE: You can't set the limit either, that's used to control
47
- # the batch sizes.
48
- def find_each(options = {})
65
+ # NOTE: By its nature, batch processing is subject to race conditions if
66
+ # other processes are modifying the database.
67
+ def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
49
68
  if block_given?
50
- find_in_batches(options) do |records|
69
+ find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do |records|
51
70
  records.each { |record| yield record }
52
71
  end
53
72
  else
54
- enum_for :find_each, options do
55
- options[:start] ? where(table[primary_key].gteq(options[:start])).size : size
73
+ enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
74
+ relation = self
75
+ apply_limits(relation, start, finish).size
56
76
  end
57
77
  end
58
78
  end
59
79
 
60
- # Yields each batch of records that was found by the find +options+ as
80
+ # Yields each batch of records that was found by the find options as
61
81
  # an array.
62
82
  #
63
83
  # Person.where("age > 21").find_in_batches do |group|
@@ -76,63 +96,192 @@ module ActiveRecord
76
96
  # To be yielded each record one by one, use #find_each instead.
77
97
  #
78
98
  # ==== Options
79
- # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000.
80
- # * <tt>:start</tt> - Specifies the starting point for the batch processing.
81
- # This is especially useful if you want multiple workers dealing with
82
- # the same processing queue. You can make worker 1 handle all the records
83
- # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond
84
- # (by setting the +:start+ option on that worker).
85
- #
86
- # # Let's process the next 2000 records
87
- # Person.find_in_batches(start: 2000, batch_size: 2000) do |group|
99
+ # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
100
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
101
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
102
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
103
+ # an order is present in the relation.
104
+ #
105
+ # Limits are honored, and if present there is no requirement for the batch
106
+ # size: it can be less than, equal to, or greater than the limit.
107
+ #
108
+ # The options +start+ and +finish+ are especially useful if you want
109
+ # multiple workers dealing with the same processing queue. You can make
110
+ # worker 1 handle all the records between id 1 and 9999 and worker 2
111
+ # handle from 10000 and beyond by setting the +:start+ and +:finish+
112
+ # option on each worker.
113
+ #
114
+ # # Let's process from record 10_000 on.
115
+ # Person.find_in_batches(start: 10_000) do |group|
88
116
  # group.each { |person| person.party_all_night! }
89
117
  # end
90
118
  #
91
119
  # NOTE: It's not possible to set the order. That is automatically set to
92
120
  # ascending on the primary key ("id ASC") to make the batch ordering
93
- # work. This also means that this method only works with integer-based
94
- # primary keys.
121
+ # work. This also means that this method only works when the primary key is
122
+ # orderable (e.g. an integer or string).
95
123
  #
96
- # NOTE: You can't set the limit either, that's used to control
97
- # the batch sizes.
98
- def find_in_batches(options = {})
99
- options.assert_valid_keys(:start, :batch_size)
100
-
124
+ # NOTE: By its nature, batch processing is subject to race conditions if
125
+ # other processes are modifying the database.
126
+ def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
101
127
  relation = self
102
- start = options[:start]
103
- batch_size = options[:batch_size] || 1000
104
-
105
128
  unless block_given?
106
- return to_enum(:find_in_batches, options) do
107
- total = start ? where(table[primary_key].gteq(start)).size : size
129
+ return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
130
+ total = apply_limits(relation, start, finish).size
108
131
  (total - 1).div(batch_size) + 1
109
132
  end
110
133
  end
111
134
 
112
- if logger && (arel.orders.present? || arel.taken.present?)
113
- logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
135
+ in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore) do |batch|
136
+ yield batch.to_a
137
+ end
138
+ end
139
+
140
+ # Yields ActiveRecord::Relation objects to work with a batch of records.
141
+ #
142
+ # Person.where("age > 21").in_batches do |relation|
143
+ # relation.delete_all
144
+ # sleep(10) # Throttle the delete queries
145
+ # end
146
+ #
147
+ # If you do not provide a block to #in_batches, it will return a
148
+ # BatchEnumerator which is enumerable.
149
+ #
150
+ # Person.in_batches.each_with_index do |relation, batch_index|
151
+ # puts "Processing relation ##{batch_index}"
152
+ # relation.delete_all
153
+ # end
154
+ #
155
+ # Examples of calling methods on the returned BatchEnumerator object:
156
+ #
157
+ # Person.in_batches.delete_all
158
+ # Person.in_batches.update_all(awesome: true)
159
+ # Person.in_batches.each_record(&:party_all_night!)
160
+ #
161
+ # ==== Options
162
+ # * <tt>:of</tt> - Specifies the size of the batch. Defaults to 1000.
163
+ # * <tt>:load</tt> - Specifies if the relation should be loaded. Defaults to false.
164
+ # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
165
+ # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
166
+ # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
167
+ # an order is present in the relation.
168
+ #
169
+ # Limits are honored, and if present there is no requirement for the batch
170
+ # size, it can be less than, equal, or greater than the limit.
171
+ #
172
+ # The options +start+ and +finish+ are especially useful if you want
173
+ # multiple workers dealing with the same processing queue. You can make
174
+ # worker 1 handle all the records between id 1 and 9999 and worker 2
175
+ # handle from 10000 and beyond by setting the +:start+ and +:finish+
176
+ # option on each worker.
177
+ #
178
+ # # Let's process from record 10_000 on.
179
+ # Person.in_batches(start: 10_000).update_all(awesome: true)
180
+ #
181
+ # An example of calling where query method on the relation:
182
+ #
183
+ # Person.in_batches.each do |relation|
184
+ # relation.update_all('age = age + 1')
185
+ # relation.where('age > 21').update_all(should_party: true)
186
+ # relation.where('age <= 21').delete_all
187
+ # end
188
+ #
189
+ # NOTE: If you are going to iterate through each record, you should call
190
+ # #each_record on the yielded BatchEnumerator:
191
+ #
192
+ # Person.in_batches.each_record(&:party_all_night!)
193
+ #
194
+ # NOTE: It's not possible to set the order. That is automatically set to
195
+ # ascending on the primary key ("id ASC") to make the batch ordering
196
+ # consistent. Therefore the primary key must be orderable, e.g. an integer
197
+ # or a string.
198
+ #
199
+ # NOTE: By its nature, batch processing is subject to race conditions if
200
+ # other processes are modifying the database.
201
+ def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil)
202
+ relation = self
203
+ unless block_given?
204
+ return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self)
205
+ end
206
+
207
+ if arel.orders.present?
208
+ act_on_ignored_order(error_on_ignore)
114
209
  end
115
210
 
116
- relation = relation.reorder(batch_order).limit(batch_size)
117
- records = start ? relation.where(table[primary_key].gteq(start)).to_a : relation.to_a
211
+ batch_limit = of
212
+ if limit_value
213
+ remaining = limit_value
214
+ batch_limit = remaining if remaining < batch_limit
215
+ end
216
+
217
+ relation = relation.reorder(batch_order).limit(batch_limit)
218
+ relation = apply_limits(relation, start, finish)
219
+ relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
220
+ batch_relation = relation
221
+
222
+ loop do
223
+ if load
224
+ records = batch_relation.records
225
+ ids = records.map(&:id)
226
+ yielded_relation = where(primary_key => ids)
227
+ yielded_relation.load_records(records)
228
+ else
229
+ ids = batch_relation.pluck(primary_key)
230
+ yielded_relation = where(primary_key => ids)
231
+ end
232
+
233
+ break if ids.empty?
234
+
235
+ primary_key_offset = ids.last
236
+ raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
237
+
238
+ yield yielded_relation
118
239
 
119
- while records.any?
120
- records_size = records.size
121
- primary_key_offset = records.last.id
122
- raise "Primary key not included in the custom select clause" unless primary_key_offset
240
+ break if ids.length < batch_limit
123
241
 
124
- yield records
242
+ if limit_value
243
+ remaining -= ids.length
125
244
 
126
- break if records_size < batch_size
245
+ if remaining == 0
246
+ # Saves a useless iteration when the limit is a multiple of the
247
+ # batch size.
248
+ break
249
+ elsif remaining < batch_limit
250
+ relation = relation.limit(remaining)
251
+ end
252
+ end
127
253
 
128
- records = relation.where(table[primary_key].gt(primary_key_offset)).to_a
254
+ attr = Relation::QueryAttribute.new(primary_key, primary_key_offset, klass.type_for_attribute(primary_key))
255
+ batch_relation = relation.where(arel_attribute(primary_key).gt(Arel::Nodes::BindParam.new(attr)))
129
256
  end
130
257
  end
131
258
 
132
259
  private
133
260
 
134
- def batch_order
135
- "#{quoted_table_name}.#{quoted_primary_key} ASC"
136
- end
261
+ def apply_limits(relation, start, finish)
262
+ if start
263
+ attr = Relation::QueryAttribute.new(primary_key, start, klass.type_for_attribute(primary_key))
264
+ relation = relation.where(arel_attribute(primary_key).gteq(Arel::Nodes::BindParam.new(attr)))
265
+ end
266
+ if finish
267
+ attr = Relation::QueryAttribute.new(primary_key, finish, klass.type_for_attribute(primary_key))
268
+ relation = relation.where(arel_attribute(primary_key).lteq(Arel::Nodes::BindParam.new(attr)))
269
+ end
270
+ relation
271
+ end
272
+
273
+ def batch_order
274
+ arel_attribute(primary_key).asc
275
+ end
276
+
277
+ def act_on_ignored_order(error_on_ignore)
278
+ raise_error = (error_on_ignore.nil? ? klass.error_on_ignored_order : error_on_ignore)
279
+
280
+ if raise_error
281
+ raise ArgumentError.new(ORDER_IGNORE_MESSAGE)
282
+ elsif logger
283
+ logger.warn(ORDER_IGNORE_MESSAGE)
284
+ end
285
+ end
137
286
  end
138
287
  end