mongoid 9.1.0 → 9.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/lib/config/locales/en.yml +69 -0
  3. data/lib/mongoid/association/accessors.rb +27 -5
  4. data/lib/mongoid/association/depending.rb +14 -10
  5. data/lib/mongoid/association/eager.rb +9 -0
  6. data/lib/mongoid/association/eager_load/discriminated_inclusion.rb +88 -0
  7. data/lib/mongoid/association/eager_load/embedded_distributor.rb +151 -0
  8. data/lib/mongoid/association/eager_load/inclusion.rb +173 -0
  9. data/lib/mongoid/association/eager_load/inclusion_tree.rb +96 -0
  10. data/lib/mongoid/association/eager_load/lookup_pipeline.rb +121 -0
  11. data/lib/mongoid/association/eager_load/polymorphic_preloader.rb +61 -0
  12. data/lib/mongoid/association/eager_load/polymorphic_targets.rb +151 -0
  13. data/lib/mongoid/association/eager_loadable.rb +65 -101
  14. data/lib/mongoid/association/embedded/batchable.rb +17 -1
  15. data/lib/mongoid/association/embedded/embeds_many/proxy.rb +6 -1
  16. data/lib/mongoid/association/embedded/embeds_one.rb +1 -0
  17. data/lib/mongoid/association/macros.rb +13 -1
  18. data/lib/mongoid/association/nested/many.rb +34 -5
  19. data/lib/mongoid/association/nested/nested_buildable.rb +14 -0
  20. data/lib/mongoid/association/nested/one.rb +6 -1
  21. data/lib/mongoid/association/options.rb +17 -0
  22. data/lib/mongoid/association/referenced/belongs_to/eager.rb +6 -1
  23. data/lib/mongoid/association/referenced/belongs_to.rb +1 -0
  24. data/lib/mongoid/association/referenced/counter_cache.rb +3 -3
  25. data/lib/mongoid/association/referenced/has_and_belongs_to_many/buildable.rb +6 -0
  26. data/lib/mongoid/association/referenced/has_and_belongs_to_many.rb +5 -0
  27. data/lib/mongoid/association/referenced/has_many/enumerable.rb +3 -1
  28. data/lib/mongoid/association/referenced/has_many/proxy.rb +100 -1
  29. data/lib/mongoid/association/referenced/has_many_through/eager.rb +99 -0
  30. data/lib/mongoid/association/referenced/has_many_through/proxy.rb +68 -0
  31. data/lib/mongoid/association/referenced/has_many_through.rb +201 -0
  32. data/lib/mongoid/association/referenced/has_one.rb +1 -0
  33. data/lib/mongoid/association/referenced/has_one_through/eager.rb +65 -0
  34. data/lib/mongoid/association/referenced/has_one_through/proxy.rb +30 -0
  35. data/lib/mongoid/association/referenced/has_one_through.rb +167 -0
  36. data/lib/mongoid/association/referenced.rb +2 -0
  37. data/lib/mongoid/association/relatable.rb +58 -0
  38. data/lib/mongoid/association.rb +15 -0
  39. data/lib/mongoid/atomic/modifiers.rb +100 -16
  40. data/lib/mongoid/clients/factory.rb +9 -13
  41. data/lib/mongoid/collection_configurable.rb +8 -0
  42. data/lib/mongoid/config/defaults.rb +0 -1
  43. data/lib/mongoid/config/encryption.rb +170 -22
  44. data/lib/mongoid/config.rb +56 -5
  45. data/lib/mongoid/contextual/aggregable/memory.rb +6 -2
  46. data/lib/mongoid/contextual/memory.rb +18 -7
  47. data/lib/mongoid/criteria/includable.rb +35 -11
  48. data/lib/mongoid/criteria/queryable/mergeable.rb +4 -0
  49. data/lib/mongoid/criteria/queryable/selectable.rb +123 -7
  50. data/lib/mongoid/deprecable.rb +45 -2
  51. data/lib/mongoid/document.rb +3 -0
  52. data/lib/mongoid/encryptable.rb +73 -0
  53. data/lib/mongoid/errors/config_redactor.rb +43 -0
  54. data/lib/mongoid/errors/in_memory_regexp_timeout.rb +26 -0
  55. data/lib/mongoid/errors/invalid_encryption_schema.rb +27 -0
  56. data/lib/mongoid/errors/mixed_client_configuration.rb +1 -1
  57. data/lib/mongoid/errors/no_client_database.rb +1 -1
  58. data/lib/mongoid/errors/no_client_hosts.rb +1 -1
  59. data/lib/mongoid/errors/no_encryption_schema.rb +30 -0
  60. data/lib/mongoid/errors/readonly_association.rb +29 -0
  61. data/lib/mongoid/errors.rb +5 -0
  62. data/lib/mongoid/extensions/vector.rb +64 -0
  63. data/lib/mongoid/extensions.rb +1 -0
  64. data/lib/mongoid/field_readable.rb +69 -0
  65. data/lib/mongoid/fields.rb +7 -0
  66. data/lib/mongoid/indexable.rb +38 -26
  67. data/lib/mongoid/matchable.rb +6 -1
  68. data/lib/mongoid/matcher/eq_impl_with_regexp.rb +2 -4
  69. data/lib/mongoid/matcher/regex.rb +11 -12
  70. data/lib/mongoid/matcher/regexp_budget.rb +372 -0
  71. data/lib/mongoid/matcher.rb +1 -0
  72. data/lib/mongoid/persistable/creatable.rb +120 -14
  73. data/lib/mongoid/persistable/updatable.rb +9 -11
  74. data/lib/mongoid/persistable.rb +15 -15
  75. data/lib/mongoid/persistence_context.rb +77 -0
  76. data/lib/mongoid/search_indexable.rb +69 -24
  77. data/lib/mongoid/serializable.rb +12 -4
  78. data/lib/mongoid/tasks/database.rb +33 -16
  79. data/lib/mongoid/threaded.rb +36 -0
  80. data/lib/mongoid/timestamps/timeless.rb +84 -10
  81. data/lib/mongoid/touchable.rb +28 -1
  82. data/lib/mongoid/traversable.rb +2 -2
  83. data/lib/mongoid/validatable/associated.rb +18 -2
  84. data/lib/mongoid/version.rb +1 -1
  85. data/lib/mongoid/warnings.rb +4 -0
  86. data/spec/integration/app_spec.rb +7 -1
  87. data/spec/integration/associations/belongs_to_spec.rb +13 -0
  88. data/spec/integration/associations/embedded_delayed_atomic_sets_spec.rb +216 -0
  89. data/spec/integration/associations/has_and_belongs_to_many_spec.rb +31 -2
  90. data/spec/integration/atomic/modifiers_spec.rb +283 -0
  91. data/spec/integration/dots_and_dollars_spec.rb +12 -2
  92. data/spec/integration/encryption_spec.rb +300 -0
  93. data/spec/integration/matcher_operator_data/regex.yml +21 -0
  94. data/spec/integration/matcher_regexp_timeout_spec.rb +215 -0
  95. data/spec/integration/query_operator_guard_spec.rb +89 -0
  96. data/spec/mongoid/association/accessors_spec.rb +35 -0
  97. data/spec/mongoid/association/eager_loadable_spec.rb +47 -0
  98. data/spec/mongoid/association/eager_spec.rb +2 -2
  99. data/spec/mongoid/association/embedded/embeds_many/proxy_spec.rb +33 -0
  100. data/spec/mongoid/association/fallback_spec.rb +425 -0
  101. data/spec/mongoid/association/macros_spec.rb +26 -0
  102. data/spec/mongoid/association/referenced/belongs_to/eager_spec.rb +17 -0
  103. data/spec/mongoid/association/referenced/has_and_belongs_to_many/buildable_spec.rb +11 -0
  104. data/spec/mongoid/association/referenced/has_and_belongs_to_many/proxy_spec.rb +48 -0
  105. data/spec/mongoid/association/referenced/has_and_belongs_to_many_spec.rb +50 -0
  106. data/spec/mongoid/association/referenced/has_many/eager_spec.rb +33 -0
  107. data/spec/mongoid/association/referenced/has_many/proxy_spec.rb +145 -0
  108. data/spec/mongoid/association/referenced/has_many_through/eager_spec.rb +58 -0
  109. data/spec/mongoid/association/referenced/has_many_through/proxy_spec.rb +62 -0
  110. data/spec/mongoid/association/referenced/has_many_through_spec.rb +399 -0
  111. data/spec/mongoid/association/referenced/has_one_through/eager_spec.rb +58 -0
  112. data/spec/mongoid/association/referenced/has_one_through/proxy_spec.rb +49 -0
  113. data/spec/mongoid/association/referenced/has_one_through_spec.rb +211 -0
  114. data/spec/mongoid/atomic/modifiers_spec.rb +347 -0
  115. data/spec/mongoid/attributes/nested_spec.rb +278 -8
  116. data/spec/mongoid/clients/factory_spec.rb +36 -0
  117. data/spec/mongoid/config/defaults_spec.rb +33 -0
  118. data/spec/mongoid/config/encryption_spec.rb +451 -0
  119. data/spec/mongoid/contextual/aggregable/memory_spec.rb +91 -0
  120. data/spec/mongoid/contextual/memory_spec.rb +177 -0
  121. data/spec/mongoid/criteria/includable_spec.rb +1221 -0
  122. data/spec/mongoid/criteria/queryable/selectable_logical_spec.rb +15 -1
  123. data/spec/mongoid/criteria/queryable/selectable_where_spec.rb +201 -0
  124. data/spec/mongoid/criteria_spec.rb +3 -0
  125. data/spec/mongoid/encryptable_spec.rb +115 -0
  126. data/spec/mongoid/errors/config_redactor_spec.rb +112 -0
  127. data/spec/mongoid/errors/mixed_client_configuration_spec.rb +31 -0
  128. data/spec/mongoid/errors/no_client_database_spec.rb +29 -0
  129. data/spec/mongoid/errors/no_client_hosts_spec.rb +26 -0
  130. data/spec/mongoid/errors/readonly_association_spec.rb +34 -0
  131. data/spec/mongoid/extensions/vector_spec.rb +175 -0
  132. data/spec/mongoid/fields_spec.rb +8 -0
  133. data/spec/mongoid/matcher/regexp_budget_spec.rb +570 -0
  134. data/spec/mongoid/search_indexable_spec.rb +321 -9
  135. data/spec/mongoid/tasks/database_spec.rb +31 -0
  136. data/spec/mongoid/timestamps/timeless_spec.rb +197 -5
  137. data/spec/mongoid/touchable_spec.rb +242 -0
  138. data/spec/mongoid/touchable_spec_models.rb +47 -0
  139. data/spec/mongoid/validatable/associated_spec.rb +19 -0
  140. data/spec/spec_helper.rb +5 -1
  141. data/spec/support/crypt/models.rb +253 -0
  142. data/spec/support/models/modifier_models.rb +65 -0
  143. metadata +55 -2
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mongoid
4
+ module Extensions
5
+ # Adds type-casting behavior to BSON::Vector class so that a field
6
+ # declared with +type: BSON::Vector+ is stored as a BSON binary of the
7
+ # vector subtype and read back as a BSON::Vector.
8
+ #
9
+ # Querying by an exact vector value (e.g. +where(embedding: vector)+) is
10
+ # not supported: BSON::Vector subclasses Array, so the criteria selector
11
+ # treats it as a list of elements rather than a scalar. Use Atlas Vector
12
+ # Search ($vectorSearch) for similarity queries on vector fields.
13
+ module Vector
14
+ # Turn the object from the ruby type we deal with to a Mongo friendly
15
+ # type.
16
+ #
17
+ # @example Mongoize the object.
18
+ # object.mongoize
19
+ #
20
+ # @return [ BSON::Binary | nil ] The object as a vector binary.
21
+ def mongoize
22
+ BSON::Vector.mongoize(self)
23
+ end
24
+
25
+ module ClassMethods
26
+ # Mongoize an object of any type to how it's stored in the db.
27
+ #
28
+ # @example Mongoize the object.
29
+ # BSON::Vector.mongoize(vector)
30
+ #
31
+ # @param [ Object ] object The object to Mongoize.
32
+ #
33
+ # @return [ BSON::Binary | nil ] A vector binary or nil.
34
+ def mongoize(object)
35
+ case object
36
+ when BSON::Vector then BSON::Binary.from_vector(object)
37
+ when BSON::Binary then object
38
+ end
39
+ end
40
+
41
+ # Convert the object from its mongo friendly ruby type back to a
42
+ # BSON::Vector.
43
+ #
44
+ # @example Demongoize the object.
45
+ # BSON::Vector.demongoize(binary)
46
+ #
47
+ # @param [ Object ] object The object to demongoize.
48
+ #
49
+ # @return [ BSON::Vector | nil ] The vector or nil.
50
+ def demongoize(object)
51
+ case object
52
+ when BSON::Binary then (object.type == :vector) ? object.as_vector : nil
53
+ when BSON::Vector then object
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
60
+
61
+ if defined?(BSON::Vector)
62
+ BSON::Vector.include Mongoid::Extensions::Vector
63
+ BSON::Vector.extend(Mongoid::Extensions::Vector::ClassMethods)
64
+ end
@@ -26,3 +26,4 @@ require 'mongoid/extensions/symbol'
26
26
  require 'mongoid/extensions/time'
27
27
  require 'mongoid/extensions/time_with_zone'
28
28
  require 'mongoid/extensions/true_class'
29
+ require 'mongoid/extensions/vector'
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mongoid
4
+ # Reads the value of a field name from a document without dispatching the
5
+ # name as an arbitrary method.
6
+ #
7
+ # Field names may come from application input, where a name like +destroy+
8
+ # or +attributes+ would delete the document or disclose its contents.
9
+ # Declared field and association names are validated when they are declared
10
+ # (see Mongoid.destructive_fields), so a name that resolves to one of them
11
+ # is safe to send to a document. Any other name is read from the attributes
12
+ # hash, which is what the database-backed query contexts do.
13
+ #
14
+ # @api private
15
+ module FieldReadable
16
+ private
17
+
18
+ # Read the value of the given field name from the given document.
19
+ #
20
+ # @param [ Document ] document The document to read from.
21
+ # @param [ String | Symbol ] name The name of the field.
22
+ #
23
+ # @return [ Object | nil ] The value of the field, or nil when the name
24
+ # is neither a declared field nor present in the attributes.
25
+ def read_field_value(document, name)
26
+ name = name.to_s
27
+ # A blank name cannot name a field. Return nil, which is what reading
28
+ # one has always done.
29
+ return nil if name.blank?
30
+
31
+ if (meth = readable_method_for(document.class, name))
32
+ document.public_send(meth)
33
+ else
34
+ document.attributes[document.class.database_field_name(name)]
35
+ end
36
+ end
37
+
38
+ # Resolve the given name to a method that is declared by the given class,
39
+ # and therefore safe to send to one of its instances.
40
+ #
41
+ # @param [ Class ] klass The document class.
42
+ # @param [ String ] name The name of the field.
43
+ #
44
+ # @return [ String | nil ] The method to send, or nil when the name is
45
+ # not declared by the class.
46
+ def readable_method_for(klass, name)
47
+ # Fields, associations, and field aliases each define a reader of their
48
+ # own name. Note that associations must be resolved before aliases: a
49
+ # belongs_to aliases its own name to its foreign key, and reading
50
+ # `band` must give the document, not the id.
51
+ return name if klass.relations.key?(name) ||
52
+ klass.fields.key?(name) ||
53
+ klass.aliased_fields.key?(name)
54
+
55
+ # An association may also be named by its `store_as`, which has no
56
+ # reader of its own, or by its ids accessor, which does.
57
+ if (assoc = klass.relations[klass.aliased_associations[name]])
58
+ return (assoc.store_as == name) ? assoc.name.to_s : name
59
+ end
60
+
61
+ # Localized fields also get a _translations reader, which does not
62
+ # appear in the fields hash.
63
+ base = name.delete_suffix(Fields::TRANSLATIONS_SFX)
64
+ return nil if base == name
65
+
66
+ name if klass.fields[klass.database_field_name(base)]&.localized?
67
+ end
68
+ end
69
+ end
@@ -412,6 +412,13 @@ module Mongoid
412
412
  return '' unless name.present?
413
413
 
414
414
  key = name.to_s
415
+
416
+ # A key without a "." refers to an immediate field, so it resolves to
417
+ # its alias (or itself) with no splitting. Short-circuiting here avoids
418
+ # the split and dup on every read, which is a hot path. This mirrors the
419
+ # same optimization already applied to the matcher.
420
+ return aliased_fields[key]&.dup || key unless key.include?('.')
421
+
415
422
  segment, remaining = key.split('.', 2)
416
423
 
417
424
  # Don't get the alias for the field when a belongs_to association
@@ -24,18 +24,9 @@ module Mongoid
24
24
  def create_indexes
25
25
  return unless index_specifications
26
26
 
27
- default_options = { background: Config.background_indexing }
28
-
29
- index_specifications.each do |spec|
30
- key, options = spec.key, default_options.merge(spec.options)
31
- if database = options[:database]
32
- with(database: database) do |klass|
33
- klass.collection.indexes(session: _session).create_one(key, options.except(:database))
34
- end
35
- else
36
- collection.indexes(session: _session).create_one(key, options)
37
- end
38
- end and true
27
+ Threaded.with_collection_management do
28
+ perform_create_indexes
29
+ end
39
30
  end
40
31
 
41
32
  # Send the actual index removal comments to the MongoDB driver,
@@ -46,20 +37,9 @@ module Mongoid
46
37
  #
47
38
  # @return [ true ] If the operation succeeded.
48
39
  def remove_indexes
49
- indexed_database_names.each do |database|
50
- with(database: database) do |klass|
51
- klass.collection.indexes(session: _session).each do |spec|
52
- next if spec['name'] == '_id_'
53
-
54
- klass.collection.indexes(session: _session).drop_one(spec['key'])
55
- logger.info(
56
- "MONGOID: Removed index '#{spec['name']}' on collection " +
57
- "'#{klass.collection.name}' in database '#{database}'."
58
- )
59
- end
60
- rescue Mongo::Error::OperationFailure
61
- end
62
- end and true
40
+ Threaded.with_collection_management do
41
+ perform_remove_indexes
42
+ end
63
43
  end
64
44
 
65
45
  # Add the default indexes to the root document if they do not already
@@ -119,6 +99,38 @@ module Mongoid
119
99
 
120
100
  private
121
101
 
102
+ def perform_create_indexes
103
+ default_options = { background: Config.background_indexing }
104
+
105
+ index_specifications.each do |spec|
106
+ key, options = spec.key, default_options.merge(spec.options)
107
+ if database = options[:database]
108
+ with(database: database) do |klass|
109
+ klass.collection.indexes(session: _session).create_one(key, options.except(:database))
110
+ end
111
+ else
112
+ collection.indexes(session: _session).create_one(key, options)
113
+ end
114
+ end and true
115
+ end
116
+
117
+ def perform_remove_indexes
118
+ indexed_database_names.each do |database|
119
+ with(database: database) do |klass|
120
+ klass.collection.indexes(session: _session).each do |spec|
121
+ next if spec['name'] == '_id_'
122
+
123
+ klass.collection.indexes(session: _session).drop_one(spec['key'])
124
+ logger.info(
125
+ "MONGOID: Removed index '#{spec['name']}' on collection " +
126
+ "'#{klass.collection.name}' in database '#{database}'."
127
+ )
128
+ end
129
+ rescue Mongo::Error::OperationFailure
130
+ end
131
+ end and true
132
+ end
133
+
122
134
  # Get the names of all databases for this model that have index
123
135
  # definitions.
124
136
  #
@@ -16,7 +16,12 @@ module Mongoid
16
16
  #
17
17
  # @return [ true | false ] True if matches, false if not.
18
18
  def _matches?(selector)
19
- Matcher::Expression.matches?(self, selector)
19
+ # Opens a regexp budget for this document only. Callers that match many
20
+ # documents against one selector open a budget of their own first, and
21
+ # this one joins it rather than giving every document a fresh limit.
22
+ Matcher::RegexpBudget.open(selector) do
23
+ Matcher::Expression.matches?(self, selector)
24
+ end
20
25
  end
21
26
  end
22
27
  end
@@ -18,10 +18,8 @@ module Mongoid
18
18
  # @api private
19
19
  module_function def matches?(_original_operator, value, condition)
20
20
  case condition
21
- when Regexp
22
- value.respond_to?(:=~) && value =~ condition
23
- when ::BSON::Regexp::Raw
24
- value.respond_to?(:=~) && value =~ condition.compile
21
+ when Regexp, ::BSON::Regexp::Raw
22
+ value.respond_to?(:=~) && RegexpBudget.match?(value, condition)
25
23
  else
26
24
  if value.is_a?(Time) && condition.is_a?(Time)
27
25
  EqImpl.time_eq?(value, condition)
@@ -16,24 +16,23 @@ module Mongoid
16
16
  #
17
17
  # @api private
18
18
  module_function def matches?(_exists, value, condition)
19
- condition = case condition
20
- when Regexp
21
- condition
22
- when BSON::Regexp::Raw
23
- condition.compile
24
- else
25
- # Note that strings must have been converted to a regular expression
26
- # instance already (with $options taken into account, if provided).
27
- raise Errors::InvalidQuery, "$regex requires a regular expression argument: #{Errors::InvalidQuery.truncate_expr(condition)}"
28
- end
19
+ unless condition.is_a?(Regexp) || condition.is_a?(BSON::Regexp::Raw)
20
+ # Note that strings must have been converted to a regular expression
21
+ # instance already (with $options taken into account, if provided).
22
+ raise Errors::InvalidQuery, "$regex requires a regular expression argument: #{Errors::InvalidQuery.truncate_expr(condition)}"
23
+ end
29
24
 
25
+ # The condition is compiled by RegexpBudget rather than here, so that
26
+ # the budget's timeout can be baked into the pattern.
30
27
  case value
31
28
  when Array
29
+ # Object#=~ is gone as of Ruby 3.2, so an element that cannot be
30
+ # matched against has to be rejected rather than passed to =~.
32
31
  value.any? do |v|
33
- v =~ condition
32
+ v.respond_to?(:=~) && RegexpBudget.match?(v, condition)
34
33
  end
35
34
  when String
36
- value =~ condition
35
+ RegexpBudget.match?(value, condition)
37
36
  else
38
37
  false
39
38
  end
@@ -0,0 +1,372 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'timeout'
4
+
5
+ module Mongoid
6
+ module Matcher
7
+ # Bounds the time spent executing regular expressions while evaluating a
8
+ # single in-memory match operation.
9
+ #
10
+ # A query condition can carry an application-supplied pattern, and the
11
+ # in-memory matcher compiles and runs that pattern in the caller's thread.
12
+ # Both the cost of one match and the number of matches performed are under
13
+ # the control of whoever supplied the condition, so the limit is cumulative
14
+ # over an entire operation rather than per match.
15
+ #
16
+ # The budget is held in thread- or fiber-local storage, so concurrent
17
+ # queries are accounted for independently.
18
+ #
19
+ # @api private
20
+ module RegexpBudget
21
+ # Whether a per-Regexp timeout can be relied on to reach Regexp.new.
22
+ #
23
+ # MRI added them in 3.2. JRuby 10.0.6 defines Regexp::TimeoutError,
24
+ # reports Ruby 3.4, and does honour a timeout that reaches it, but its
25
+ # Regexp.new accepts the keyword only for the first couple of calls
26
+ # through a given call site and raises ArgumentError from then on.
27
+ # Because that breakage is per call site, no load-time probe can predict
28
+ # it: a probe at its own call site reports a capability that the call in
29
+ # Budget#compile does not have. So non-MRI engines are excluded outright
30
+ # and use the Timeout fallback, which does interrupt a Joni match already
31
+ # under way. Worth revisiting if JRuby fixes the keyword handling.
32
+ PER_REGEXP_TIMEOUT =
33
+ if RUBY_ENGINE == 'ruby' && defined?(::Regexp::TimeoutError)
34
+ true
35
+ else
36
+ false
37
+ end
38
+
39
+ # The exception raised by a per-Regexp timeout. Tied to the constant
40
+ # rather than to the probe, so that a timeout set some other way (an
41
+ # application assigning Regexp.timeout, say) is still translated. On
42
+ # Rubies with no such constant, a class that is never raised stands in.
43
+ TIMEOUT_ERROR = defined?(::Regexp::TimeoutError) ? ::Regexp::TimeoutError : Class.new(StandardError)
44
+
45
+ # Raised by Timeout on Rubies without per-Regexp timeouts, and converted
46
+ # immediately. It is private to this module so that an application's own
47
+ # Timeout, firing inside our block, is never mistaken for ours.
48
+ class TimedOut < StandardError; end
49
+
50
+ # The state of one open budget: what is left of the limit, and the
51
+ # patterns compiled under it.
52
+ #
53
+ # @api private
54
+ class Budget
55
+ # @return [ Float ] The limit this budget started with.
56
+ attr_reader :limit
57
+
58
+ # @return [ Float ] The seconds left before the budget is spent.
59
+ attr_reader :remaining
60
+
61
+ # @param [ Float ] limit The seconds this budget may spend.
62
+ def initialize(limit)
63
+ @limit = limit
64
+ @remaining = limit
65
+ @cache = {}
66
+ # A per-Regexp timeout bounds one match; the cumulative budget bounds
67
+ # the operation. So the timeout is fixed for the life of the scope
68
+ # rather than following the drawdown, which is what lets a pattern be
69
+ # compiled once instead of once per match. It means a match that
70
+ # starts with almost nothing left can still run for a whole limit, so
71
+ # an operation can overshoot by at most one limit -- bounded, which is
72
+ # the point, and far cheaper than recompiling.
73
+ #
74
+ # An application that has set a stricter global Regexp.timeout keeps
75
+ # it: baking in a larger value would leave it less protected than it
76
+ # asked to be. A timeout set on one individual pattern is a different
77
+ # matter, and is not preserved -- Budget#compile rebuilds the pattern
78
+ # from its source and flags, neither of which carries one, so this
79
+ # value takes its place. Only trusted code can supply such a pattern:
80
+ # a condition decoded from JSON or BSON arrives as a string or a
81
+ # BSON::Regexp::Raw, with no timeout of its own.
82
+ @timeout = [ limit, ::Regexp.timeout ].compact.min if PER_REGEXP_TIMEOUT
83
+ end
84
+
85
+ # Draws the elapsed time down from the budget.
86
+ #
87
+ # @param [ Float ] elapsed The seconds to charge.
88
+ def charge(elapsed)
89
+ @remaining -= elapsed
90
+ end
91
+
92
+ # @return [ true | false ] Whether the budget is spent.
93
+ def exhausted?
94
+ @remaining <= 0
95
+ end
96
+
97
+ # Returns the condition as a Regexp which, where the Ruby in use
98
+ # supports it, gives up once its timeout is spent.
99
+ #
100
+ # The condition is taken uncompiled so that the cache can answer before
101
+ # any compiling happens. A BSON::Regexp::Raw memoizes its own compile,
102
+ # but FieldExpression builds a fresh one for every $regex it evaluates,
103
+ # so that memo is worth nothing across documents and the source would
104
+ # otherwise be compiled once per document.
105
+ #
106
+ # @param [ Regexp | BSON::Regexp::Raw ] condition The condition.
107
+ #
108
+ # @return [ Regexp ] The compiled pattern.
109
+ def compile(condition)
110
+ @cache[cache_key(condition)] ||= bake(RegexpBudget.coerce(condition))
111
+ end
112
+
113
+ private
114
+
115
+ # BSON::Regexp::Raw aliases eql? to == but leaves hash alone, so two
116
+ # equal instances hash differently and cannot key the cache. What they
117
+ # are equal by can. Anything else keys on itself and is left to coerce
118
+ # to reject.
119
+ def cache_key(condition)
120
+ case condition
121
+ when BSON::Regexp::Raw then [ condition.pattern, condition.options ]
122
+ else condition
123
+ end
124
+ end
125
+
126
+ # Rebuilds the pattern with the budget's timeout, where the Ruby in use
127
+ # has them.
128
+ def bake(regexp)
129
+ return regexp unless PER_REGEXP_TIMEOUT
130
+
131
+ ::Regexp.new(regexp.source, regexp.options, timeout: @timeout)
132
+ end
133
+ end
134
+
135
+ class << self
136
+ # Opens a budget scope for the duration of the block.
137
+ #
138
+ # Nested calls join the enclosing budget instead of starting a new one,
139
+ # which is what lets a scan over many documents share a single limit.
140
+ # It also keeps the recursion in Expression.matches? (through
141
+ # $elemMatch, $and, $or and $nor) from resetting the budget.
142
+ #
143
+ # No budget is opened for a selector that carries no regular
144
+ # expression. There would be nothing for it to bound, and on the
145
+ # Timeout path it would put a deadline on in-memory work that has
146
+ # nothing to do with regular expressions.
147
+ #
148
+ # The scope covers everything nested inside the block, a selector other
149
+ # than this one included: a nested call joins the scope rather than
150
+ # deciding for itself, which is what keeps a scan from walking the
151
+ # selector once per document. Where the scope has nothing to bound, that
152
+ # means nested selectors are not bounded either -- so do not open one
153
+ # around work that can run application code. Loading documents runs find
154
+ # callbacks, and a query in one of those brings its own selector.
155
+ #
156
+ # Where a selector does carry one, the Timeout path still measures the
157
+ # whole scope rather than the matching alone, so a long scan can trip
158
+ # the limit with a cheap pattern. That imprecision is accepted: the
159
+ # alternative is a Timeout around each individual match, which was
160
+ # measured at about nine seconds per million matches, and the limit
161
+ # exists to bound a scan of exactly that size. The error message is
162
+ # worded to hold either way, and Rubies with per-Regexp timeouts --
163
+ # every supported MRI from 3.2 on -- do not take this path at all.
164
+ #
165
+ # Code inside the block that mutates state should be wrapped in
166
+ # .protect, since on Rubies without a per-Regexp timeout the budget is
167
+ # enforced with an asynchronous exception that can land anywhere.
168
+ #
169
+ # @param [ Hash ] selector The selector about to be evaluated.
170
+ #
171
+ # @return [ Object ] The value of the block.
172
+ def open(selector, &block)
173
+ # The key is present but nil where an enclosing scope found nothing
174
+ # to bound, so that a nested call does not scan the selector again.
175
+ # Deciding before this check, in a default argument say, would walk
176
+ # the selector once per document on a scan.
177
+ return yield if Threaded.has?(Threaded::REGEXP_BUDGET_KEY)
178
+
179
+ open_with(limit_for(selector), &block)
180
+ end
181
+
182
+ # Opens a budget scope for a limit the caller has already decided on.
183
+ #
184
+ # A caller that rearranges its work around the decision -- loading
185
+ # documents up front so that nothing is mutated before the scan
186
+ # finishes, say -- has to make it before it can act on it, and must not
187
+ # then make it a second time. Asking .limit_for and letting .open ask
188
+ # again reads the configured limit twice, and the two reads can differ:
189
+ # a limit that becomes positive in between would establish a budget in
190
+ # the branch that was chosen for not needing one, and on the Timeout
191
+ # path that arms a deadline over work the branch never made
192
+ # interruptible.
193
+ #
194
+ # See .open for what the scope does and does not bound, and for the
195
+ # note about mutating state inside it.
196
+ #
197
+ # @param [ Float | nil ] limit The seconds the scope may spend, or nil
198
+ # for a scope with nothing to bound.
199
+ #
200
+ # @return [ Object ] The value of the block.
201
+ def open_with(limit, &block)
202
+ return yield if Threaded.has?(Threaded::REGEXP_BUDGET_KEY)
203
+
204
+ budget = Budget.new(limit) if limit&.positive?
205
+
206
+ begin
207
+ # Set inside the begin so that an asynchronous exception from an
208
+ # enclosing timeout cannot leave the key behind on a pooled thread.
209
+ Threaded.set(Threaded::REGEXP_BUDGET_KEY, budget)
210
+
211
+ if budget.nil? || PER_REGEXP_TIMEOUT
212
+ yield
213
+ else
214
+ begin
215
+ Timeout.timeout(budget.limit, TimedOut, &block)
216
+ rescue TimedOut
217
+ raise timeout_error(budget)
218
+ end
219
+ end
220
+ ensure
221
+ Threaded.delete(Threaded::REGEXP_BUDGET_KEY)
222
+ end
223
+ end
224
+
225
+ # The limit a scope evaluating this selector would be bounded by.
226
+ #
227
+ # A caller that has to rearrange its work to make the scan
228
+ # interruptible -- loading documents up front so that nothing is
229
+ # mutated before the scan finishes, say -- can ask this first and skip
230
+ # the rearrangement, and whatever it costs, when there is no pattern to
231
+ # bound. It then passes what it got to .open_with, so that the decision
232
+ # it acted on is the one the scope is opened with. Callers with nothing
233
+ # to rearrange should just call .open, which asks this itself.
234
+ #
235
+ # @param [ Hash ] selector The selector about to be evaluated.
236
+ #
237
+ # @return [ Float | nil ] The limit, or nil where there is nothing to
238
+ # bound.
239
+ def limit_for(selector)
240
+ # nil.to_f is 0.0, so an unset limit and a limit of zero or less are
241
+ # the same thing here: no limit. Zero is a common way to spell
242
+ # "disabled", and taking it literally would mean a budget that is
243
+ # spent before the first match and a query that can never run.
244
+ limit = Mongoid::Config.in_memory_regexp_time_limit.to_f
245
+ return nil unless limit.positive?
246
+
247
+ limit if contains_regexp?(selector)
248
+ end
249
+
250
+ # Matches a value against a regular expression condition, charging the
251
+ # time it takes against the open budget.
252
+ #
253
+ # @param [ Object ] value The value to match.
254
+ # @param [ Regexp | BSON::Regexp::Raw ] condition The condition.
255
+ #
256
+ # @raise [ Errors::InMemoryRegexpTimeout ] if the budget is exhausted.
257
+ #
258
+ # @return [ Integer | nil ] The offset of the match, or nil.
259
+ def match?(value, condition)
260
+ budget = current
261
+ return value =~ coerce(condition) unless budget
262
+
263
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
264
+ pattern = nil
265
+ begin
266
+ raise timeout_error(budget) if budget.exhausted?
267
+
268
+ # Compiling is charged too. It is not free, and for a pattern with
269
+ # very many branches it costs far more than running the pattern
270
+ # does, so leaving it out would leave a way to spend unbounded time
271
+ # without the budget ever noticing.
272
+ pattern = budget.compile(condition)
273
+ value =~ pattern
274
+ rescue TIMEOUT_ERROR
275
+ # Name the limit that actually fired, which is not always the
276
+ # budget's. A baked pattern carries it: the smaller of the budget's
277
+ # limit and any global Regexp.timeout the application has set.
278
+ # Where nothing was baked -- an engine that raises this error but
279
+ # will not take a per-Regexp timeout, which is JRuby -- the global
280
+ # is the only thing that can have fired, and the pattern reports
281
+ # nil. Naming the budget's limit in either case would state a time
282
+ # that was never spent and send the reader after a setting that is
283
+ # not the one in the way.
284
+ #
285
+ # Both readers arrived together with Regexp::TimeoutError, so every
286
+ # engine that can reach this rescue at all has them.
287
+ raise timeout_error(budget, pattern&.timeout || ::Regexp.timeout || budget.limit)
288
+ ensure
289
+ budget.charge(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
290
+ end
291
+ end
292
+
293
+ # Runs the block without letting a scope timeout tear it in half.
294
+ #
295
+ # Where the budget is enforced with Timeout, the exception is raised
296
+ # asynchronously and can arrive at any point. Wrapping a mutation in
297
+ # this holds the exception back until the block has finished, so the
298
+ # interruption is deferred rather than given up.
299
+ #
300
+ # @return [ Object ] The value of the block.
301
+ def protect(&block)
302
+ Thread.handle_interrupt(TimedOut => :never, &block)
303
+ end
304
+
305
+ # The time left in the open budget, or nil when no budget is open.
306
+ #
307
+ # @return [ Float | nil ] The remaining seconds.
308
+ def remaining
309
+ current&.remaining
310
+ end
311
+
312
+ # Returns the condition as a Regexp, without a timeout.
313
+ #
314
+ # @param [ Regexp | BSON::Regexp::Raw ] condition The condition.
315
+ #
316
+ # @return [ Regexp ] The pattern.
317
+ def coerce(condition)
318
+ case condition
319
+ when ::Regexp then condition
320
+ when BSON::Regexp::Raw then condition.compile
321
+ else raise ArgumentError, "Not a regular expression: #{condition.inspect}"
322
+ end
323
+ end
324
+
325
+ private
326
+
327
+ # The budget for the open scope, if there is one.
328
+ #
329
+ # @return [ Budget | nil ] The open budget.
330
+ def current
331
+ Threaded.get(Threaded::REGEXP_BUDGET_KEY)
332
+ end
333
+
334
+ # Whether evaluating the selector could run a regular expression.
335
+ #
336
+ # A string under $regex counts: FieldExpression turns it into a pattern
337
+ # at match time.
338
+ def contains_regexp?(object)
339
+ case object
340
+ when ::Regexp, BSON::Regexp::Raw
341
+ true
342
+ when Hash
343
+ object.any? do |k, v|
344
+ k.to_s == '$regex' || contains_regexp?(v)
345
+ end
346
+ when Array
347
+ object.any? { |v| contains_regexp?(v) }
348
+ else
349
+ false
350
+ end
351
+ end
352
+
353
+ # Builds the error with the scope timeout held back.
354
+ #
355
+ # Composing the message goes through I18n, which reads locale files the
356
+ # first time it runs. An asynchronous TimedOut landing in the middle of
357
+ # that is caught by I18n and reraised as a locale-loading failure, so
358
+ # the real error never surfaces.
359
+ #
360
+ # @param [ Budget ] budget The open budget.
361
+ # @param [ Float ] limit The limit that was exceeded. Defaults to the
362
+ # budget's own, which is the right one to name everywhere the budget
363
+ # itself ran out.
364
+ def timeout_error(budget, limit = budget.limit)
365
+ protect do
366
+ Errors::InMemoryRegexpTimeout.new(limit)
367
+ end
368
+ end
369
+ end
370
+ end
371
+ end
372
+ end
@@ -134,6 +134,7 @@ require 'mongoid/matcher/nor'
134
134
  require 'mongoid/matcher/not'
135
135
  require 'mongoid/matcher/or'
136
136
  require 'mongoid/matcher/regex'
137
+ require 'mongoid/matcher/regexp_budget'
137
138
  require 'mongoid/matcher/size'
138
139
  require 'mongoid/matcher/type'
139
140
  require 'mongoid/matcher/expression_operator'