mongoid 9.1.0 → 9.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/config/locales/en.yml +49 -0
- data/lib/mongoid/association/accessors.rb +9 -1
- data/lib/mongoid/association/depending.rb +14 -10
- data/lib/mongoid/association/eager.rb +9 -0
- data/lib/mongoid/association/eager_load/discriminated_inclusion.rb +88 -0
- data/lib/mongoid/association/eager_load/embedded_distributor.rb +151 -0
- data/lib/mongoid/association/eager_load/inclusion.rb +173 -0
- data/lib/mongoid/association/eager_load/inclusion_tree.rb +96 -0
- data/lib/mongoid/association/eager_load/lookup_pipeline.rb +121 -0
- data/lib/mongoid/association/eager_load/polymorphic_preloader.rb +61 -0
- data/lib/mongoid/association/eager_load/polymorphic_targets.rb +151 -0
- data/lib/mongoid/association/eager_loadable.rb +57 -100
- data/lib/mongoid/association/embedded/embeds_many/proxy.rb +6 -1
- data/lib/mongoid/association/embedded/embeds_one.rb +1 -0
- data/lib/mongoid/association/macros.rb +13 -1
- data/lib/mongoid/association/nested/many.rb +34 -5
- data/lib/mongoid/association/nested/nested_buildable.rb +14 -0
- data/lib/mongoid/association/nested/one.rb +6 -1
- data/lib/mongoid/association/options.rb +17 -0
- data/lib/mongoid/association/referenced/belongs_to/eager.rb +6 -1
- data/lib/mongoid/association/referenced/belongs_to.rb +1 -0
- data/lib/mongoid/association/referenced/counter_cache.rb +3 -3
- data/lib/mongoid/association/referenced/has_and_belongs_to_many.rb +5 -0
- data/lib/mongoid/association/referenced/has_many/enumerable.rb +3 -1
- data/lib/mongoid/association/referenced/has_many/proxy.rb +100 -1
- data/lib/mongoid/association/referenced/has_many_through/eager.rb +99 -0
- data/lib/mongoid/association/referenced/has_many_through/proxy.rb +68 -0
- data/lib/mongoid/association/referenced/has_many_through.rb +201 -0
- data/lib/mongoid/association/referenced/has_one.rb +1 -0
- data/lib/mongoid/association/referenced/has_one_through/eager.rb +65 -0
- data/lib/mongoid/association/referenced/has_one_through/proxy.rb +30 -0
- data/lib/mongoid/association/referenced/has_one_through.rb +167 -0
- data/lib/mongoid/association/referenced.rb +2 -0
- data/lib/mongoid/association/relatable.rb +40 -0
- data/lib/mongoid/association.rb +7 -0
- data/lib/mongoid/clients/factory.rb +7 -13
- data/lib/mongoid/collection_configurable.rb +8 -0
- data/lib/mongoid/config/defaults.rb +0 -1
- data/lib/mongoid/config/encryption.rb +36 -19
- data/lib/mongoid/config.rb +56 -5
- data/lib/mongoid/contextual/aggregable/memory.rb +6 -2
- data/lib/mongoid/contextual/memory.rb +18 -7
- data/lib/mongoid/criteria/includable.rb +35 -11
- data/lib/mongoid/criteria/queryable/mergeable.rb +4 -0
- data/lib/mongoid/criteria/queryable/selectable.rb +123 -7
- data/lib/mongoid/deprecable.rb +45 -2
- data/lib/mongoid/encryptable.rb +46 -0
- data/lib/mongoid/errors/config_redactor.rb +43 -0
- data/lib/mongoid/errors/in_memory_regexp_timeout.rb +26 -0
- data/lib/mongoid/errors/mixed_client_configuration.rb +1 -1
- data/lib/mongoid/errors/no_client_database.rb +1 -1
- data/lib/mongoid/errors/no_client_hosts.rb +1 -1
- data/lib/mongoid/errors/no_encryption_schema.rb +28 -0
- data/lib/mongoid/errors/readonly_association.rb +29 -0
- data/lib/mongoid/errors.rb +4 -0
- data/lib/mongoid/extensions/vector.rb +64 -0
- data/lib/mongoid/extensions.rb +1 -0
- data/lib/mongoid/field_readable.rb +69 -0
- data/lib/mongoid/indexable.rb +38 -26
- data/lib/mongoid/matchable.rb +6 -1
- data/lib/mongoid/matcher/eq_impl_with_regexp.rb +2 -4
- data/lib/mongoid/matcher/regex.rb +11 -12
- data/lib/mongoid/matcher/regexp_budget.rb +372 -0
- data/lib/mongoid/matcher.rb +1 -0
- data/lib/mongoid/persistable/creatable.rb +1 -1
- data/lib/mongoid/persistable/updatable.rb +9 -11
- data/lib/mongoid/persistence_context.rb +35 -0
- data/lib/mongoid/search_indexable.rb +64 -23
- data/lib/mongoid/serializable.rb +12 -4
- data/lib/mongoid/tasks/database.rb +23 -9
- data/lib/mongoid/threaded.rb +36 -0
- data/lib/mongoid/timestamps/timeless.rb +84 -10
- data/lib/mongoid/traversable.rb +2 -2
- data/lib/mongoid/validatable/associated.rb +18 -2
- data/lib/mongoid/version.rb +1 -1
- data/spec/integration/app_spec.rb +7 -1
- data/spec/integration/associations/belongs_to_spec.rb +13 -0
- data/spec/integration/associations/has_and_belongs_to_many_spec.rb +17 -2
- data/spec/integration/dots_and_dollars_spec.rb +12 -2
- data/spec/integration/encryption_spec.rb +151 -0
- data/spec/integration/matcher_operator_data/regex.yml +21 -0
- data/spec/integration/matcher_regexp_timeout_spec.rb +215 -0
- data/spec/integration/query_operator_guard_spec.rb +89 -0
- data/spec/mongoid/association/eager_loadable_spec.rb +47 -0
- data/spec/mongoid/association/eager_spec.rb +2 -2
- data/spec/mongoid/association/embedded/embeds_many/proxy_spec.rb +33 -0
- data/spec/mongoid/association/fallback_spec.rb +425 -0
- data/spec/mongoid/association/macros_spec.rb +26 -0
- data/spec/mongoid/association/referenced/belongs_to/eager_spec.rb +17 -0
- data/spec/mongoid/association/referenced/has_and_belongs_to_many/proxy_spec.rb +48 -0
- data/spec/mongoid/association/referenced/has_and_belongs_to_many_spec.rb +50 -0
- data/spec/mongoid/association/referenced/has_many/eager_spec.rb +33 -0
- data/spec/mongoid/association/referenced/has_many/proxy_spec.rb +145 -0
- data/spec/mongoid/association/referenced/has_many_through/eager_spec.rb +58 -0
- data/spec/mongoid/association/referenced/has_many_through/proxy_spec.rb +62 -0
- data/spec/mongoid/association/referenced/has_many_through_spec.rb +399 -0
- data/spec/mongoid/association/referenced/has_one_through/eager_spec.rb +58 -0
- data/spec/mongoid/association/referenced/has_one_through/proxy_spec.rb +49 -0
- data/spec/mongoid/association/referenced/has_one_through_spec.rb +211 -0
- data/spec/mongoid/attributes/nested_spec.rb +249 -8
- data/spec/mongoid/config/defaults_spec.rb +33 -0
- data/spec/mongoid/config/encryption_spec.rb +228 -0
- data/spec/mongoid/contextual/aggregable/memory_spec.rb +91 -0
- data/spec/mongoid/contextual/memory_spec.rb +177 -0
- data/spec/mongoid/criteria/includable_spec.rb +875 -0
- data/spec/mongoid/criteria/queryable/selectable_logical_spec.rb +15 -1
- data/spec/mongoid/criteria/queryable/selectable_where_spec.rb +201 -0
- data/spec/mongoid/criteria_spec.rb +3 -0
- data/spec/mongoid/encryptable_spec.rb +61 -0
- data/spec/mongoid/errors/config_redactor_spec.rb +112 -0
- data/spec/mongoid/errors/mixed_client_configuration_spec.rb +31 -0
- data/spec/mongoid/errors/no_client_database_spec.rb +29 -0
- data/spec/mongoid/errors/no_client_hosts_spec.rb +26 -0
- data/spec/mongoid/errors/readonly_association_spec.rb +34 -0
- data/spec/mongoid/extensions/vector_spec.rb +175 -0
- data/spec/mongoid/matcher/regexp_budget_spec.rb +570 -0
- data/spec/mongoid/search_indexable_spec.rb +295 -9
- data/spec/mongoid/tasks/database_spec.rb +18 -0
- data/spec/mongoid/timestamps/timeless_spec.rb +197 -5
- data/spec/mongoid/touchable_spec.rb +25 -0
- data/spec/mongoid/touchable_spec_models.rb +10 -0
- data/spec/mongoid/validatable/associated_spec.rb +19 -0
- data/spec/spec_helper.rb +5 -1
- data/spec/support/crypt/models.rb +157 -0
- metadata +52 -2
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'timeout'
|
|
4
|
+
|
|
5
|
+
module Mongoid
|
|
6
|
+
module Matcher
|
|
7
|
+
# Bounds the time spent executing regular expressions while evaluating a
|
|
8
|
+
# single in-memory match operation.
|
|
9
|
+
#
|
|
10
|
+
# A query condition can carry an application-supplied pattern, and the
|
|
11
|
+
# in-memory matcher compiles and runs that pattern in the caller's thread.
|
|
12
|
+
# Both the cost of one match and the number of matches performed are under
|
|
13
|
+
# the control of whoever supplied the condition, so the limit is cumulative
|
|
14
|
+
# over an entire operation rather than per match.
|
|
15
|
+
#
|
|
16
|
+
# The budget is held in thread- or fiber-local storage, so concurrent
|
|
17
|
+
# queries are accounted for independently.
|
|
18
|
+
#
|
|
19
|
+
# @api private
|
|
20
|
+
module RegexpBudget
|
|
21
|
+
# Whether a per-Regexp timeout can be relied on to reach Regexp.new.
|
|
22
|
+
#
|
|
23
|
+
# MRI added them in 3.2. JRuby 10.0.6 defines Regexp::TimeoutError,
|
|
24
|
+
# reports Ruby 3.4, and does honour a timeout that reaches it, but its
|
|
25
|
+
# Regexp.new accepts the keyword only for the first couple of calls
|
|
26
|
+
# through a given call site and raises ArgumentError from then on.
|
|
27
|
+
# Because that breakage is per call site, no load-time probe can predict
|
|
28
|
+
# it: a probe at its own call site reports a capability that the call in
|
|
29
|
+
# Budget#compile does not have. So non-MRI engines are excluded outright
|
|
30
|
+
# and use the Timeout fallback, which does interrupt a Joni match already
|
|
31
|
+
# under way. Worth revisiting if JRuby fixes the keyword handling.
|
|
32
|
+
PER_REGEXP_TIMEOUT =
|
|
33
|
+
if RUBY_ENGINE == 'ruby' && defined?(::Regexp::TimeoutError)
|
|
34
|
+
true
|
|
35
|
+
else
|
|
36
|
+
false
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# The exception raised by a per-Regexp timeout. Tied to the constant
|
|
40
|
+
# rather than to the probe, so that a timeout set some other way (an
|
|
41
|
+
# application assigning Regexp.timeout, say) is still translated. On
|
|
42
|
+
# Rubies with no such constant, a class that is never raised stands in.
|
|
43
|
+
TIMEOUT_ERROR = defined?(::Regexp::TimeoutError) ? ::Regexp::TimeoutError : Class.new(StandardError)
|
|
44
|
+
|
|
45
|
+
# Raised by Timeout on Rubies without per-Regexp timeouts, and converted
|
|
46
|
+
# immediately. It is private to this module so that an application's own
|
|
47
|
+
# Timeout, firing inside our block, is never mistaken for ours.
|
|
48
|
+
class TimedOut < StandardError; end
|
|
49
|
+
|
|
50
|
+
# The state of one open budget: what is left of the limit, and the
|
|
51
|
+
# patterns compiled under it.
|
|
52
|
+
#
|
|
53
|
+
# @api private
|
|
54
|
+
class Budget
|
|
55
|
+
# @return [ Float ] The limit this budget started with.
|
|
56
|
+
attr_reader :limit
|
|
57
|
+
|
|
58
|
+
# @return [ Float ] The seconds left before the budget is spent.
|
|
59
|
+
attr_reader :remaining
|
|
60
|
+
|
|
61
|
+
# @param [ Float ] limit The seconds this budget may spend.
|
|
62
|
+
def initialize(limit)
|
|
63
|
+
@limit = limit
|
|
64
|
+
@remaining = limit
|
|
65
|
+
@cache = {}
|
|
66
|
+
# A per-Regexp timeout bounds one match; the cumulative budget bounds
|
|
67
|
+
# the operation. So the timeout is fixed for the life of the scope
|
|
68
|
+
# rather than following the drawdown, which is what lets a pattern be
|
|
69
|
+
# compiled once instead of once per match. It means a match that
|
|
70
|
+
# starts with almost nothing left can still run for a whole limit, so
|
|
71
|
+
# an operation can overshoot by at most one limit -- bounded, which is
|
|
72
|
+
# the point, and far cheaper than recompiling.
|
|
73
|
+
#
|
|
74
|
+
# An application that has set a stricter global Regexp.timeout keeps
|
|
75
|
+
# it: baking in a larger value would leave it less protected than it
|
|
76
|
+
# asked to be. A timeout set on one individual pattern is a different
|
|
77
|
+
# matter, and is not preserved -- Budget#compile rebuilds the pattern
|
|
78
|
+
# from its source and flags, neither of which carries one, so this
|
|
79
|
+
# value takes its place. Only trusted code can supply such a pattern:
|
|
80
|
+
# a condition decoded from JSON or BSON arrives as a string or a
|
|
81
|
+
# BSON::Regexp::Raw, with no timeout of its own.
|
|
82
|
+
@timeout = [ limit, ::Regexp.timeout ].compact.min if PER_REGEXP_TIMEOUT
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Draws the elapsed time down from the budget.
|
|
86
|
+
#
|
|
87
|
+
# @param [ Float ] elapsed The seconds to charge.
|
|
88
|
+
def charge(elapsed)
|
|
89
|
+
@remaining -= elapsed
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# @return [ true | false ] Whether the budget is spent.
|
|
93
|
+
def exhausted?
|
|
94
|
+
@remaining <= 0
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Returns the condition as a Regexp which, where the Ruby in use
|
|
98
|
+
# supports it, gives up once its timeout is spent.
|
|
99
|
+
#
|
|
100
|
+
# The condition is taken uncompiled so that the cache can answer before
|
|
101
|
+
# any compiling happens. A BSON::Regexp::Raw memoizes its own compile,
|
|
102
|
+
# but FieldExpression builds a fresh one for every $regex it evaluates,
|
|
103
|
+
# so that memo is worth nothing across documents and the source would
|
|
104
|
+
# otherwise be compiled once per document.
|
|
105
|
+
#
|
|
106
|
+
# @param [ Regexp | BSON::Regexp::Raw ] condition The condition.
|
|
107
|
+
#
|
|
108
|
+
# @return [ Regexp ] The compiled pattern.
|
|
109
|
+
def compile(condition)
|
|
110
|
+
@cache[cache_key(condition)] ||= bake(RegexpBudget.coerce(condition))
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
private
|
|
114
|
+
|
|
115
|
+
# BSON::Regexp::Raw aliases eql? to == but leaves hash alone, so two
|
|
116
|
+
# equal instances hash differently and cannot key the cache. What they
|
|
117
|
+
# are equal by can. Anything else keys on itself and is left to coerce
|
|
118
|
+
# to reject.
|
|
119
|
+
def cache_key(condition)
|
|
120
|
+
case condition
|
|
121
|
+
when BSON::Regexp::Raw then [ condition.pattern, condition.options ]
|
|
122
|
+
else condition
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Rebuilds the pattern with the budget's timeout, where the Ruby in use
|
|
127
|
+
# has them.
|
|
128
|
+
def bake(regexp)
|
|
129
|
+
return regexp unless PER_REGEXP_TIMEOUT
|
|
130
|
+
|
|
131
|
+
::Regexp.new(regexp.source, regexp.options, timeout: @timeout)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
class << self
|
|
136
|
+
# Opens a budget scope for the duration of the block.
|
|
137
|
+
#
|
|
138
|
+
# Nested calls join the enclosing budget instead of starting a new one,
|
|
139
|
+
# which is what lets a scan over many documents share a single limit.
|
|
140
|
+
# It also keeps the recursion in Expression.matches? (through
|
|
141
|
+
# $elemMatch, $and, $or and $nor) from resetting the budget.
|
|
142
|
+
#
|
|
143
|
+
# No budget is opened for a selector that carries no regular
|
|
144
|
+
# expression. There would be nothing for it to bound, and on the
|
|
145
|
+
# Timeout path it would put a deadline on in-memory work that has
|
|
146
|
+
# nothing to do with regular expressions.
|
|
147
|
+
#
|
|
148
|
+
# The scope covers everything nested inside the block, a selector other
|
|
149
|
+
# than this one included: a nested call joins the scope rather than
|
|
150
|
+
# deciding for itself, which is what keeps a scan from walking the
|
|
151
|
+
# selector once per document. Where the scope has nothing to bound, that
|
|
152
|
+
# means nested selectors are not bounded either -- so do not open one
|
|
153
|
+
# around work that can run application code. Loading documents runs find
|
|
154
|
+
# callbacks, and a query in one of those brings its own selector.
|
|
155
|
+
#
|
|
156
|
+
# Where a selector does carry one, the Timeout path still measures the
|
|
157
|
+
# whole scope rather than the matching alone, so a long scan can trip
|
|
158
|
+
# the limit with a cheap pattern. That imprecision is accepted: the
|
|
159
|
+
# alternative is a Timeout around each individual match, which was
|
|
160
|
+
# measured at about nine seconds per million matches, and the limit
|
|
161
|
+
# exists to bound a scan of exactly that size. The error message is
|
|
162
|
+
# worded to hold either way, and Rubies with per-Regexp timeouts --
|
|
163
|
+
# every supported MRI from 3.2 on -- do not take this path at all.
|
|
164
|
+
#
|
|
165
|
+
# Code inside the block that mutates state should be wrapped in
|
|
166
|
+
# .protect, since on Rubies without a per-Regexp timeout the budget is
|
|
167
|
+
# enforced with an asynchronous exception that can land anywhere.
|
|
168
|
+
#
|
|
169
|
+
# @param [ Hash ] selector The selector about to be evaluated.
|
|
170
|
+
#
|
|
171
|
+
# @return [ Object ] The value of the block.
|
|
172
|
+
def open(selector, &block)
|
|
173
|
+
# The key is present but nil where an enclosing scope found nothing
|
|
174
|
+
# to bound, so that a nested call does not scan the selector again.
|
|
175
|
+
# Deciding before this check, in a default argument say, would walk
|
|
176
|
+
# the selector once per document on a scan.
|
|
177
|
+
return yield if Threaded.has?(Threaded::REGEXP_BUDGET_KEY)
|
|
178
|
+
|
|
179
|
+
open_with(limit_for(selector), &block)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Opens a budget scope for a limit the caller has already decided on.
|
|
183
|
+
#
|
|
184
|
+
# A caller that rearranges its work around the decision -- loading
|
|
185
|
+
# documents up front so that nothing is mutated before the scan
|
|
186
|
+
# finishes, say -- has to make it before it can act on it, and must not
|
|
187
|
+
# then make it a second time. Asking .limit_for and letting .open ask
|
|
188
|
+
# again reads the configured limit twice, and the two reads can differ:
|
|
189
|
+
# a limit that becomes positive in between would establish a budget in
|
|
190
|
+
# the branch that was chosen for not needing one, and on the Timeout
|
|
191
|
+
# path that arms a deadline over work the branch never made
|
|
192
|
+
# interruptible.
|
|
193
|
+
#
|
|
194
|
+
# See .open for what the scope does and does not bound, and for the
|
|
195
|
+
# note about mutating state inside it.
|
|
196
|
+
#
|
|
197
|
+
# @param [ Float | nil ] limit The seconds the scope may spend, or nil
|
|
198
|
+
# for a scope with nothing to bound.
|
|
199
|
+
#
|
|
200
|
+
# @return [ Object ] The value of the block.
|
|
201
|
+
def open_with(limit, &block)
|
|
202
|
+
return yield if Threaded.has?(Threaded::REGEXP_BUDGET_KEY)
|
|
203
|
+
|
|
204
|
+
budget = Budget.new(limit) if limit&.positive?
|
|
205
|
+
|
|
206
|
+
begin
|
|
207
|
+
# Set inside the begin so that an asynchronous exception from an
|
|
208
|
+
# enclosing timeout cannot leave the key behind on a pooled thread.
|
|
209
|
+
Threaded.set(Threaded::REGEXP_BUDGET_KEY, budget)
|
|
210
|
+
|
|
211
|
+
if budget.nil? || PER_REGEXP_TIMEOUT
|
|
212
|
+
yield
|
|
213
|
+
else
|
|
214
|
+
begin
|
|
215
|
+
Timeout.timeout(budget.limit, TimedOut, &block)
|
|
216
|
+
rescue TimedOut
|
|
217
|
+
raise timeout_error(budget)
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
ensure
|
|
221
|
+
Threaded.delete(Threaded::REGEXP_BUDGET_KEY)
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# The limit a scope evaluating this selector would be bounded by.
|
|
226
|
+
#
|
|
227
|
+
# A caller that has to rearrange its work to make the scan
|
|
228
|
+
# interruptible -- loading documents up front so that nothing is
|
|
229
|
+
# mutated before the scan finishes, say -- can ask this first and skip
|
|
230
|
+
# the rearrangement, and whatever it costs, when there is no pattern to
|
|
231
|
+
# bound. It then passes what it got to .open_with, so that the decision
|
|
232
|
+
# it acted on is the one the scope is opened with. Callers with nothing
|
|
233
|
+
# to rearrange should just call .open, which asks this itself.
|
|
234
|
+
#
|
|
235
|
+
# @param [ Hash ] selector The selector about to be evaluated.
|
|
236
|
+
#
|
|
237
|
+
# @return [ Float | nil ] The limit, or nil where there is nothing to
|
|
238
|
+
# bound.
|
|
239
|
+
def limit_for(selector)
|
|
240
|
+
# nil.to_f is 0.0, so an unset limit and a limit of zero or less are
|
|
241
|
+
# the same thing here: no limit. Zero is a common way to spell
|
|
242
|
+
# "disabled", and taking it literally would mean a budget that is
|
|
243
|
+
# spent before the first match and a query that can never run.
|
|
244
|
+
limit = Mongoid::Config.in_memory_regexp_time_limit.to_f
|
|
245
|
+
return nil unless limit.positive?
|
|
246
|
+
|
|
247
|
+
limit if contains_regexp?(selector)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Matches a value against a regular expression condition, charging the
|
|
251
|
+
# time it takes against the open budget.
|
|
252
|
+
#
|
|
253
|
+
# @param [ Object ] value The value to match.
|
|
254
|
+
# @param [ Regexp | BSON::Regexp::Raw ] condition The condition.
|
|
255
|
+
#
|
|
256
|
+
# @raise [ Errors::InMemoryRegexpTimeout ] if the budget is exhausted.
|
|
257
|
+
#
|
|
258
|
+
# @return [ Integer | nil ] The offset of the match, or nil.
|
|
259
|
+
def match?(value, condition)
|
|
260
|
+
budget = current
|
|
261
|
+
return value =~ coerce(condition) unless budget
|
|
262
|
+
|
|
263
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
264
|
+
pattern = nil
|
|
265
|
+
begin
|
|
266
|
+
raise timeout_error(budget) if budget.exhausted?
|
|
267
|
+
|
|
268
|
+
# Compiling is charged too. It is not free, and for a pattern with
|
|
269
|
+
# very many branches it costs far more than running the pattern
|
|
270
|
+
# does, so leaving it out would leave a way to spend unbounded time
|
|
271
|
+
# without the budget ever noticing.
|
|
272
|
+
pattern = budget.compile(condition)
|
|
273
|
+
value =~ pattern
|
|
274
|
+
rescue TIMEOUT_ERROR
|
|
275
|
+
# Name the limit that actually fired, which is not always the
|
|
276
|
+
# budget's. A baked pattern carries it: the smaller of the budget's
|
|
277
|
+
# limit and any global Regexp.timeout the application has set.
|
|
278
|
+
# Where nothing was baked -- an engine that raises this error but
|
|
279
|
+
# will not take a per-Regexp timeout, which is JRuby -- the global
|
|
280
|
+
# is the only thing that can have fired, and the pattern reports
|
|
281
|
+
# nil. Naming the budget's limit in either case would state a time
|
|
282
|
+
# that was never spent and send the reader after a setting that is
|
|
283
|
+
# not the one in the way.
|
|
284
|
+
#
|
|
285
|
+
# Both readers arrived together with Regexp::TimeoutError, so every
|
|
286
|
+
# engine that can reach this rescue at all has them.
|
|
287
|
+
raise timeout_error(budget, pattern&.timeout || ::Regexp.timeout || budget.limit)
|
|
288
|
+
ensure
|
|
289
|
+
budget.charge(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
# Runs the block without letting a scope timeout tear it in half.
|
|
294
|
+
#
|
|
295
|
+
# Where the budget is enforced with Timeout, the exception is raised
|
|
296
|
+
# asynchronously and can arrive at any point. Wrapping a mutation in
|
|
297
|
+
# this holds the exception back until the block has finished, so the
|
|
298
|
+
# interruption is deferred rather than given up.
|
|
299
|
+
#
|
|
300
|
+
# @return [ Object ] The value of the block.
|
|
301
|
+
def protect(&block)
|
|
302
|
+
Thread.handle_interrupt(TimedOut => :never, &block)
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# The time left in the open budget, or nil when no budget is open.
|
|
306
|
+
#
|
|
307
|
+
# @return [ Float | nil ] The remaining seconds.
|
|
308
|
+
def remaining
|
|
309
|
+
current&.remaining
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# Returns the condition as a Regexp, without a timeout.
|
|
313
|
+
#
|
|
314
|
+
# @param [ Regexp | BSON::Regexp::Raw ] condition The condition.
|
|
315
|
+
#
|
|
316
|
+
# @return [ Regexp ] The pattern.
|
|
317
|
+
def coerce(condition)
|
|
318
|
+
case condition
|
|
319
|
+
when ::Regexp then condition
|
|
320
|
+
when BSON::Regexp::Raw then condition.compile
|
|
321
|
+
else raise ArgumentError, "Not a regular expression: #{condition.inspect}"
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
private
|
|
326
|
+
|
|
327
|
+
# The budget for the open scope, if there is one.
|
|
328
|
+
#
|
|
329
|
+
# @return [ Budget | nil ] The open budget.
|
|
330
|
+
def current
|
|
331
|
+
Threaded.get(Threaded::REGEXP_BUDGET_KEY)
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# Whether evaluating the selector could run a regular expression.
|
|
335
|
+
#
|
|
336
|
+
# A string under $regex counts: FieldExpression turns it into a pattern
|
|
337
|
+
# at match time.
|
|
338
|
+
def contains_regexp?(object)
|
|
339
|
+
case object
|
|
340
|
+
when ::Regexp, BSON::Regexp::Raw
|
|
341
|
+
true
|
|
342
|
+
when Hash
|
|
343
|
+
object.any? do |k, v|
|
|
344
|
+
k.to_s == '$regex' || contains_regexp?(v)
|
|
345
|
+
end
|
|
346
|
+
when Array
|
|
347
|
+
object.any? { |v| contains_regexp?(v) }
|
|
348
|
+
else
|
|
349
|
+
false
|
|
350
|
+
end
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# Builds the error with the scope timeout held back.
|
|
354
|
+
#
|
|
355
|
+
# Composing the message goes through I18n, which reads locale files the
|
|
356
|
+
# first time it runs. An asynchronous TimedOut landing in the middle of
|
|
357
|
+
# that is caught by I18n and reraised as a locale-loading failure, so
|
|
358
|
+
# the real error never surfaces.
|
|
359
|
+
#
|
|
360
|
+
# @param [ Budget ] budget The open budget.
|
|
361
|
+
# @param [ Float ] limit The limit that was exceeded. Defaults to the
|
|
362
|
+
# budget's own, which is the right one to name everywhere the budget
|
|
363
|
+
# itself ran out.
|
|
364
|
+
def timeout_error(budget, limit = budget.limit)
|
|
365
|
+
protect do
|
|
366
|
+
Errors::InMemoryRegexpTimeout.new(limit)
|
|
367
|
+
end
|
|
368
|
+
end
|
|
369
|
+
end
|
|
370
|
+
end
|
|
371
|
+
end
|
|
372
|
+
end
|
data/lib/mongoid/matcher.rb
CHANGED
|
@@ -134,6 +134,7 @@ require 'mongoid/matcher/nor'
|
|
|
134
134
|
require 'mongoid/matcher/not'
|
|
135
135
|
require 'mongoid/matcher/or'
|
|
136
136
|
require 'mongoid/matcher/regex'
|
|
137
|
+
require 'mongoid/matcher/regexp_budget'
|
|
137
138
|
require 'mongoid/matcher/size'
|
|
138
139
|
require 'mongoid/matcher/type'
|
|
139
140
|
require 'mongoid/matcher/expression_operator'
|
|
@@ -65,7 +65,7 @@ module Mongoid
|
|
|
65
65
|
if _touchable_parent?
|
|
66
66
|
touches = _parent._gather_touch_updates(Time.current)
|
|
67
67
|
if touches.present?
|
|
68
|
-
operations['$set'] = touches
|
|
68
|
+
operations['$set'] = (operations['$set'] || {}).merge(touches)
|
|
69
69
|
Threaded.begin_touch_merged(self)
|
|
70
70
|
end
|
|
71
71
|
end
|
|
@@ -103,7 +103,7 @@ module Mongoid
|
|
|
103
103
|
|
|
104
104
|
process_flagged_destroys
|
|
105
105
|
update_children = cascadable_children(:update)
|
|
106
|
-
process_touch_option(options
|
|
106
|
+
process_touch_option(options) do
|
|
107
107
|
run_all_callbacks_for_update(update_children) do
|
|
108
108
|
result = yield(self)
|
|
109
109
|
self.previously_new_record = false
|
|
@@ -160,24 +160,22 @@ module Mongoid
|
|
|
160
160
|
end
|
|
161
161
|
end
|
|
162
162
|
|
|
163
|
-
# If there is a touch option and it is false, this method
|
|
164
|
-
#
|
|
165
|
-
#
|
|
166
|
-
#
|
|
163
|
+
# If there is a touch option and it is false, this method suppresses
|
|
164
|
+
# timestamping for the duration of the update using a block-based
|
|
165
|
+
# timeless scope, which covers this document and every cascaded child
|
|
166
|
+
# (at any nesting depth), and also suppresses touch callbacks.
|
|
167
167
|
#
|
|
168
168
|
# @param [ Hash ] options The options.
|
|
169
|
-
# @param [ Array<Document> ] children The children that the :update
|
|
170
|
-
# callbacks will be executed on.
|
|
171
169
|
#
|
|
172
170
|
# @option options [ true | false ] :touch Whether or not the updated_at
|
|
173
171
|
# attribute will be updated with the current time.
|
|
174
|
-
def process_touch_option(options,
|
|
172
|
+
def process_touch_option(options, &block)
|
|
175
173
|
if options.fetch(:touch, true)
|
|
176
174
|
yield
|
|
177
175
|
else
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
176
|
+
Mongoid::Timestamps::Timeless.with_timeless do
|
|
177
|
+
suppress_touch_callbacks(&block)
|
|
178
|
+
end
|
|
181
179
|
end
|
|
182
180
|
end
|
|
183
181
|
|
|
@@ -125,6 +125,8 @@ module Mongoid
|
|
|
125
125
|
|
|
126
126
|
client = client.with(options) unless options.empty?
|
|
127
127
|
|
|
128
|
+
verify_encryption_schema!(client)
|
|
129
|
+
|
|
128
130
|
client
|
|
129
131
|
end
|
|
130
132
|
end
|
|
@@ -199,6 +201,39 @@ module Mongoid
|
|
|
199
201
|
name.respond_to?(:call) ? name.call.to_sym : name.to_sym
|
|
200
202
|
end
|
|
201
203
|
|
|
204
|
+
# Refuse to use a client that would store a model's encrypted fields in
|
|
205
|
+
# plaintext.
|
|
206
|
+
#
|
|
207
|
+
# The automatic encryption schema is keyed by namespace and is built once,
|
|
208
|
+
# when the client is created. The driver looks the target namespace up in
|
|
209
|
+
# that schema, and when it is absent it asks the server for a schema
|
|
210
|
+
# instead; a collection without a validator then yields no encryption at
|
|
211
|
+
# all, and no error. Checking the namespace here is what turns that silent
|
|
212
|
+
# downgrade into a failure.
|
|
213
|
+
#
|
|
214
|
+
# @param [ Mongo::Client ] client The client this context resolved to.
|
|
215
|
+
#
|
|
216
|
+
# @raise [ Errors::NoEncryptionSchema ] if the model needs an encryption
|
|
217
|
+
# schema and the client's schema does not cover the target namespace.
|
|
218
|
+
def verify_encryption_schema!(client)
|
|
219
|
+
klass = @object.is_a?(Class) ? @object : @object.class
|
|
220
|
+
# A model whose encrypted fields all live on embedded models declares no
|
|
221
|
+
# encryption of its own, and still needs a schema for its collection.
|
|
222
|
+
return unless klass.respond_to?(:requires_encryption_schema?) && klass.requires_encryption_schema?
|
|
223
|
+
# An embedded document is persisted as part of its parent, so the
|
|
224
|
+
# parent's context is what governs the namespace.
|
|
225
|
+
return if klass.embedded?
|
|
226
|
+
# Collection and index management sends no document data, so it does not
|
|
227
|
+
# need an encryption-capable client.
|
|
228
|
+
return if Threaded.managing_collection?
|
|
229
|
+
|
|
230
|
+
schema_map = client.options.dig(:auto_encryption_options, :schema_map)
|
|
231
|
+
namespace = "#{client.database.name}.#{collection_name}"
|
|
232
|
+
return if schema_map&.key?(namespace)
|
|
233
|
+
|
|
234
|
+
raise Errors::NoEncryptionSchema.new(klass, namespace, client_name)
|
|
235
|
+
end
|
|
236
|
+
|
|
202
237
|
def client_options
|
|
203
238
|
@client_options ||= begin
|
|
204
239
|
opts = options.select do |k, _v|
|
|
@@ -69,6 +69,9 @@ module Mongoid
|
|
|
69
69
|
# @param [ Integer ] limit The maximum number of results (default: 10).
|
|
70
70
|
# @param [ Integer | nil ] num_candidates The number of candidates to
|
|
71
71
|
# consider during the ANN search; defaults to limit * 10.
|
|
72
|
+
# @param [ true | false ] exact Use exact nearest-neighbor (ENN) search
|
|
73
|
+
# instead of ANN (default: false). When true, numCandidates is omitted.
|
|
74
|
+
# Required when using a flat vector search index.
|
|
72
75
|
# @param [ Hash | nil ] filter An optional MongoDB filter to pre-filter
|
|
73
76
|
# candidates before scoring.
|
|
74
77
|
# @param [ Array ] pipeline Additional aggregation stages to append after
|
|
@@ -76,7 +79,7 @@ module Mongoid
|
|
|
76
79
|
#
|
|
77
80
|
# @return [ Array<Mongoid::Document> ] matching documents, each with
|
|
78
81
|
# a populated +vector_search_score+ attribute.
|
|
79
|
-
def vector_search(index: nil, path: nil, limit: 10, num_candidates: nil, filter: nil, pipeline: [])
|
|
82
|
+
def vector_search(index: nil, path: nil, limit: 10, num_candidates: nil, exact: false, filter: nil, pipeline: []) # rubocop:disable Metrics/ParameterLists
|
|
80
83
|
_index, resolved_path = self.class.send(:resolve_vector_index, index, path)
|
|
81
84
|
query_vector = public_send(resolved_path)
|
|
82
85
|
|
|
@@ -85,17 +88,19 @@ module Mongoid
|
|
|
85
88
|
"#{resolved_path} is nil on this document; cannot perform vector search"
|
|
86
89
|
end
|
|
87
90
|
|
|
88
|
-
|
|
89
|
-
|
|
91
|
+
self_exclusion = { '$match' => { '_id' => { '$ne' => _id } } }
|
|
92
|
+
post_pipeline = [ self_exclusion, { '$limit' => limit }, *Array(pipeline) ]
|
|
93
|
+
effective_candidates = num_candidates || (limit * 10)
|
|
90
94
|
|
|
91
95
|
self.class.vector_search(
|
|
92
96
|
query_vector,
|
|
93
97
|
index: index,
|
|
94
98
|
path: path,
|
|
95
|
-
limit: limit,
|
|
96
|
-
num_candidates:
|
|
97
|
-
|
|
98
|
-
|
|
99
|
+
limit: limit + 1,
|
|
100
|
+
num_candidates: effective_candidates,
|
|
101
|
+
exact: exact,
|
|
102
|
+
filter: filter,
|
|
103
|
+
pipeline: post_pipeline
|
|
99
104
|
)
|
|
100
105
|
end
|
|
101
106
|
|
|
@@ -129,19 +134,20 @@ module Mongoid
|
|
|
129
134
|
"#{resolved_path} is nil on this document; cannot perform auto-embed search"
|
|
130
135
|
end
|
|
131
136
|
|
|
132
|
-
|
|
133
|
-
|
|
137
|
+
self_exclusion = { '$match' => { '_id' => { '$ne' => _id } } }
|
|
138
|
+
post_pipeline = [ self_exclusion, { '$limit' => limit }, *Array(pipeline) ]
|
|
139
|
+
effective_candidates = num_candidates || (limit * 10)
|
|
134
140
|
|
|
135
141
|
self.class.auto_embed_search(
|
|
136
142
|
text,
|
|
137
143
|
index: index,
|
|
138
144
|
path: path,
|
|
139
|
-
limit: limit,
|
|
140
|
-
num_candidates:
|
|
141
|
-
filter:
|
|
145
|
+
limit: limit + 1,
|
|
146
|
+
num_candidates: effective_candidates,
|
|
147
|
+
filter: filter,
|
|
142
148
|
exact: exact,
|
|
143
149
|
model: model,
|
|
144
|
-
pipeline:
|
|
150
|
+
pipeline: post_pipeline
|
|
145
151
|
)
|
|
146
152
|
end
|
|
147
153
|
|
|
@@ -155,7 +161,9 @@ module Mongoid
|
|
|
155
161
|
def create_search_indexes
|
|
156
162
|
return if search_index_specs.empty?
|
|
157
163
|
|
|
158
|
-
|
|
164
|
+
Threaded.with_collection_management do
|
|
165
|
+
collection.search_indexes.create_many(search_index_specs)
|
|
166
|
+
end
|
|
159
167
|
end
|
|
160
168
|
|
|
161
169
|
# Waits for the named search indexes to be created.
|
|
@@ -186,7 +194,9 @@ module Mongoid
|
|
|
186
194
|
# @option options [ Hash ] :aggregate The options hash to pass to the
|
|
187
195
|
# aggregate command (optional)
|
|
188
196
|
def search_indexes(options = {})
|
|
189
|
-
|
|
197
|
+
Threaded.with_collection_management do
|
|
198
|
+
collection.search_indexes(options)
|
|
199
|
+
end
|
|
190
200
|
end
|
|
191
201
|
|
|
192
202
|
# Removes the search index specified by the given name or id. Either
|
|
@@ -195,12 +205,14 @@ module Mongoid
|
|
|
195
205
|
# @param [ String | nil ] name the name of the index to remove
|
|
196
206
|
# @param [ String | nil ] id the id of the index to remove
|
|
197
207
|
def remove_search_index(name: nil, id: nil)
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
208
|
+
Threaded.with_collection_management do
|
|
209
|
+
logger.info(
|
|
210
|
+
"MONGOID: Removing search index '#{name || id}' " \
|
|
211
|
+
"on collection '#{collection.name}'."
|
|
212
|
+
)
|
|
202
213
|
|
|
203
|
-
|
|
214
|
+
collection.search_indexes.drop_one(name: name, id: id)
|
|
215
|
+
end
|
|
204
216
|
end
|
|
205
217
|
|
|
206
218
|
# Request the removal of all registered search indexes. Note
|
|
@@ -253,6 +265,15 @@ module Mongoid
|
|
|
253
265
|
# vector_search_index :my_vector_index, { fields: [...] }
|
|
254
266
|
# end
|
|
255
267
|
#
|
|
268
|
+
# @example Create a flat vector search index.
|
|
269
|
+
# class Person
|
|
270
|
+
# include Mongoid::Document
|
|
271
|
+
# vector_search_index fields: [
|
|
272
|
+
# { type: 'vector', path: 'embedding', numDimensions: 1536,
|
|
273
|
+
# similarity: 'cosine', indexingMethod: 'flat' }
|
|
274
|
+
# ]
|
|
275
|
+
# end
|
|
276
|
+
#
|
|
256
277
|
# @param [ Symbol | String | Hash ] name_or_defn Either the name of the index to
|
|
257
278
|
# define, or the index definition.
|
|
258
279
|
# @param [ Hash ] defn The vector search index definition.
|
|
@@ -260,6 +281,8 @@ module Mongoid
|
|
|
260
281
|
name = name_or_defn
|
|
261
282
|
name, defn = nil, name if name.is_a?(Hash)
|
|
262
283
|
|
|
284
|
+
validate_vector_index_definition!(defn)
|
|
285
|
+
|
|
263
286
|
spec = { type: 'vectorSearch', definition: defn }.tap { |s| s[:name] = name.to_s if name }
|
|
264
287
|
search_index_specs.push(spec)
|
|
265
288
|
|
|
@@ -290,22 +313,25 @@ module Mongoid
|
|
|
290
313
|
# consider during the ANN search; defaults to limit * 10.
|
|
291
314
|
# @param [ Hash | nil ] filter An optional MongoDB filter to pre-filter
|
|
292
315
|
# candidates before scoring.
|
|
316
|
+
# @param [ true | false ] exact Use exact nearest-neighbor (ENN) search
|
|
317
|
+
# instead of ANN (default: false). When true, numCandidates is omitted.
|
|
318
|
+
# Required when using a flat vector search index.
|
|
293
319
|
# @param [ Array ] pipeline Additional aggregation stages to append after
|
|
294
320
|
# the vector search and score projection.
|
|
295
321
|
#
|
|
296
322
|
# @return [ Array<Mongoid::Document> ] matching documents, each with
|
|
297
323
|
# a populated +vector_search_score+ attribute.
|
|
298
|
-
def vector_search(vector, index: nil, path: nil, limit: 10, num_candidates: nil, filter: nil, pipeline: []) # rubocop:disable Metrics/ParameterLists
|
|
324
|
+
def vector_search(vector, index: nil, path: nil, limit: 10, num_candidates: nil, exact: false, filter: nil, pipeline: []) # rubocop:disable Metrics/ParameterLists
|
|
299
325
|
resolved_index, resolved_path = resolve_vector_index(index, path)
|
|
300
|
-
num_candidates ||= limit * 10
|
|
301
326
|
|
|
302
327
|
vs_options = {
|
|
303
328
|
'index' => resolved_index,
|
|
304
329
|
'path' => resolved_path,
|
|
305
330
|
'queryVector' => vector,
|
|
306
|
-
'numCandidates' => num_candidates,
|
|
307
331
|
'limit' => limit
|
|
308
332
|
}
|
|
333
|
+
vs_options['numCandidates'] = num_candidates || (limit * 10) unless exact
|
|
334
|
+
vs_options['exact'] = true if exact
|
|
309
335
|
vs_options['filter'] = filter if filter
|
|
310
336
|
|
|
311
337
|
agg_pipeline = [
|
|
@@ -374,6 +400,21 @@ module Mongoid
|
|
|
374
400
|
|
|
375
401
|
private
|
|
376
402
|
|
|
403
|
+
# Validates the vector index definition, raising ArgumentError for
|
|
404
|
+
# combinations that MongoDB does not support.
|
|
405
|
+
#
|
|
406
|
+
# @param [ Hash ] defn The vector search index definition.
|
|
407
|
+
def validate_vector_index_definition!(defn)
|
|
408
|
+
fields = defn[:fields] || defn['fields'] || []
|
|
409
|
+
fields.each do |field|
|
|
410
|
+
method = field[:indexingMethod] || field['indexingMethod']
|
|
411
|
+
next unless method.to_s == 'flat'
|
|
412
|
+
next unless field[:hnswOptions] || field['hnswOptions']
|
|
413
|
+
|
|
414
|
+
raise ArgumentError, 'hnswOptions is only supported with indexingMethod: hnsw'
|
|
415
|
+
end
|
|
416
|
+
end
|
|
417
|
+
|
|
377
418
|
# Retrieves the index records for the indexes with the given names.
|
|
378
419
|
#
|
|
379
420
|
# @param [ Array<String> ] names the index names to query
|