parse-stack-next 5.5.6 → 5.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +634 -0
- data/README.md +378 -6
- data/docs/caching.md +748 -0
- data/examples/basic_client.rb +3 -3
- data/examples/basic_server.rb +3 -3
- data/examples/live_query_listener.rb +12 -12
- data/examples/rag_chatbot.rb +14 -14
- data/examples/transaction_example.rb +44 -45
- data/examples/webhook_server.rb +3 -3
- data/lib/parse/access.rb +493 -0
- data/lib/parse/acl_scope.rb +125 -52
- data/lib/parse/agent/approval_gate.rb +0 -0
- data/lib/parse/agent/cancellation_token.rb +3 -3
- data/lib/parse/agent/constraint_translator.rb +4 -4
- data/lib/parse/agent/describe.rb +34 -34
- data/lib/parse/agent/errors.rb +9 -9
- data/lib/parse/agent/mcp_client.rb +61 -59
- data/lib/parse/agent/mcp_dispatcher.rb +89 -101
- data/lib/parse/agent/mcp_rack_app.rb +92 -93
- data/lib/parse/agent/mcp_server.rb +1 -1
- data/lib/parse/agent/mcp_subscriptions.rb +22 -22
- data/lib/parse/agent/metadata_audit.rb +1 -2
- data/lib/parse/agent/metadata_dsl.rb +17 -7
- data/lib/parse/agent/metadata_registry.rb +12 -12
- data/lib/parse/agent/prompt_hardening.rb +4 -4
- data/lib/parse/agent/prompts.rb +10 -10
- data/lib/parse/agent/result_formatter.rb +4 -4
- data/lib/parse/agent/tools.rb +493 -400
- data/lib/parse/agent.rb +157 -116
- data/lib/parse/api/hooks.rb +1 -1
- data/lib/parse/api/server.rb +2 -2
- data/lib/parse/api/users.rb +2 -2
- data/lib/parse/atlas_search/index_manager.rb +1 -1
- data/lib/parse/atlas_search/session.rb +40 -218
- data/lib/parse/atlas_search.rb +181 -60
- data/lib/parse/authorization.rb +466 -0
- data/lib/parse/cache/invalidation.rb +167 -0
- data/lib/parse/cache/keyspace.rb +306 -0
- data/lib/parse/cache/moneta_surface.rb +126 -0
- data/lib/parse/cache/pool.rb +48 -5
- data/lib/parse/cache/redis.rb +425 -10
- data/lib/parse/cache/scoped_view.rb +443 -0
- data/lib/parse/cache/sub_cache.rb +264 -0
- data/lib/parse/cache/upstream_roles.rb +230 -0
- data/lib/parse/client/authentication.rb +1 -1
- data/lib/parse/client/body_builder.rb +1 -2
- data/lib/parse/client/caching.rb +84 -5
- data/lib/parse/client.rb +327 -189
- data/lib/parse/clp_scope.rb +225 -28
- data/lib/parse/console.rb +3 -3
- data/lib/parse/embeddings/batch_embedder.rb +3 -3
- data/lib/parse/embeddings/binding_audit.rb +211 -0
- data/lib/parse/embeddings/cache.rb +17 -17
- data/lib/parse/embeddings/cohere.rb +21 -22
- data/lib/parse/embeddings/image_fetch.rb +22 -21
- data/lib/parse/embeddings/jina.rb +21 -21
- data/lib/parse/embeddings/local_http.rb +6 -7
- data/lib/parse/embeddings/media_file.rb +137 -0
- data/lib/parse/embeddings/openai.rb +5 -5
- data/lib/parse/embeddings/provider.rb +40 -1
- data/lib/parse/embeddings/qwen.rb +10 -10
- data/lib/parse/embeddings/spend_cap.rb +2 -3
- data/lib/parse/embeddings/streaming_body.rb +170 -0
- data/lib/parse/embeddings/video_source.rb +121 -0
- data/lib/parse/embeddings/voyage.rb +554 -143
- data/lib/parse/embeddings.rb +83 -27
- data/lib/parse/graphql/type_generator.rb +1 -1
- data/lib/parse/graphql.rb +5 -5
- data/lib/parse/live_query/client.rb +1 -1
- data/lib/parse/live_query.rb +1 -1
- data/lib/parse/lock.rb +15 -18
- data/lib/parse/lock_backend.rb +1 -1
- data/lib/parse/lookup_rewriter.rb +6 -6
- data/lib/parse/model/classes/role.rb +227 -28
- data/lib/parse/model/classes/user.rb +60 -1
- data/lib/parse/model/clp.rb +4 -4
- data/lib/parse/model/core/actions.rb +97 -15
- data/lib/parse/model/core/create_lock.rb +0 -2
- data/lib/parse/model/core/describe.rb +57 -57
- data/lib/parse/model/core/embed_managed.rb +43 -19
- data/lib/parse/model/core/fetching.rb +2 -0
- data/lib/parse/model/core/indexing.rb +14 -14
- data/lib/parse/model/core/parse_reference.rb +1 -1
- data/lib/parse/model/core/properties.rb +39 -3
- data/lib/parse/model/core/querying.rb +1 -1
- data/lib/parse/model/core/schema.rb +2 -2
- data/lib/parse/model/core/search_indexing.rb +2 -2
- data/lib/parse/model/core/vector_searchable.rb +34 -8
- data/lib/parse/model/file.rb +21 -15
- data/lib/parse/model/geojson.rb +2 -2
- data/lib/parse/model/geopoint.rb +1 -0
- data/lib/parse/model/object.rb +132 -66
- data/lib/parse/model/pointer.rb +2 -0
- data/lib/parse/model/polygon.rb +3 -6
- data/lib/parse/model/push.rb +2 -2
- data/lib/parse/model/vector.rb +3 -1
- data/lib/parse/mongodb.rb +512 -282
- data/lib/parse/pipeline_security.rb +3 -0
- data/lib/parse/query/constraints.rb +46 -46
- data/lib/parse/query.rb +146 -59
- data/lib/parse/retrieval/agent_tool.rb +34 -20
- data/lib/parse/retrieval/chunk.rb +1 -0
- data/lib/parse/retrieval/reranker/cohere.rb +11 -11
- data/lib/parse/retrieval/reranker.rb +3 -4
- data/lib/parse/retrieval/retriever.rb +4 -6
- data/lib/parse/schema/index_migrator.rb +26 -26
- data/lib/parse/schema/search_index_migrator.rb +19 -19
- data/lib/parse/stack/tasks.rb +6 -6
- data/lib/parse/stack/version.rb +1 -1
- data/lib/parse/stack.rb +10 -10
- data/lib/parse/vector_search/hybrid.rb +166 -33
- data/lib/parse/vector_search.rb +141 -13
- data/lib/parse/webhooks/payload.rb +1 -0
- data/lib/parse/webhooks/registration.rb +4 -4
- data/lib/parse/webhooks/trigger_audit.rb +50 -49
- data/lib/parse/webhooks.rb +22 -4
- data/parse-stack-next.gemspec +6 -6
- metadata +14 -1
|
@@ -224,8 +224,7 @@ module Parse
|
|
|
224
224
|
chunker ||= default_chunker
|
|
225
225
|
text_wire = wire_name(klass, resolved_text_field)
|
|
226
226
|
|
|
227
|
-
raw_hits =
|
|
228
|
-
if hybrid
|
|
227
|
+
raw_hits = if hybrid
|
|
229
228
|
fetch_hybrid_hits(klass, query, k, field, filter, merged_vector_filter,
|
|
230
229
|
tenant_scope, hybrid, scope_opts)
|
|
231
230
|
else
|
|
@@ -253,8 +252,8 @@ module Parse
|
|
|
253
252
|
tenant_scope, hybrid, scope_opts)
|
|
254
253
|
cfg = hybrid.is_a?(Hash) ? hybrid : {}
|
|
255
254
|
lexical = (cfg[:lexical] || cfg["lexical"] || {}).dup
|
|
256
|
-
vector
|
|
257
|
-
fusion
|
|
255
|
+
vector = (cfg[:vector] || cfg["vector"] || {}).dup
|
|
256
|
+
fusion = cfg[:fusion] || cfg["fusion"]
|
|
258
257
|
|
|
259
258
|
lexical[:query] ||= query
|
|
260
259
|
# Tenant scope must be AUTHORITATIVE in BOTH branches. The previous
|
|
@@ -278,8 +277,7 @@ module Parse
|
|
|
278
277
|
|
|
279
278
|
# @!visibility private
|
|
280
279
|
def resolve_class!(klass)
|
|
281
|
-
resolved =
|
|
282
|
-
case klass
|
|
280
|
+
resolved = case klass
|
|
283
281
|
when nil
|
|
284
282
|
nil
|
|
285
283
|
when Class
|
|
@@ -110,7 +110,7 @@ module Parse
|
|
|
110
110
|
managed, ours = partition_parse_managed(existing)
|
|
111
111
|
to_create, in_sync, conflicts = diff_declarations(declared, ours)
|
|
112
112
|
declared_names = declared.map { |d| d[:options][:name] }.compact.to_set
|
|
113
|
-
declared_sigs
|
|
113
|
+
declared_sigs = declared.map { |d| key_sig(d[:keys]) }.to_set
|
|
114
114
|
orphans = ours.reject do |idx|
|
|
115
115
|
declared_sigs.include?(key_sig(idx["key"] || idx[:key])) ||
|
|
116
116
|
declared_names.include?(idx["name"] || idx[:name])
|
|
@@ -122,21 +122,21 @@ module Parse
|
|
|
122
122
|
after_with_drop = after_no_drop - orphans.size
|
|
123
123
|
|
|
124
124
|
{
|
|
125
|
-
collection:
|
|
126
|
-
declared:
|
|
127
|
-
existing:
|
|
128
|
-
parse_managed:
|
|
129
|
-
to_create:
|
|
130
|
-
in_sync:
|
|
131
|
-
conflicts:
|
|
132
|
-
orphans:
|
|
133
|
-
capacity_used:
|
|
134
|
-
capacity_after:
|
|
135
|
-
capacity_remaining:
|
|
136
|
-
capacity_ok:
|
|
125
|
+
collection: collection,
|
|
126
|
+
declared: declared,
|
|
127
|
+
existing: existing,
|
|
128
|
+
parse_managed: managed.map { |i| i["name"] || i[:name] },
|
|
129
|
+
to_create: to_create,
|
|
130
|
+
in_sync: in_sync,
|
|
131
|
+
conflicts: conflicts,
|
|
132
|
+
orphans: orphans.map { |i| i["name"] || i[:name] }.compact,
|
|
133
|
+
capacity_used: used,
|
|
134
|
+
capacity_after: after_no_drop,
|
|
135
|
+
capacity_remaining: max - after_no_drop,
|
|
136
|
+
capacity_ok: after_no_drop <= max,
|
|
137
137
|
capacity_after_with_drop: after_with_drop,
|
|
138
138
|
capacity_remaining_with_drop: max - after_with_drop,
|
|
139
|
-
capacity_ok_with_drop:
|
|
139
|
+
capacity_ok_with_drop: after_with_drop <= max,
|
|
140
140
|
}
|
|
141
141
|
end
|
|
142
142
|
|
|
@@ -181,7 +181,7 @@ module Parse
|
|
|
181
181
|
p[:orphans].each do |name|
|
|
182
182
|
confirm = "drop:#{collection}:#{name}"
|
|
183
183
|
res = Parse::MongoDB.drop_index(collection, name, confirm: confirm,
|
|
184
|
-
|
|
184
|
+
allow_system_classes: collection.start_with?("_Join:"))
|
|
185
185
|
dropped << name if res == :dropped
|
|
186
186
|
end
|
|
187
187
|
end
|
|
@@ -190,21 +190,21 @@ module Parse
|
|
|
190
190
|
result = Parse::MongoDB.create_index(
|
|
191
191
|
collection,
|
|
192
192
|
decl[:keys],
|
|
193
|
-
name:
|
|
194
|
-
unique:
|
|
195
|
-
sparse:
|
|
196
|
-
partial_filter:
|
|
197
|
-
expire_after:
|
|
193
|
+
name: decl[:options][:name],
|
|
194
|
+
unique: decl[:options][:unique] == true,
|
|
195
|
+
sparse: decl[:options][:sparse] == true,
|
|
196
|
+
partial_filter: decl[:options][:partial_filter],
|
|
197
|
+
expire_after: decl[:options][:expire_after],
|
|
198
198
|
allow_system_classes: collection.start_with?("_Join:"),
|
|
199
199
|
)
|
|
200
200
|
(result == :exists ? skipped : created) << decl
|
|
201
201
|
end
|
|
202
202
|
|
|
203
203
|
{
|
|
204
|
-
created:
|
|
204
|
+
created: created,
|
|
205
205
|
skipped_exists: skipped,
|
|
206
|
-
dropped:
|
|
207
|
-
conflicts:
|
|
206
|
+
dropped: dropped,
|
|
207
|
+
conflicts: p[:conflicts],
|
|
208
208
|
capacity_blocked: false,
|
|
209
209
|
}
|
|
210
210
|
end
|
|
@@ -235,12 +235,12 @@ module Parse
|
|
|
235
235
|
|
|
236
236
|
def diff_declarations(declared, existing_ours)
|
|
237
237
|
to_create = []
|
|
238
|
-
in_sync
|
|
238
|
+
in_sync = []
|
|
239
239
|
conflicts = []
|
|
240
240
|
|
|
241
241
|
declared.each do |decl|
|
|
242
242
|
decl_sig = key_sig(decl[:keys])
|
|
243
|
-
named
|
|
243
|
+
named = decl[:options][:name]
|
|
244
244
|
|
|
245
245
|
# Prefer a name match when the declaration named one — that's
|
|
246
246
|
# the operator's authoritative target. Otherwise match by key
|
|
@@ -280,7 +280,7 @@ module Parse
|
|
|
280
280
|
def serialize_existing(idx)
|
|
281
281
|
{
|
|
282
282
|
name: idx["name"] || idx[:name],
|
|
283
|
-
key:
|
|
283
|
+
key: idx["key"] || idx[:key],
|
|
284
284
|
unique: idx["unique"] == true,
|
|
285
285
|
sparse: idx["sparse"] == true,
|
|
286
286
|
partial_filter: idx["partialFilterExpression"],
|
|
@@ -75,8 +75,8 @@ module Parse
|
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
to_create = []
|
|
78
|
-
in_sync
|
|
79
|
-
drifted
|
|
78
|
+
in_sync = []
|
|
79
|
+
drifted = []
|
|
80
80
|
|
|
81
81
|
declared.each do |decl|
|
|
82
82
|
target = existing_by_name[decl[:name]]
|
|
@@ -93,14 +93,14 @@ module Parse
|
|
|
93
93
|
orphans = existing_by_name.keys.reject { |name| declared_names.include?(name) }
|
|
94
94
|
|
|
95
95
|
{
|
|
96
|
-
collection:
|
|
97
|
-
declared:
|
|
98
|
-
existing:
|
|
96
|
+
collection: coll,
|
|
97
|
+
declared: declared,
|
|
98
|
+
existing: existing,
|
|
99
99
|
atlas_available: available,
|
|
100
|
-
to_create:
|
|
101
|
-
in_sync:
|
|
102
|
-
drifted:
|
|
103
|
-
orphans:
|
|
100
|
+
to_create: to_create,
|
|
101
|
+
in_sync: in_sync,
|
|
102
|
+
drifted: drifted,
|
|
103
|
+
orphans: orphans,
|
|
104
104
|
}
|
|
105
105
|
end
|
|
106
106
|
|
|
@@ -176,14 +176,14 @@ module Parse
|
|
|
176
176
|
end
|
|
177
177
|
|
|
178
178
|
{
|
|
179
|
-
created:
|
|
180
|
-
skipped_exists:
|
|
181
|
-
in_sync:
|
|
182
|
-
updated:
|
|
179
|
+
created: created,
|
|
180
|
+
skipped_exists: skipped_exists,
|
|
181
|
+
in_sync: p[:in_sync],
|
|
182
|
+
updated: updated,
|
|
183
183
|
drifted_skipped: drifted_skipped,
|
|
184
|
-
dropped:
|
|
184
|
+
dropped: dropped,
|
|
185
185
|
orphans_skipped: orphans_skipped,
|
|
186
|
-
wait_results:
|
|
186
|
+
wait_results: wait_results,
|
|
187
187
|
}
|
|
188
188
|
end
|
|
189
189
|
|
|
@@ -321,10 +321,10 @@ module Parse
|
|
|
321
321
|
# history, statusDetail) so operator-facing output stays readable.
|
|
322
322
|
def serialize_existing(idx)
|
|
323
323
|
{
|
|
324
|
-
name:
|
|
325
|
-
status:
|
|
326
|
-
queryable:
|
|
327
|
-
latest_definition:
|
|
324
|
+
name: (idx["name"] || idx[:name]).to_s,
|
|
325
|
+
status: (idx["status"] || idx[:status]).to_s,
|
|
326
|
+
queryable: idx["queryable"] == true,
|
|
327
|
+
latest_definition: idx["latestDefinition"] || idx[:latestDefinition],
|
|
328
328
|
}
|
|
329
329
|
end
|
|
330
330
|
|
data/lib/parse/stack/tasks.rb
CHANGED
|
@@ -110,7 +110,7 @@ module Parse
|
|
|
110
110
|
|
|
111
111
|
batch_size = (ENV["BATCH_SIZE"] || "100").to_i
|
|
112
112
|
batch_size = 100 if batch_size <= 0
|
|
113
|
-
dry_run
|
|
113
|
+
dry_run = ENV["DRY_RUN"].to_s.downcase == "true"
|
|
114
114
|
|
|
115
115
|
if dry_run
|
|
116
116
|
puts "[parse:references:populate] DRY_RUN=true — no writes will be issued"
|
|
@@ -120,7 +120,7 @@ module Parse
|
|
|
120
120
|
fields = Array(klass._parse_reference_fields)
|
|
121
121
|
fields.each do |field_name|
|
|
122
122
|
populated_total = 0
|
|
123
|
-
scanned_total
|
|
123
|
+
scanned_total = 0
|
|
124
124
|
loops_without_progress = 0
|
|
125
125
|
loop do
|
|
126
126
|
# Query for records where the reference column is null/
|
|
@@ -207,7 +207,7 @@ module Parse
|
|
|
207
207
|
puts " to_create:"
|
|
208
208
|
p[:to_create].each do |d|
|
|
209
209
|
flags = d[:options].dup
|
|
210
|
-
name
|
|
210
|
+
name = flags.delete(:name) || "(auto)"
|
|
211
211
|
puts " + #{d[:keys].inspect} name=#{name} opts=#{flags.inspect}"
|
|
212
212
|
end
|
|
213
213
|
end
|
|
@@ -423,13 +423,13 @@ module Parse
|
|
|
423
423
|
end
|
|
424
424
|
|
|
425
425
|
update = ENV["UPDATE"].to_s.downcase == "true"
|
|
426
|
-
drop
|
|
427
|
-
wait
|
|
426
|
+
drop = ENV["DROP"].to_s.downcase == "true"
|
|
427
|
+
wait = ENV["WAIT"].to_s.downcase == "true"
|
|
428
428
|
timeout = (ENV["WAIT_TIMEOUT"] || "600").to_i
|
|
429
429
|
modes = []
|
|
430
430
|
modes << "additive"
|
|
431
431
|
modes << "update-drifted" if update
|
|
432
|
-
modes << "drop-orphans"
|
|
432
|
+
modes << "drop-orphans" if drop
|
|
433
433
|
modes << "wait-for-ready (#{timeout}s)" if wait
|
|
434
434
|
puts "[parse:mongo:search_indexes:apply] mode: #{modes.join(" + ")}"
|
|
435
435
|
if drop
|
data/lib/parse/stack/version.rb
CHANGED
data/lib/parse/stack.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "stack/version"
|
|
|
5
5
|
require_relative "client"
|
|
6
6
|
require_relative "query"
|
|
7
7
|
require_relative "model/object"
|
|
8
|
+
require_relative "access"
|
|
8
9
|
require_relative "webhooks"
|
|
9
10
|
require_relative "agent"
|
|
10
11
|
require_relative "two_factor_auth"
|
|
@@ -303,7 +304,7 @@ module Parse
|
|
|
303
304
|
"Parse.login: credentials rejected for #{username.inspect} (server returned no session)."
|
|
304
305
|
end
|
|
305
306
|
Fiber[SESSION_TOKEN_STATE_KEY] = user.session_token
|
|
306
|
-
Fiber[CURRENT_USER_STATE_KEY]
|
|
307
|
+
Fiber[CURRENT_USER_STATE_KEY] = user
|
|
307
308
|
user
|
|
308
309
|
end
|
|
309
310
|
|
|
@@ -323,7 +324,7 @@ module Parse
|
|
|
323
324
|
def self.logout(revoke: true)
|
|
324
325
|
token = Fiber[SESSION_TOKEN_STATE_KEY]
|
|
325
326
|
Fiber[SESSION_TOKEN_STATE_KEY] = nil
|
|
326
|
-
Fiber[CURRENT_USER_STATE_KEY]
|
|
327
|
+
Fiber[CURRENT_USER_STATE_KEY] = nil
|
|
327
328
|
if revoke && token.is_a?(String) && !token.empty?
|
|
328
329
|
begin
|
|
329
330
|
Parse::Client.client.logout(token)
|
|
@@ -346,7 +347,7 @@ module Parse
|
|
|
346
347
|
resolved = token.respond_to?(:session_token) ? token.session_token : token
|
|
347
348
|
resolved = resolved.to_s if resolved
|
|
348
349
|
Fiber[SESSION_TOKEN_STATE_KEY] = (resolved && !resolved.empty?) ? resolved : nil
|
|
349
|
-
Fiber[CURRENT_USER_STATE_KEY]
|
|
350
|
+
Fiber[CURRENT_USER_STATE_KEY] = nil
|
|
350
351
|
Fiber[SESSION_TOKEN_STATE_KEY]
|
|
351
352
|
end
|
|
352
353
|
|
|
@@ -721,17 +722,16 @@ module Parse
|
|
|
721
722
|
next if duration_ms < threshold
|
|
722
723
|
logger = respond_to?(:logger) ? Parse.logger : nil
|
|
723
724
|
next unless logger
|
|
724
|
-
detail =
|
|
725
|
-
|
|
726
|
-
"stages=#{payload[:stage_count]} types=#{Array(payload[:stage_types]).join(',')}"
|
|
725
|
+
detail = if name == "parse.mongodb.aggregate"
|
|
726
|
+
"stages=#{payload[:stage_count]} types=#{Array(payload[:stage_types]).join(",")}"
|
|
727
727
|
else
|
|
728
|
-
"filter=#{!!payload[:has_filter]} projection=#{Array(payload[:projection_keys]).join(
|
|
728
|
+
"filter=#{!!payload[:has_filter]} projection=#{Array(payload[:projection_keys]).join(",")}"
|
|
729
729
|
end
|
|
730
730
|
logger.warn(
|
|
731
731
|
"[Parse::MongoDB] SLOW #{name} #{duration_ms}ms " \
|
|
732
|
-
"collection=#{payload[:collection]} scope=#{payload[:scope] ||
|
|
733
|
-
"#{detail} result_count=#{payload[:result_count] ||
|
|
734
|
-
"max_time_ms=#{payload[:max_time_ms] ||
|
|
732
|
+
"collection=#{payload[:collection]} scope=#{payload[:scope] || "n/a"} " \
|
|
733
|
+
"#{detail} result_count=#{payload[:result_count] || "n/a"} " \
|
|
734
|
+
"max_time_ms=#{payload[:max_time_ms] || "n/a"}",
|
|
735
735
|
)
|
|
736
736
|
end
|
|
737
737
|
ActiveSupport::Notifications.subscribe("parse.mongodb.aggregate", &handler)
|
|
@@ -66,6 +66,22 @@ module Parse
|
|
|
66
66
|
# uses a comparable internal oversample.
|
|
67
67
|
DEFAULT_OVERSAMPLE_MULTIPLIER = 5
|
|
68
68
|
|
|
69
|
+
# Emitted once per {.search}, mirroring
|
|
70
|
+
# {Parse::VectorSearch::AS_NOTIFICATION_NAME} so hybrid attrition
|
|
71
|
+
# is observable through the same subscriber. Carries `method`
|
|
72
|
+
# (`:rrf_client` / `:rrf_native`), `branch_depth`,
|
|
73
|
+
# `candidate_window`, `post_filter_count`, `returned_count`, and
|
|
74
|
+
# `underfilled`.
|
|
75
|
+
#
|
|
76
|
+
# `branch_depth` is the rows each branch actually retained, and it
|
|
77
|
+
# legitimately DIFFERS between the two methods: the client path
|
|
78
|
+
# enforces ACL inside each branch, so it can retain the narrower
|
|
79
|
+
# fusion depth, while the native path enforces after
|
|
80
|
+
# `$rankFusion` and must therefore retain the full candidate
|
|
81
|
+
# window. Exact parity is not achievable while that ordering
|
|
82
|
+
# difference exists, so the number reported is the one that ran.
|
|
83
|
+
AS_NOTIFICATION_NAME = "parse.vector_search.hybrid"
|
|
84
|
+
|
|
69
85
|
# Hard ceiling on the fused result count, matching
|
|
70
86
|
# {Parse::VectorSearch::MAX_K}.
|
|
71
87
|
MAX_K = Parse::VectorSearch::MAX_K
|
|
@@ -120,11 +136,11 @@ module Parse
|
|
|
120
136
|
acc.values
|
|
121
137
|
.sort_by { |e| [-e[:score], row_id(e[:doc]).to_s, e[:seq]] }
|
|
122
138
|
.map do |e|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
139
|
+
row = e[:doc].dup
|
|
140
|
+
row["_hybrid_score"] = e[:score]
|
|
141
|
+
row["_hybrid_ranks"] = e[:ranks]
|
|
142
|
+
row
|
|
143
|
+
end
|
|
128
144
|
end
|
|
129
145
|
|
|
130
146
|
# Detect whether the cluster backing `collection` supports the
|
|
@@ -140,14 +156,21 @@ module Parse
|
|
|
140
156
|
# cached per collection for {PROBE_CACHE_TTL}.
|
|
141
157
|
#
|
|
142
158
|
# @param collection [String] Parse class / Mongo collection name.
|
|
159
|
+
# @param authorizing_client [Parse::Client, nil] client whose
|
|
160
|
+
# application binding must be checked before probing.
|
|
143
161
|
# @return [Boolean]
|
|
144
|
-
def rank_fusion_supported?(collection)
|
|
162
|
+
def rank_fusion_supported?(collection, authorizing_client: nil)
|
|
163
|
+
# Check on every call, including cache hits. Otherwise a verdict
|
|
164
|
+
# cached by application A could bypass the unidentified/mismatched
|
|
165
|
+
# caller guard when application B asks about the same collection.
|
|
166
|
+
Parse::MongoDB.verify_client!(authorizing_client)
|
|
167
|
+
|
|
145
168
|
key = collection.to_s
|
|
146
169
|
now = monotonic
|
|
147
170
|
cached = probe_cache_get(key, now)
|
|
148
171
|
return cached unless cached.nil?
|
|
149
172
|
|
|
150
|
-
supported = run_probe(key)
|
|
173
|
+
supported = run_probe(key, authorizing_client: authorizing_client)
|
|
151
174
|
probe_cache_put(key, supported, now)
|
|
152
175
|
supported
|
|
153
176
|
end
|
|
@@ -219,8 +242,15 @@ module Parse
|
|
|
219
242
|
"hybrid search: fusion[:method] must be :rrf, :rrf_client, or :rrf_native (got #{method.inspect})."
|
|
220
243
|
end
|
|
221
244
|
k_constant = fusion[:k_constant] || DEFAULT_K_CONSTANT
|
|
222
|
-
weights
|
|
223
|
-
|
|
245
|
+
weights = fusion[:weights]
|
|
246
|
+
# Two distinct numbers, deliberately not one. `fusion_depth` is
|
|
247
|
+
# how many rows each branch RETAINS for RRF (bounded by
|
|
248
|
+
# VectorSearch::MAX_K, since it becomes a branch `k`);
|
|
249
|
+
# `candidate_window` is how many rows Atlas CONSIDERS before
|
|
250
|
+
# ACL enforcement (bounded by 10_000). Collapsing them lets
|
|
251
|
+
# the window be multiplied twice — once here and again inside
|
|
252
|
+
# VectorSearch.search — and lets a branch `k` exceed MAX_K.
|
|
253
|
+
fusion_depth, candidate_window = resolve_windows(k_int, vec[:candidate_limit])
|
|
224
254
|
|
|
225
255
|
# NOTE (deviation from plan §8.3): the default fuses CLIENT-SIDE.
|
|
226
256
|
# The native single-roundtrip `$rankFusion` path is OPT-IN
|
|
@@ -233,20 +263,103 @@ module Parse
|
|
|
233
263
|
# two-aggregate client path unless a caller explicitly opts into
|
|
234
264
|
# native AND the cluster supports it. Native still falls back to
|
|
235
265
|
# the client path on any execution error.
|
|
236
|
-
if method == :rrf_native
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
266
|
+
if method == :rrf_native
|
|
267
|
+
native_resolution = Parse::ACLScope.resolve!(
|
|
268
|
+
scope_opts.dup, method_name: :"VectorSearch::Hybrid.search",
|
|
269
|
+
)
|
|
270
|
+
if rank_fusion_supported?(
|
|
271
|
+
collection_name,
|
|
272
|
+
authorizing_client: Parse::ACLScope.client_of(native_resolution),
|
|
273
|
+
)
|
|
274
|
+
# The native pipeline enforces ACL AFTER fusion, so its
|
|
275
|
+
# per-branch limit is the pre-ACL depth: the candidate
|
|
276
|
+
# window, not the fusion depth.
|
|
277
|
+
fused = run_native(collection_name, lex, vec, candidate_window,
|
|
278
|
+
k_constant: k_constant, weights: weights, scope_opts: scope_opts,
|
|
279
|
+
resolution: native_resolution)
|
|
280
|
+
if fused
|
|
281
|
+
trimmed = fused.first(k_int)
|
|
282
|
+
# Native retains the full candidate window per branch,
|
|
283
|
+
# NOT the client path's fusion depth: its ACL `$match`
|
|
284
|
+
# runs after `$rankFusion`, so the branch limit and the
|
|
285
|
+
# pre-ACL window are necessarily the same number. Report
|
|
286
|
+
# what actually executed rather than the client figure.
|
|
287
|
+
emit_hybrid_stats(collection_name: collection_name, k: k_int,
|
|
288
|
+
method: :rrf_native, branch_depth: candidate_window,
|
|
289
|
+
candidate_window: candidate_window,
|
|
290
|
+
post_filter_count: fused.length,
|
|
291
|
+
returned_count: trimmed.length)
|
|
292
|
+
return trimmed
|
|
293
|
+
end
|
|
294
|
+
end
|
|
240
295
|
end
|
|
241
296
|
|
|
242
|
-
lexical_rows = run_lexical(collection_name, lex,
|
|
243
|
-
vector_rows
|
|
244
|
-
rrf({ lexical: lexical_rows, vector: vector_rows },
|
|
245
|
-
|
|
297
|
+
lexical_rows = run_lexical(collection_name, lex, fusion_depth, scope_opts)
|
|
298
|
+
vector_rows = run_vector(collection_name, vec, fusion_depth, candidate_window, scope_opts)
|
|
299
|
+
fused = rrf({ lexical: lexical_rows, vector: vector_rows },
|
|
300
|
+
k_constant: k_constant, weights: weights)
|
|
301
|
+
trimmed = fused.first(k_int)
|
|
302
|
+
emit_hybrid_stats(collection_name: collection_name, k: k_int,
|
|
303
|
+
method: :rrf_client, branch_depth: fusion_depth,
|
|
304
|
+
candidate_window: candidate_window,
|
|
305
|
+
post_filter_count: fused.length,
|
|
306
|
+
returned_count: trimmed.length)
|
|
307
|
+
trimmed
|
|
246
308
|
end
|
|
247
309
|
|
|
248
310
|
private
|
|
249
311
|
|
|
312
|
+
# Resolve the two windows.
|
|
313
|
+
#
|
|
314
|
+
# * `candidate_window` — rows Atlas considers before ACL. Never
|
|
315
|
+
# narrower than the plain search's window, so opting into
|
|
316
|
+
# hybrid cannot make ACL underfill worse than a straight
|
|
317
|
+
# vector search. Bounded by Atlas's 10_000 ceiling.
|
|
318
|
+
# * `fusion_depth` — rows each branch retains for RRF. This
|
|
319
|
+
# becomes a branch `k`, so it is additionally bounded by
|
|
320
|
+
# VectorSearch::MAX_K; without that bound a hybrid `k` above
|
|
321
|
+
# 100 produces a branch `k` the plain search refuses outright.
|
|
322
|
+
#
|
|
323
|
+
# An out-of-range explicit `candidate_limit` is REFUSED rather
|
|
324
|
+
# than clamped: silently shrinking a caller's stated window
|
|
325
|
+
# would hide the very underfill they were trying to avoid.
|
|
326
|
+
#
|
|
327
|
+
# @return [Array(Integer, Integer)] `[fusion_depth, candidate_window]`
|
|
328
|
+
def resolve_windows(k_int, candidate_limit)
|
|
329
|
+
window = if candidate_limit
|
|
330
|
+
limit = Integer(candidate_limit)
|
|
331
|
+
if limit < k_int
|
|
332
|
+
raise ArgumentError,
|
|
333
|
+
"hybrid search: vector[:candidate_limit] (#{limit}) must be >= k (#{k_int})."
|
|
334
|
+
end
|
|
335
|
+
if limit > Parse::VectorSearch::MAX_CANDIDATE_LIMIT
|
|
336
|
+
raise ArgumentError,
|
|
337
|
+
"hybrid search: vector[:candidate_limit] (#{limit}) exceeds the Atlas " \
|
|
338
|
+
"ceiling (#{Parse::VectorSearch::MAX_CANDIDATE_LIMIT})."
|
|
339
|
+
end
|
|
340
|
+
limit
|
|
341
|
+
else
|
|
342
|
+
multiplier = [DEFAULT_OVERSAMPLE_MULTIPLIER,
|
|
343
|
+
Parse::VectorSearch::DEFAULT_CANDIDATE_MULTIPLIER].max
|
|
344
|
+
[[k_int * multiplier, k_int].max,
|
|
345
|
+
Parse::VectorSearch::MAX_CANDIDATE_LIMIT].min
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
depth = [[window, Parse::VectorSearch::MAX_K].min, k_int].max
|
|
349
|
+
[depth, window]
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
# Mirrors the plain search's attrition telemetry. `post_filter_count`
|
|
353
|
+
# is the fused row count after each branch has enforced ACL/CLP;
|
|
354
|
+
# `underfilled` means the caller received fewer than `k`.
|
|
355
|
+
def emit_hybrid_stats(**payload)
|
|
356
|
+
return unless defined?(ActiveSupport::Notifications)
|
|
357
|
+
|
|
358
|
+
payload[:underfilled] = payload[:returned_count] < payload[:k]
|
|
359
|
+
ActiveSupport::Notifications.instrument(AS_NOTIFICATION_NAME, payload)
|
|
360
|
+
nil
|
|
361
|
+
end
|
|
362
|
+
|
|
250
363
|
# -- client-side branch execution --------------------------------
|
|
251
364
|
|
|
252
365
|
def run_lexical(collection_name, lex, oversample, scope_opts)
|
|
@@ -263,12 +376,16 @@ module Parse
|
|
|
263
376
|
)
|
|
264
377
|
end
|
|
265
378
|
|
|
266
|
-
def run_vector(collection_name, vec,
|
|
379
|
+
def run_vector(collection_name, vec, fusion_depth, candidate_window, scope_opts)
|
|
267
380
|
Parse::VectorSearch.search(
|
|
268
381
|
collection_name,
|
|
269
382
|
field: vec[:field],
|
|
270
383
|
query_vector: vec[:query_vector],
|
|
271
|
-
k
|
|
384
|
+
# `k` is what this branch keeps; `candidate_limit` is the
|
|
385
|
+
# pre-ACL window. Passing the window as `k` and letting the
|
|
386
|
+
# plain search derive its own would multiply it twice.
|
|
387
|
+
k: fusion_depth,
|
|
388
|
+
candidate_limit: candidate_window,
|
|
272
389
|
num_candidates: vec[:num_candidates],
|
|
273
390
|
filter: vec[:filter],
|
|
274
391
|
vector_filter: vec[:vector_filter],
|
|
@@ -312,11 +429,17 @@ module Parse
|
|
|
312
429
|
fusion = symbolize(fusion || {})
|
|
313
430
|
lex = symbolize(lexical || {})
|
|
314
431
|
vec = symbolize(vector || {})
|
|
315
|
-
|
|
432
|
+
# Same resolution the live path uses, so the shape this
|
|
433
|
+
# returns is the shape that actually executes.
|
|
434
|
+
_depth, candidate_window = resolve_windows(Integer(k), vec[:candidate_limit])
|
|
316
435
|
resolution = Parse::ACLScope.resolve!(scope_opts.dup, method_name: :"VectorSearch::Hybrid.search")
|
|
317
|
-
|
|
436
|
+
# `limit:` must match {#run_native}'s — the final `$limit` runs
|
|
437
|
+
# AFTER the ACL `$match`, so trimming to `k` there would
|
|
438
|
+
# reintroduce the underfill the window exists to prevent. The
|
|
439
|
+
# trim to `k` happens client-side once enforcement is done.
|
|
440
|
+
native_pipeline_for(lex, vec, candidate_window, resolution,
|
|
318
441
|
k_constant: fusion[:k_constant] || DEFAULT_K_CONSTANT,
|
|
319
|
-
weights: fusion[:weights], limit:
|
|
442
|
+
weights: fusion[:weights], limit: candidate_window)
|
|
320
443
|
end
|
|
321
444
|
|
|
322
445
|
def native_pipeline_for(lex, vec, oversample, resolution, k_constant:, weights:, limit:)
|
|
@@ -333,8 +456,8 @@ module Parse
|
|
|
333
456
|
pipeline
|
|
334
457
|
end
|
|
335
458
|
|
|
336
|
-
def run_native(collection_name, lex, vec, oversample, k_constant:, weights:, scope_opts:)
|
|
337
|
-
resolution
|
|
459
|
+
def run_native(collection_name, lex, vec, oversample, k_constant:, weights:, scope_opts:, resolution: nil)
|
|
460
|
+
resolution ||= Parse::ACLScope.resolve!(scope_opts.dup, method_name: :"VectorSearch::Hybrid.search")
|
|
338
461
|
assert_clp_find!(collection_name, resolution)
|
|
339
462
|
pointer_fields = resolve_pointer_fields!(collection_name, resolution)
|
|
340
463
|
protected_fields = Parse::CLPScope.protected_fields_for(
|
|
@@ -347,7 +470,8 @@ module Parse
|
|
|
347
470
|
|
|
348
471
|
pipeline = native_pipeline_for(lex, vec, oversample, resolution,
|
|
349
472
|
k_constant: k_constant, weights: weights, limit: oversample)
|
|
350
|
-
rows = run_pipeline!(collection_name, pipeline
|
|
473
|
+
rows = run_pipeline!(collection_name, pipeline,
|
|
474
|
+
authorizing_client: Parse::ACLScope.client_of(resolution))
|
|
351
475
|
|
|
352
476
|
unless resolution.master?
|
|
353
477
|
# Defense-in-depth top-level row gate. The in-pipeline ACL
|
|
@@ -407,11 +531,11 @@ module Parse
|
|
|
407
531
|
num_candidates = (vec[:num_candidates] || oversample * Parse::VectorSearch::DEFAULT_NUM_CANDIDATES_MULTIPLIER).to_i
|
|
408
532
|
num_candidates = [[num_candidates, oversample].max, 10_000].min
|
|
409
533
|
stage = {
|
|
410
|
-
"index"
|
|
411
|
-
"path"
|
|
412
|
-
"queryVector"
|
|
534
|
+
"index" => vec[:index].to_s,
|
|
535
|
+
"path" => vec[:field].to_s,
|
|
536
|
+
"queryVector" => vec[:query_vector],
|
|
413
537
|
"numCandidates" => num_candidates,
|
|
414
|
-
"limit"
|
|
538
|
+
"limit" => oversample,
|
|
415
539
|
}
|
|
416
540
|
stage["filter"] = vec[:vector_filter] if vec[:vector_filter] && !vec[:vector_filter].empty?
|
|
417
541
|
inner = [{ "$vectorSearch" => stage }]
|
|
@@ -435,10 +559,18 @@ module Parse
|
|
|
435
559
|
|
|
436
560
|
# -- the $rankFusion support probe -------------------------------
|
|
437
561
|
|
|
438
|
-
|
|
439
|
-
|
|
562
|
+
# Capability probe only: runs `$rankFusion` with an empty input and a
|
|
563
|
+
# `$limit 0`, so it reads no rows. It still carries the authorizing
|
|
564
|
+
# client because the process-global MongoDB connection must enforce
|
|
565
|
+
# its application binding even for a zero-row probe.
|
|
566
|
+
def run_probe(collection_name, authorizing_client: nil)
|
|
567
|
+
coll = Parse::MongoDB.collection(collection_name, authorizing_client: authorizing_client)
|
|
440
568
|
coll.aggregate([{ "$rankFusion" => { "input" => {} } }, { "$limit" => 0 }]).to_a
|
|
441
569
|
true
|
|
570
|
+
rescue Parse::MongoDB::ClientMismatch
|
|
571
|
+
# A binding violation is not evidence that the stage exists. It is a
|
|
572
|
+
# security boundary and must reach the caller without being cached.
|
|
573
|
+
raise
|
|
442
574
|
rescue StandardError => e
|
|
443
575
|
# "Unknown aggregation stage $rankFusion" (or an unrecognized-
|
|
444
576
|
# operator variant) means the cluster predates native support.
|
|
@@ -534,8 +666,9 @@ module Parse
|
|
|
534
666
|
end
|
|
535
667
|
end
|
|
536
668
|
|
|
537
|
-
def run_pipeline!(collection_name, pipeline)
|
|
538
|
-
Parse::MongoDB.collection(collection_name)
|
|
669
|
+
def run_pipeline!(collection_name, pipeline, authorizing_client: nil)
|
|
670
|
+
Parse::MongoDB.collection(collection_name, authorizing_client: authorizing_client)
|
|
671
|
+
.aggregate(pipeline).to_a
|
|
539
672
|
end
|
|
540
673
|
|
|
541
674
|
def assert_clp_find!(collection_name, resolution)
|