parse-stack-next 5.7.5 → 5.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +856 -0
- data/README.md +15 -4
- data/docs/TEST_SERVER.md +2 -2
- data/docs/acl_clp_guide.md +7 -0
- data/docs/atlas_vector_search_guide.md +190 -14
- data/docs/client_sdk_guide.md +11 -0
- data/docs/mcp_guide.md +318 -6
- data/docs/mongodb_direct_guide.md +27 -0
- data/docs/usage_guide.md +38 -0
- data/docs/webhooks_guide.md +74 -17
- data/lib/parse/acl_scope.rb +159 -41
- data/lib/parse/agent/approval_gate.rb +0 -0
- data/lib/parse/agent/constraint_translator.rb +42 -15
- data/lib/parse/agent/describe.rb +3 -1
- data/lib/parse/agent/field_names.rb +53 -0
- data/lib/parse/agent/field_policy.rb +74 -0
- data/lib/parse/agent/mcp_deployments.rb +426 -0
- data/lib/parse/agent/mcp_rack_app.rb +424 -45
- data/lib/parse/agent/mcp_server.rb +23 -1
- data/lib/parse/agent/mcp_subscriptions.rb +124 -6
- data/lib/parse/agent/metadata_registry.rb +67 -8
- data/lib/parse/agent/prompt_hardening.rb +9 -3
- data/lib/parse/agent/tools.rb +378 -29
- data/lib/parse/agent.rb +93 -1
- data/lib/parse/api/batch.rb +10 -1
- data/lib/parse/api/schema.rb +23 -4
- data/lib/parse/api/sessions.rb +6 -2
- data/lib/parse/api/users.rb +88 -14
- data/lib/parse/atlas_search/protected_paths.rb +236 -0
- data/lib/parse/atlas_search.rb +95 -23
- data/lib/parse/authorization.rb +54 -1
- data/lib/parse/client/batch.rb +231 -35
- data/lib/parse/client/body_builder.rb +21 -0
- data/lib/parse/client/caching.rb +371 -27
- data/lib/parse/client/request.rb +26 -14
- data/lib/parse/client/response.rb +49 -6
- data/lib/parse/client.rb +201 -38
- data/lib/parse/clp_scope.rb +281 -23
- data/lib/parse/console.rb +2 -2
- data/lib/parse/embeddings/voyage.rb +181 -17
- data/lib/parse/graphql/type_generator.rb +3 -0
- data/lib/parse/model/acl.rb +119 -21
- data/lib/parse/model/associations/belongs_to.rb +25 -3
- data/lib/parse/model/associations/collection_proxy.rb +138 -17
- data/lib/parse/model/associations/has_many.rb +38 -9
- data/lib/parse/model/associations/has_one.rb +3 -1
- data/lib/parse/model/associations/pointer_collection_proxy.rb +109 -17
- data/lib/parse/model/associations/relation_collection_proxy.rb +134 -28
- data/lib/parse/model/bytes.rb +13 -5
- data/lib/parse/model/classes/role.rb +72 -0
- data/lib/parse/model/classes/session.rb +43 -0
- data/lib/parse/model/classes/user.rb +78 -3
- data/lib/parse/model/core/actions.rb +269 -67
- data/lib/parse/model/core/builder.rb +100 -8
- data/lib/parse/model/core/create_lock.rb +27 -2
- data/lib/parse/model/core/describe.rb +2 -0
- data/lib/parse/model/core/fetching.rb +21 -3
- data/lib/parse/model/core/pluralized_aliases.rb +8 -4
- data/lib/parse/model/core/properties.rb +488 -39
- data/lib/parse/model/core/querying.rb +7 -0
- data/lib/parse/model/core/schema.rb +5 -3
- data/lib/parse/model/core/search_indexing.rb +63 -0
- data/lib/parse/model/core/vector_searchable.rb +35 -6
- data/lib/parse/model/file.rb +9 -2
- data/lib/parse/model/geopoint.rb +61 -13
- data/lib/parse/model/model.rb +160 -9
- data/lib/parse/model/object.rb +265 -17
- data/lib/parse/model/phone.rb +54 -5
- data/lib/parse/model/pointer.rb +40 -6
- data/lib/parse/mongodb.rb +170 -60
- data/lib/parse/pipeline_security.rb +415 -26
- data/lib/parse/query/constraint.rb +30 -0
- data/lib/parse/query/constraints.rb +58 -32
- data/lib/parse/query/cursor.rb +3 -1
- data/lib/parse/query/operation.rb +62 -8
- data/lib/parse/query/ordering.rb +34 -6
- data/lib/parse/query.rb +1100 -134
- data/lib/parse/retrieval/agent_tool.rb +290 -17
- data/lib/parse/retrieval/benchmark.rb +149 -0
- data/lib/parse/retrieval/profiles.rb +320 -0
- data/lib/parse/retrieval/retriever.rb +10 -1
- data/lib/parse/retrieval.rb +2 -0
- data/lib/parse/schema/search_index_migrator.rb +23 -5
- data/lib/parse/schema.rb +74 -18
- data/lib/parse/stack/tasks.rb +6 -4
- data/lib/parse/stack/version.rb +1 -1
- data/lib/parse/stack.rb +72 -14
- data/lib/parse/two_factor_auth/user_extension.rb +14 -2
- data/lib/parse/two_factor_auth.rb +11 -0
- data/lib/parse/vector_search/hybrid.rb +36 -18
- data/lib/parse/vector_search/index_definition.rb +237 -0
- data/lib/parse/vector_search.rb +46 -17
- data/lib/parse/webhooks/payload.rb +93 -6
- data/lib/parse/webhooks/replay_protection.rb +58 -20
- data/lib/parse/webhooks.rb +412 -40
- metadata +8 -1
data/lib/parse/stack.rb
CHANGED
|
@@ -115,9 +115,12 @@ module Parse
|
|
|
115
115
|
#
|
|
116
116
|
# The `token` argument may be a String, a {Parse::User} (its
|
|
117
117
|
# `session_token` is read), a {Parse::Session} (its `session_token` is
|
|
118
|
-
# read), or `nil`. Passing `nil
|
|
119
|
-
#
|
|
120
|
-
# session
|
|
118
|
+
# read), or `nil`. Passing `nil`, or a user or session that carries no
|
|
119
|
+
# token, runs the block ANONYMOUSLY: requests inside it send neither a
|
|
120
|
+
# session token nor the master key, and a client's bound token is not
|
|
121
|
+
# used either. This is useful for performing one anonymous call inside an
|
|
122
|
+
# otherwise session-scoped region. A call inside the block can still opt
|
|
123
|
+
# out with an explicit `session_token:` or `use_master_key: true`.
|
|
121
124
|
#
|
|
122
125
|
# Fiber-local, not thread-local: concurrent fibers (and threads, since
|
|
123
126
|
# each thread starts with its own root fiber) do not share state.
|
|
@@ -156,17 +159,37 @@ module Parse
|
|
|
156
159
|
"token is refused so the block cannot silently execute with master-key " \
|
|
157
160
|
"authority — pass a valid session token, or `nil` for no ambient session."
|
|
158
161
|
end
|
|
159
|
-
|
|
162
|
+
# `nil` (no token, or a user/session without one) installs the
|
|
163
|
+
# anonymous sentinel rather than clearing the slot. A cleared slot meant
|
|
164
|
+
# "no ambient", which the request layer resolves to the master key on a
|
|
165
|
+
# master-keyed client: the opposite of the documented anonymous block.
|
|
166
|
+
Fiber[SESSION_TOKEN_STATE_KEY] = resolved.nil? ? ANONYMOUS_SESSION : resolved
|
|
160
167
|
yield
|
|
161
168
|
ensure
|
|
162
169
|
Fiber[SESSION_TOKEN_STATE_KEY] = previous
|
|
163
170
|
end
|
|
164
171
|
|
|
172
|
+
# Fiber-state sentinel installed by `Parse.with_session(nil)`. The request
|
|
173
|
+
# layer treats it as "anonymous": no session token and no master key.
|
|
174
|
+
# @!visibility private
|
|
175
|
+
ANONYMOUS_SESSION = :__parse_anonymous_session__
|
|
176
|
+
|
|
165
177
|
# The ambient session token set by {.with_session} for the current
|
|
166
|
-
# fiber, or `nil` when not inside such a block
|
|
178
|
+
# fiber, or `nil` when not inside such a block or inside an anonymous
|
|
179
|
+
# `with_session(nil)` block (see {.anonymous_session?}).
|
|
167
180
|
# @return [String, nil]
|
|
168
181
|
def self.current_session_token
|
|
169
|
-
Fiber[SESSION_TOKEN_STATE_KEY]
|
|
182
|
+
value = Fiber[SESSION_TOKEN_STATE_KEY]
|
|
183
|
+
value == ANONYMOUS_SESSION ? nil : value
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Whether the current fiber is inside an anonymous `Parse.with_session(nil)`
|
|
187
|
+
# block (directly, or nested without a token-bearing block inside it).
|
|
188
|
+
# Requests there carry no session token and no master key unless the call
|
|
189
|
+
# passes `session_token:` or `use_master_key: true` explicitly.
|
|
190
|
+
# @return [Boolean]
|
|
191
|
+
def self.anonymous_session?
|
|
192
|
+
Fiber[SESSION_TOKEN_STATE_KEY] == ANONYMOUS_SESSION
|
|
170
193
|
end
|
|
171
194
|
|
|
172
195
|
# @!visibility private
|
|
@@ -776,10 +799,17 @@ module Parse
|
|
|
776
799
|
# - the name singularizes to a *different* string (i.e. looks plural),
|
|
777
800
|
# - the singular form does NOT already end in `s` (per design: classes
|
|
778
801
|
# whose name ends in `s` are not auto-aliased),
|
|
779
|
-
# - the singular constant
|
|
780
|
-
#
|
|
781
|
-
#
|
|
782
|
-
#
|
|
802
|
+
# - the singular constant resolves from the referencing module and
|
|
803
|
+
# is a `Parse::Object` subclass,
|
|
804
|
+
# - the plural is not already defined in the singular class's own
|
|
805
|
+
# namespace.
|
|
806
|
+
#
|
|
807
|
+
# The alias is installed in the namespace that defines the singular
|
|
808
|
+
# class (`Object` for a top-level `Post`, `Blog` for `Blog::Post`), never
|
|
809
|
+
# on the module where the lookup happened. A reference to `Posts` from
|
|
810
|
+
# inside an unrelated class or module therefore resolves, but does not
|
|
811
|
+
# add a `Posts` constant to that module. A frozen namespace is left
|
|
812
|
+
# alone (no `FrozenError`).
|
|
783
813
|
#
|
|
784
814
|
# @param mod [Module] the module/class on which `const_missing` fired.
|
|
785
815
|
# @param name [Symbol] the missing constant name.
|
|
@@ -795,15 +825,40 @@ module Parse
|
|
|
795
825
|
return nil unless mod.const_defined?(sym, true)
|
|
796
826
|
klass = mod.const_get(sym)
|
|
797
827
|
return nil unless klass.is_a?(Class) && klass < Parse::Object
|
|
798
|
-
|
|
799
|
-
|
|
828
|
+
home = __pluralized_alias_home(klass, sym)
|
|
829
|
+
return nil if home.nil?
|
|
830
|
+
if home.const_defined?(name, false)
|
|
831
|
+
existing = home.const_get(name, false)
|
|
832
|
+
return existing.equal?(klass) ? klass : nil
|
|
833
|
+
end
|
|
834
|
+
return nil if home.frozen?
|
|
835
|
+
home.const_set(name, klass)
|
|
800
836
|
klass
|
|
801
|
-
rescue NameError, LoadError
|
|
837
|
+
rescue NameError, LoadError, FrozenError
|
|
802
838
|
# const_get/const_defined? can raise on malformed names or autoload
|
|
803
839
|
# failures; never let alias resolution mask the original lookup.
|
|
804
840
|
nil
|
|
805
841
|
end
|
|
806
842
|
|
|
843
|
+
# @!visibility private
|
|
844
|
+
# The namespace that directly defines `klass` under the name `singular`,
|
|
845
|
+
# which is where its pluralized alias belongs. nil when it cannot be
|
|
846
|
+
# determined (anonymous class, or a constant that only resolves through
|
|
847
|
+
# an ancestor).
|
|
848
|
+
# @param klass [Class]
|
|
849
|
+
# @param singular [Symbol]
|
|
850
|
+
# @return [Module, nil]
|
|
851
|
+
def __pluralized_alias_home(klass, singular)
|
|
852
|
+
name = klass.name
|
|
853
|
+
return nil if name.nil?
|
|
854
|
+
parent_name = name.rpartition("::").first
|
|
855
|
+
home = parent_name.empty? ? ::Object : ::Object.const_get(parent_name, false)
|
|
856
|
+
return nil unless home.is_a?(Module) && home.const_defined?(singular, false)
|
|
857
|
+
home.const_get(singular, false).equal?(klass) ? home : nil
|
|
858
|
+
rescue NameError
|
|
859
|
+
nil
|
|
860
|
+
end
|
|
861
|
+
|
|
807
862
|
# Verify that every association target across the loaded {Parse::Object}
|
|
808
863
|
# subclasses resolves to a known Parse class. Covers `belongs_to` and
|
|
809
864
|
# `property … as:` pointer targets (via each class's `references`),
|
|
@@ -1048,4 +1103,7 @@ Parse._attach_slow_query_subscriber! if Parse.slow_query_threshold_ms
|
|
|
1048
1103
|
# already defined. Gated at runtime on Parse.pluralized_aliases?.
|
|
1049
1104
|
require_relative "model/core/pluralized_aliases"
|
|
1050
1105
|
|
|
1051
|
-
|
|
1106
|
+
# Only hook into Rails when railties is loaded. A bare `Rails` module (for
|
|
1107
|
+
# example the one rails-html-sanitizer defines) has no `Rails::Railtie`, and
|
|
1108
|
+
# subclassing it would abort loading the gem.
|
|
1109
|
+
require_relative "stack/railtie" if defined?(::Rails::Railtie)
|
|
@@ -50,8 +50,14 @@ module Parse
|
|
|
50
50
|
def login_with_mfa(username, password, mfa_token)
|
|
51
51
|
raise MFA::RequiredError, "MFA token is required" if mfa_token.blank?
|
|
52
52
|
|
|
53
|
+
# `client.login_with_mfa` always goes out without the master key
|
|
54
|
+
# (see Parse::API::Users#login_with_mfa): with it, Parse Server
|
|
55
|
+
# would skip this verification and overwrite the enrolled secret.
|
|
53
56
|
response = client.login_with_mfa(username, password, mfa_token)
|
|
54
|
-
|
|
57
|
+
unless response.success?
|
|
58
|
+
raise MFA::VerificationError, response.error.to_s if MFA.invalid_token_response?(response)
|
|
59
|
+
return nil
|
|
60
|
+
end
|
|
55
61
|
|
|
56
62
|
# Self-fetch trust: an MFA login returns the authenticating
|
|
57
63
|
# user's own row, so authData here is legitimately theirs.
|
|
@@ -447,8 +453,14 @@ module Parse
|
|
|
447
453
|
# user = Parse::User.first
|
|
448
454
|
# user.login_with_mfa!("password123", "123456")
|
|
449
455
|
def login_with_mfa!(password, mfa_token = nil)
|
|
456
|
+
raise MFA::RequiredError, "MFA token is required for this account" if mfa_token.blank?
|
|
450
457
|
response = client.login_with_mfa(username.to_s, password.to_s, mfa_token)
|
|
451
|
-
|
|
458
|
+
unless response.success?
|
|
459
|
+
raise MFA::VerificationError, response.error.to_s if MFA.invalid_token_response?(response)
|
|
460
|
+
return false
|
|
461
|
+
end
|
|
462
|
+
# Self-fetch trust: the login response is this user's own row.
|
|
463
|
+
self.class.with_authdata_trust { apply_attributes!(response.result) }
|
|
452
464
|
session_token.present?
|
|
453
465
|
rescue Parse::Client::ResponseError => e
|
|
454
466
|
if e.message.include?("Missing additional authData")
|
|
@@ -62,6 +62,17 @@ module Parse
|
|
|
62
62
|
end
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
+
# Whether a failed login response is Parse Server rejecting the MFA code
|
|
66
|
+
# (as opposed to wrong credentials). Parse Server's MFA adapter throws a
|
|
67
|
+
# bare string, which reaches the client as code 141 with the adapter's
|
|
68
|
+
# message, e.g. "Invalid MFA token".
|
|
69
|
+
# @!visibility private
|
|
70
|
+
# @param response [Parse::Response]
|
|
71
|
+
# @return [Boolean]
|
|
72
|
+
def self.invalid_token_response?(response)
|
|
73
|
+
response.respond_to?(:error) && response.error.to_s.start_with?("Invalid MFA token", "Please enter the token")
|
|
74
|
+
end
|
|
75
|
+
|
|
65
76
|
# Error raised when MFA is required but not provided
|
|
66
77
|
class RequiredError < Parse::Error
|
|
67
78
|
def initialize(message = "MFA token is required for this account")
|
|
@@ -463,10 +463,33 @@ module Parse
|
|
|
463
463
|
protected_fields = Parse::CLPScope.protected_fields_for(
|
|
464
464
|
collection_name, resolution.permission_strings,
|
|
465
465
|
)
|
|
466
|
+
# Unconditional: `Parse::AtlasSearch` may already be partially
|
|
467
|
+
# defined by a submodule (session, protected_paths) without the
|
|
468
|
+
# main file loaded. `require_relative` is idempotent.
|
|
469
|
+
require_relative "../atlas_search"
|
|
470
|
+
Parse::AtlasSearch.send(:assert_search_fields_allowed!,
|
|
471
|
+
Array(lex[:fields]).map(&:to_s), protected_fields, resolution)
|
|
466
472
|
Parse::VectorSearch.validate_query_vector!(vec[:query_vector])
|
|
467
473
|
Parse::PipelineSecurity.validate_filter!(vec[:vector_filter]) if vec[:vector_filter]
|
|
468
474
|
Parse::PipelineSecurity.validate_filter!(vec[:filter]) if vec[:filter]
|
|
469
475
|
Parse::PipelineSecurity.validate_filter!(lex[:filter]) if lex[:filter]
|
|
476
|
+
# The client path gets these refusals from AtlasSearch.search and
|
|
477
|
+
# VectorSearch.search. The native path builds its own pipeline,
|
|
478
|
+
# so apply the same protected-field refusals here: the vector
|
|
479
|
+
# field, every filter's predicate keys, and `$expr` references.
|
|
480
|
+
Parse::VectorSearch.send(:assert_protected_fields_untouched!,
|
|
481
|
+
collection_name, vec[:field].to_s, vec[:filter],
|
|
482
|
+
vec[:vector_filter], protected_fields, resolution)
|
|
483
|
+
if lex[:filter] && !lex[:filter].empty? && !resolution.master?
|
|
484
|
+
Parse::PipelineSecurity.refuse_protected_field_references!(
|
|
485
|
+
[{ "$match" => lex[:filter] }], collection_name, resolution,
|
|
486
|
+
)
|
|
487
|
+
Parse::AtlasSearch::ProtectedPaths.assert_filter_allowed!(
|
|
488
|
+
lex[:filter], protected_fields, resolution,
|
|
489
|
+
collection_name: collection_name,
|
|
490
|
+
method_name: "Parse::VectorSearch::Hybrid.search",
|
|
491
|
+
)
|
|
492
|
+
end
|
|
470
493
|
|
|
471
494
|
pipeline = native_pipeline_for(lex, vec, oversample, resolution,
|
|
472
495
|
k_constant: k_constant, weights: weights, limit: oversample)
|
|
@@ -672,27 +695,22 @@ module Parse
|
|
|
672
695
|
end
|
|
673
696
|
|
|
674
697
|
def assert_clp_find!(collection_name, resolution)
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
end
|
|
698
|
+
# Same CLP branch evaluation as Parse::MongoDB.aggregate (public,
|
|
699
|
+
# user, and role grants first; then pointerFields / readUserFields).
|
|
700
|
+
# Raises Parse::CLPScope::Denied when the scope cannot find at all.
|
|
701
|
+
Parse::CLPScope.row_constraint_for!(collection_name, :find, resolution,
|
|
702
|
+
label: "hybrid-search")
|
|
703
|
+
nil
|
|
682
704
|
end
|
|
683
705
|
|
|
684
706
|
def resolve_pointer_fields!(collection_name, resolution)
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
"but the current hybrid-search scope has no user_id.",
|
|
693
|
-
)
|
|
694
|
-
end
|
|
695
|
-
pointer_fields
|
|
707
|
+
# nil when a public, user, or role grant already permits every row
|
|
708
|
+
# (Parse Server ignores pointer permissions then); otherwise the
|
|
709
|
+
# pointerFields plus readUserFields the rows must match. The older
|
|
710
|
+
# permits? / pointer_fields_for pair missed readUserFields entirely
|
|
711
|
+
# and over-restricted a public grant that also listed pointerFields.
|
|
712
|
+
Parse::CLPScope.row_constraint_for!(collection_name, :find, resolution,
|
|
713
|
+
label: "hybrid-search")
|
|
696
714
|
end
|
|
697
715
|
|
|
698
716
|
def validate_weights!(weights)
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Parse
|
|
5
|
+
module VectorSearch
|
|
6
|
+
# Derives an Atlas `vectorSearch` index definition from a model's
|
|
7
|
+
# declarations, so the index an operator deploys matches what the SDK
|
|
8
|
+
# will query instead of being written by hand.
|
|
9
|
+
#
|
|
10
|
+
# Sources, all read from the model:
|
|
11
|
+
#
|
|
12
|
+
# * the `:vector` property: path, `numDimensions` (`dimensions:`),
|
|
13
|
+
# `similarity` (`similarity:`, default `cosine`), and the optional
|
|
14
|
+
# index-side `quantization:` (`:scalar` / `:binary`);
|
|
15
|
+
# * `agent_searchable filter_fields:`, the fields the agent tool lets a
|
|
16
|
+
# caller pass as `vector_filter:` (pointer fields map to their
|
|
17
|
+
# `_p_<column>` storage path, matching the pointer translation
|
|
18
|
+
# {Parse::Retrieval.retrieve} applies);
|
|
19
|
+
# * the `agent_tenant_scope` field, which retrieval folds into
|
|
20
|
+
# `$vectorSearch.filter` on every scoped query.
|
|
21
|
+
#
|
|
22
|
+
# Output is deterministic: the vector entry first, then filter entries
|
|
23
|
+
# sorted by path, keys in a fixed order. Generating is side-effect free;
|
|
24
|
+
# applying an index stays explicit through
|
|
25
|
+
# {Parse::Schema::SearchIndexMigrator} (declare it with the
|
|
26
|
+
# `vector_search_index` model macro and run `apply_search_indexes!`).
|
|
27
|
+
#
|
|
28
|
+
# @example
|
|
29
|
+
# class Article < Parse::Object
|
|
30
|
+
# property :embedding, :vector, dimensions: 1024, similarity: :dotProduct,
|
|
31
|
+
# quantization: :scalar
|
|
32
|
+
# agent_searchable field: :embedding, filter_fields: %i[category]
|
|
33
|
+
# vector_search_index "article_vec" # generated; applied explicitly
|
|
34
|
+
# end
|
|
35
|
+
#
|
|
36
|
+
# Parse::VectorSearch::IndexDefinition.build(Article)
|
|
37
|
+
# # => { "fields" => [
|
|
38
|
+
# # { "type" => "vector", "path" => "embedding", "numDimensions" => 1024,
|
|
39
|
+
# # "similarity" => "dotProduct", "quantization" => "scalar" },
|
|
40
|
+
# # { "type" => "filter", "path" => "category" } ] }
|
|
41
|
+
module IndexDefinition
|
|
42
|
+
# Similarity emitted when the property declares none. Atlas requires
|
|
43
|
+
# one; cosine is the safe choice for unit-normalized embeddings.
|
|
44
|
+
DEFAULT_SIMILARITY = "cosine"
|
|
45
|
+
|
|
46
|
+
# Index-side quantization values accepted on a `:vector` property.
|
|
47
|
+
QUANTIZATIONS = %w[scalar binary].freeze
|
|
48
|
+
|
|
49
|
+
# Key order for each field entry, so serialized output is stable.
|
|
50
|
+
VECTOR_KEY_ORDER = %w[type path numDimensions similarity quantization].freeze
|
|
51
|
+
|
|
52
|
+
module_function
|
|
53
|
+
|
|
54
|
+
# Build the definition for one `:vector` field of `model_class`.
|
|
55
|
+
#
|
|
56
|
+
# @param model_class [Class] a Parse::Object subclass.
|
|
57
|
+
# @param field [Symbol, String, nil] the `:vector` property; may be
|
|
58
|
+
# omitted when the class declares exactly one searchable vector.
|
|
59
|
+
# @return [Hash] a string-keyed `{ "fields" => [...] }` definition.
|
|
60
|
+
# @raise [ArgumentError] when the field cannot be resolved.
|
|
61
|
+
def build(model_class, field: nil)
|
|
62
|
+
field_sym = resolve_field!(model_class, field)
|
|
63
|
+
meta = model_class.vector_properties.fetch(field_sym)
|
|
64
|
+
|
|
65
|
+
vector = {
|
|
66
|
+
"type" => "vector",
|
|
67
|
+
# The stored column (field_map), which is what $vectorSearch
|
|
68
|
+
# queries: `body_embedding` is saved as `bodyEmbedding`.
|
|
69
|
+
"path" => (model_class.respond_to?(:vector_storage_path) ? model_class.vector_storage_path(field_sym) : field_sym.to_s),
|
|
70
|
+
"numDimensions" => meta[:dimensions],
|
|
71
|
+
"similarity" => (meta[:similarity] || DEFAULT_SIMILARITY).to_s,
|
|
72
|
+
}
|
|
73
|
+
vector["quantization"] = meta[:quantization].to_s if meta[:quantization]
|
|
74
|
+
|
|
75
|
+
filters = filter_paths(model_class).map { |path| { "type" => "filter", "path" => path } }
|
|
76
|
+
{ "fields" => [order_keys(vector)] + filters }
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Preview the declaration a `vector_search_index` macro would apply.
|
|
80
|
+
#
|
|
81
|
+
# @return [Hash] `{ name:, type: "vectorSearch", definition: }`.
|
|
82
|
+
def preview(model_class, name:, field: nil)
|
|
83
|
+
{ name: name.to_s, type: "vectorSearch", definition: build(model_class, field: field) }
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Structured, deterministic comparison of a declared definition with a
|
|
87
|
+
# live one. Accepts either definitions or index documents carrying
|
|
88
|
+
# `latestDefinition` (the shape `$listSearchIndexes` returns).
|
|
89
|
+
#
|
|
90
|
+
# @param declared [Hash] the generated (or declared) definition.
|
|
91
|
+
# @param live [Hash, nil] the live definition or index document.
|
|
92
|
+
# @return [Hash] `{ in_sync:, vector: { key => { declared:, live: } },
|
|
93
|
+
# filters_missing: [...], filters_extra: [...] }`. `vector` lists
|
|
94
|
+
# only differing keys; `filters_missing` are declared filter paths
|
|
95
|
+
# absent from the live index (scoped queries on them fail
|
|
96
|
+
# Atlas-side), `filters_extra` are live paths not declared.
|
|
97
|
+
def diff(declared, live)
|
|
98
|
+
declared_defn = definition_of(declared)
|
|
99
|
+
live_defn = definition_of(live)
|
|
100
|
+
|
|
101
|
+
d_vec = vector_entry(declared_defn)
|
|
102
|
+
# Compare against the live vector entry for the SAME path, so an
|
|
103
|
+
# index with several vector fields does not report false drift.
|
|
104
|
+
l_vec = vector_entry(live_defn, path: d_vec["path"])
|
|
105
|
+
vector_changes = {}
|
|
106
|
+
(VECTOR_KEY_ORDER - %w[type]).each do |key|
|
|
107
|
+
dv = normalize_vector_value(key, d_vec[key])
|
|
108
|
+
lv = normalize_vector_value(key, l_vec[key])
|
|
109
|
+
vector_changes[key] = { declared: dv, live: lv } unless dv == lv
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
d_filters = filter_paths_of(declared_defn)
|
|
113
|
+
l_filters = filter_paths_of(live_defn)
|
|
114
|
+
missing = (d_filters - l_filters).sort
|
|
115
|
+
extra = (l_filters - d_filters).sort
|
|
116
|
+
|
|
117
|
+
{
|
|
118
|
+
in_sync: vector_changes.empty? && missing.empty? && extra.empty?,
|
|
119
|
+
vector: vector_changes,
|
|
120
|
+
filters_missing: missing,
|
|
121
|
+
filters_extra: extra,
|
|
122
|
+
}
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# @!visibility private
|
|
126
|
+
def resolve_field!(model_class, field)
|
|
127
|
+
unless model_class.respond_to?(:vector_properties)
|
|
128
|
+
raise ArgumentError, "#{model_class.inspect} declares no :vector properties."
|
|
129
|
+
end
|
|
130
|
+
props = model_class.vector_properties
|
|
131
|
+
searchable = props.keys.reject { |k| props[k][:searchable] == false }
|
|
132
|
+
if field
|
|
133
|
+
sym = field.to_sym
|
|
134
|
+
unless searchable.include?(sym)
|
|
135
|
+
raise ArgumentError,
|
|
136
|
+
"#{model_class}: :#{sym} is not a searchable :vector property " \
|
|
137
|
+
"(searchable: #{searchable.inspect})."
|
|
138
|
+
end
|
|
139
|
+
return sym
|
|
140
|
+
end
|
|
141
|
+
return searchable.first if searchable.length == 1
|
|
142
|
+
raise ArgumentError,
|
|
143
|
+
"#{model_class}: cannot infer the vector field (searchable: #{searchable.inspect}); " \
|
|
144
|
+
"pass field:."
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# @!visibility private
|
|
148
|
+
# Filter paths the SDK pre-filters on for this class, deduplicated and
|
|
149
|
+
# sorted. Tenant path matches the resolution the migrator's
|
|
150
|
+
# augmentation and first-query drift check use.
|
|
151
|
+
def filter_paths(model_class)
|
|
152
|
+
paths = []
|
|
153
|
+
class_name = model_class.parse_class
|
|
154
|
+
if defined?(Parse::Agent::MetadataRegistry)
|
|
155
|
+
Parse::Agent::MetadataRegistry.searchable_filter_fields(class_name).each do |f|
|
|
156
|
+
paths << storage_path(model_class, f)
|
|
157
|
+
end
|
|
158
|
+
rule = Parse::Agent::MetadataRegistry.tenant_scope_rule(class_name)
|
|
159
|
+
paths << wire_name(model_class, rule[:field]) if rule
|
|
160
|
+
end
|
|
161
|
+
paths.compact.uniq.sort
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# @!visibility private
|
|
165
|
+
def storage_path(model_class, field)
|
|
166
|
+
wire = wire_name(model_class, field)
|
|
167
|
+
type = model_class.respond_to?(:fields) ? model_class.fields[field.to_sym] : nil
|
|
168
|
+
type == :pointer ? "_p_#{wire}" : wire
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# @!visibility private
|
|
172
|
+
def wire_name(model_class, field)
|
|
173
|
+
sym = field.to_sym
|
|
174
|
+
fmap = model_class.respond_to?(:field_map) ? model_class.field_map : {}
|
|
175
|
+
(fmap[sym] || sym.to_s.columnize).to_s
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# @!visibility private
|
|
179
|
+
def order_keys(entry)
|
|
180
|
+
VECTOR_KEY_ORDER.each_with_object({}) { |k, h| h[k] = entry[k] if entry.key?(k) }
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# @!visibility private
|
|
184
|
+
def definition_of(value)
|
|
185
|
+
return {} unless value.is_a?(Hash)
|
|
186
|
+
inner = value["latestDefinition"] || value[:latestDefinition] ||
|
|
187
|
+
value[:definition] || value["definition"]
|
|
188
|
+
inner.is_a?(Hash) ? inner : value
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# @!visibility private
|
|
192
|
+
def fields_of(defn)
|
|
193
|
+
Array(defn["fields"] || defn[:fields]).select { |f| f.is_a?(Hash) }
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# @!visibility private
|
|
197
|
+
def vector_entry(defn, path: nil)
|
|
198
|
+
vectors = fields_of(defn).select { |f| (f["type"] || f[:type]).to_s == "vector" }
|
|
199
|
+
entry = (path && vectors.find { |f| (f["path"] || f[:path]).to_s == path.to_s }) || vectors.first || {}
|
|
200
|
+
entry.each_with_object({}) { |(k, v), h| h[k.to_s] = v }
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# @!visibility private
|
|
204
|
+
def filter_paths_of(defn)
|
|
205
|
+
fields_of(defn).select { |f| (f["type"] || f[:type]).to_s == "filter" }
|
|
206
|
+
.map { |f| (f["path"] || f[:path]).to_s }.uniq.sort
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# @!visibility private
|
|
210
|
+
# An absent `quantization` means none on both sides; numbers compare
|
|
211
|
+
# as integers; everything else as strings.
|
|
212
|
+
def normalize_vector_value(key, value)
|
|
213
|
+
case key
|
|
214
|
+
when "quantization"
|
|
215
|
+
value.nil? || value.to_s.empty? ? "none" : value.to_s
|
|
216
|
+
when "numDimensions"
|
|
217
|
+
# A malformed live value is reported as drift, not raised.
|
|
218
|
+
value.nil? ? nil : (Integer(value) rescue value.to_s)
|
|
219
|
+
else
|
|
220
|
+
value&.to_s
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
module Schema
|
|
227
|
+
class << self
|
|
228
|
+
# Generate the Atlas `vectorSearch` index definition for a model's
|
|
229
|
+
# `:vector` field. See {Parse::VectorSearch::IndexDefinition.build}.
|
|
230
|
+
#
|
|
231
|
+
# @return [Hash]
|
|
232
|
+
def vector_index_definition(model_class, field: nil)
|
|
233
|
+
Parse::VectorSearch::IndexDefinition.build(model_class, field: field)
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
end
|
data/lib/parse/vector_search.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "pipeline_security"
|
|
|
5
5
|
require_relative "acl_scope"
|
|
6
6
|
require_relative "clp_scope"
|
|
7
7
|
require_relative "mongodb"
|
|
8
|
+
require_relative "atlas_search/protected_paths"
|
|
8
9
|
|
|
9
10
|
module Parse
|
|
10
11
|
# Atlas Vector Search entry point. Routes through `Parse::MongoDB`
|
|
@@ -306,6 +307,8 @@ module Parse
|
|
|
306
307
|
protected_fields = Parse::CLPScope.protected_fields_for(
|
|
307
308
|
collection_name, resolution.permission_strings,
|
|
308
309
|
)
|
|
310
|
+
assert_protected_fields_untouched!(collection_name, path, filter, vector_filter,
|
|
311
|
+
protected_fields, resolution)
|
|
309
312
|
|
|
310
313
|
vs_stage = {
|
|
311
314
|
"index" => index_name.to_s,
|
|
@@ -453,31 +456,55 @@ module Parse
|
|
|
453
456
|
# scope, refuse the call when the resolved claim set can't
|
|
454
457
|
# `find` on the collection. Mirrors `Parse::AtlasSearch.search`.
|
|
455
458
|
def assert_clp_find!(collection_name, resolution)
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
459
|
+
# Same CLP branch evaluation as Parse::MongoDB.aggregate (public,
|
|
460
|
+
# user, and role grants first; then pointerFields / readUserFields).
|
|
461
|
+
# Raises Parse::CLPScope::Denied when the scope cannot find at all.
|
|
462
|
+
Parse::CLPScope.row_constraint_for!(collection_name, :find, resolution,
|
|
463
|
+
label: "VectorSearch")
|
|
464
|
+
nil
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
# Resolve and return pointerFields for `find` on the collection.
|
|
468
|
+
# Refuse a scoped vector search that lets a protected field decide
|
|
469
|
+
# which rows match or how they rank: a protected vector `field:`, a
|
|
470
|
+
# `filter:` / `vector_filter:` predicate keyed on a protected field
|
|
471
|
+
# (top level or under $and/$or/$nor/$not, dotted or `_p_` form), or
|
|
472
|
+
# an `$expr` reference to one. The output strip alone does not close
|
|
473
|
+
# that oracle. Master scopes and classes with nothing protected are
|
|
474
|
+
# unaffected.
|
|
475
|
+
#
|
|
476
|
+
# @raise [Parse::CLPScope::Denied]
|
|
477
|
+
def assert_protected_fields_untouched!(collection_name, path, filter, vector_filter,
|
|
478
|
+
protected_fields, resolution)
|
|
479
|
+
paths = Parse::AtlasSearch::ProtectedPaths
|
|
480
|
+
return unless paths.enforce?(resolution, protected_fields)
|
|
481
|
+
paths.assert_paths_allowed!(path, protected_fields, resolution,
|
|
482
|
+
collection_name: collection_name,
|
|
483
|
+
method_name: "Parse::VectorSearch.search",
|
|
484
|
+
what: "vector field")
|
|
485
|
+
[filter, vector_filter].each do |f|
|
|
486
|
+
next if f.nil? || f.empty?
|
|
487
|
+
Parse::PipelineSecurity.refuse_protected_field_references!(
|
|
488
|
+
[{ "$match" => f }], collection_name, resolution,
|
|
461
489
|
)
|
|
490
|
+
paths.assert_filter_allowed!(f, protected_fields, resolution,
|
|
491
|
+
collection_name: collection_name,
|
|
492
|
+
method_name: "Parse::VectorSearch.search")
|
|
462
493
|
end
|
|
494
|
+
nil
|
|
463
495
|
end
|
|
464
496
|
|
|
465
|
-
# Resolve and return pointerFields for `find` on the collection.
|
|
466
497
|
# Raises CLPScope::Denied when pointerFields is set but the
|
|
467
498
|
# current scope has no user_id (acl_role-only / public agents).
|
|
468
499
|
# Returns nil when master-mode or no pointerFields entry exists.
|
|
469
500
|
def resolve_pointer_fields!(collection_name, resolution)
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
"but the current VectorSearch scope has no user_id.",
|
|
478
|
-
)
|
|
479
|
-
end
|
|
480
|
-
pointer_fields
|
|
501
|
+
# nil when a public, user, or role grant already permits every row
|
|
502
|
+
# (Parse Server ignores pointer permissions then); otherwise the
|
|
503
|
+
# pointerFields plus readUserFields the rows must match. The older
|
|
504
|
+
# permits? / pointer_fields_for pair missed readUserFields entirely
|
|
505
|
+
# and over-restricted a public grant that also listed pointerFields.
|
|
506
|
+
Parse::CLPScope.row_constraint_for!(collection_name, :find, resolution,
|
|
507
|
+
label: "VectorSearch")
|
|
481
508
|
end
|
|
482
509
|
|
|
483
510
|
# Execute the pipeline directly against the MongoDB collection.
|
|
@@ -501,3 +528,5 @@ module Parse
|
|
|
501
528
|
@default_index = nil
|
|
502
529
|
end
|
|
503
530
|
end
|
|
531
|
+
|
|
532
|
+
require_relative "vector_search/index_definition"
|