parse-stack-next 5.7.6 → 5.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +830 -0
- data/README.md +14 -4
- data/docs/TEST_SERVER.md +2 -2
- data/docs/acl_clp_guide.md +7 -0
- data/docs/atlas_vector_search_guide.md +181 -13
- data/docs/client_sdk_guide.md +11 -0
- data/docs/mcp_guide.md +317 -6
- data/docs/mongodb_direct_guide.md +27 -0
- data/docs/usage_guide.md +38 -0
- data/docs/webhooks_guide.md +74 -17
- data/lib/parse/acl_scope.rb +159 -41
- data/lib/parse/agent/approval_gate.rb +0 -0
- data/lib/parse/agent/constraint_translator.rb +42 -15
- data/lib/parse/agent/describe.rb +3 -1
- data/lib/parse/agent/field_names.rb +53 -0
- data/lib/parse/agent/field_policy.rb +74 -0
- data/lib/parse/agent/mcp_deployments.rb +426 -0
- data/lib/parse/agent/mcp_rack_app.rb +424 -45
- data/lib/parse/agent/mcp_server.rb +23 -1
- data/lib/parse/agent/mcp_subscriptions.rb +124 -6
- data/lib/parse/agent/metadata_registry.rb +67 -8
- data/lib/parse/agent/prompt_hardening.rb +9 -3
- data/lib/parse/agent/tools.rb +378 -29
- data/lib/parse/agent.rb +93 -1
- data/lib/parse/api/batch.rb +10 -1
- data/lib/parse/api/schema.rb +23 -4
- data/lib/parse/api/sessions.rb +6 -2
- data/lib/parse/api/users.rb +88 -14
- data/lib/parse/atlas_search/protected_paths.rb +236 -0
- data/lib/parse/atlas_search.rb +95 -23
- data/lib/parse/authorization.rb +54 -1
- data/lib/parse/client/batch.rb +231 -35
- data/lib/parse/client/body_builder.rb +21 -0
- data/lib/parse/client/caching.rb +371 -27
- data/lib/parse/client/request.rb +26 -14
- data/lib/parse/client/response.rb +49 -6
- data/lib/parse/client.rb +201 -38
- data/lib/parse/clp_scope.rb +281 -23
- data/lib/parse/console.rb +2 -2
- data/lib/parse/embeddings/voyage.rb +181 -17
- data/lib/parse/graphql/type_generator.rb +3 -0
- data/lib/parse/model/acl.rb +119 -21
- data/lib/parse/model/associations/belongs_to.rb +25 -3
- data/lib/parse/model/associations/collection_proxy.rb +138 -17
- data/lib/parse/model/associations/has_many.rb +38 -9
- data/lib/parse/model/associations/has_one.rb +3 -1
- data/lib/parse/model/associations/pointer_collection_proxy.rb +109 -17
- data/lib/parse/model/associations/relation_collection_proxy.rb +134 -28
- data/lib/parse/model/bytes.rb +13 -5
- data/lib/parse/model/classes/role.rb +72 -0
- data/lib/parse/model/classes/session.rb +43 -0
- data/lib/parse/model/classes/user.rb +78 -3
- data/lib/parse/model/core/actions.rb +269 -67
- data/lib/parse/model/core/builder.rb +100 -8
- data/lib/parse/model/core/create_lock.rb +27 -2
- data/lib/parse/model/core/describe.rb +2 -0
- data/lib/parse/model/core/fetching.rb +21 -3
- data/lib/parse/model/core/pluralized_aliases.rb +8 -4
- data/lib/parse/model/core/properties.rb +488 -39
- data/lib/parse/model/core/querying.rb +7 -0
- data/lib/parse/model/core/schema.rb +5 -3
- data/lib/parse/model/core/search_indexing.rb +63 -0
- data/lib/parse/model/core/vector_searchable.rb +35 -6
- data/lib/parse/model/file.rb +9 -2
- data/lib/parse/model/geopoint.rb +61 -13
- data/lib/parse/model/model.rb +160 -9
- data/lib/parse/model/object.rb +265 -17
- data/lib/parse/model/phone.rb +54 -5
- data/lib/parse/model/pointer.rb +40 -6
- data/lib/parse/mongodb.rb +170 -60
- data/lib/parse/pipeline_security.rb +415 -26
- data/lib/parse/query/constraint.rb +30 -0
- data/lib/parse/query/constraints.rb +58 -32
- data/lib/parse/query/cursor.rb +3 -1
- data/lib/parse/query/operation.rb +62 -8
- data/lib/parse/query/ordering.rb +34 -6
- data/lib/parse/query.rb +1100 -134
- data/lib/parse/retrieval/agent_tool.rb +225 -8
- data/lib/parse/retrieval/benchmark.rb +149 -0
- data/lib/parse/retrieval/profiles.rb +320 -0
- data/lib/parse/retrieval/retriever.rb +10 -1
- data/lib/parse/retrieval.rb +2 -0
- data/lib/parse/schema/search_index_migrator.rb +23 -5
- data/lib/parse/schema.rb +74 -18
- data/lib/parse/stack/tasks.rb +6 -4
- data/lib/parse/stack/version.rb +1 -1
- data/lib/parse/stack.rb +72 -14
- data/lib/parse/two_factor_auth/user_extension.rb +14 -2
- data/lib/parse/two_factor_auth.rb +11 -0
- data/lib/parse/vector_search/hybrid.rb +36 -18
- data/lib/parse/vector_search/index_definition.rb +237 -0
- data/lib/parse/vector_search.rb +46 -17
- data/lib/parse/webhooks/payload.rb +93 -6
- data/lib/parse/webhooks/replay_protection.rb +58 -20
- data/lib/parse/webhooks.rb +412 -40
- metadata +8 -1
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "timeout"
|
|
5
|
+
|
|
6
|
+
module Parse
|
|
7
|
+
module Retrieval
|
|
8
|
+
# Server-configured retrieval profiles for the `semantic_search` agent
|
|
9
|
+
# tool.
|
|
10
|
+
#
|
|
11
|
+
# A profile composes the library's existing retrieval paths (vector,
|
|
12
|
+
# hybrid, reranking) with explicit budgets, so an application can offer
|
|
13
|
+
# an agent a few named strategies (for example `fast`, `balanced`,
|
|
14
|
+
# `precise`) without letting the model pick providers, endpoints, or
|
|
15
|
+
# credentials. The agent only ever names a profile; everything else is
|
|
16
|
+
# fixed on the server at registration time.
|
|
17
|
+
#
|
|
18
|
+
# @example
|
|
19
|
+
# Parse::Retrieval.register_reranker(:voyage,
|
|
20
|
+
# Parse::Retrieval::Reranker::Voyage.new(api_key: ENV.fetch("VOYAGE_API_KEY"), model: "rerank-3-lite"))
|
|
21
|
+
#
|
|
22
|
+
# Parse::Retrieval::Profiles.register(:fast, k: 5, max_k: 10)
|
|
23
|
+
# Parse::Retrieval::Profiles.register(:precise,
|
|
24
|
+
# reranker: :voyage, rerank_candidates: 30, rerank_top_n: 8,
|
|
25
|
+
# rerank_max_document_chars: 4_000, rerank_timeout: 5,
|
|
26
|
+
# on_rerank_failure: :fallback, max_total_tokens: 8_000)
|
|
27
|
+
#
|
|
28
|
+
# agent.execute(:semantic_search, class_name: "Article", query: "refund policy", profile: "precise")
|
|
29
|
+
#
|
|
30
|
+
# Registration validates the whole configuration, so a misconfigured
|
|
31
|
+
# profile (unknown option, unregistered reranker, non-positive budget)
|
|
32
|
+
# fails at boot rather than mid-request. Without a `profile:` argument
|
|
33
|
+
# `semantic_search` behaves exactly as before.
|
|
34
|
+
module Profiles
|
|
35
|
+
# A validated, frozen profile.
|
|
36
|
+
Profile = Struct.new(
|
|
37
|
+
:name, :k, :max_k, :hybrid, :reranker, :rerank_candidates, :rerank_top_n,
|
|
38
|
+
:rerank_max_document_chars, :rerank_timeout, :on_rerank_failure, :max_total_tokens,
|
|
39
|
+
keyword_init: true,
|
|
40
|
+
) do
|
|
41
|
+
# @return [Boolean] true when the profile reranks.
|
|
42
|
+
def rerank?
|
|
43
|
+
!reranker.nil?
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
ALLOWED_OPTIONS = %i[
|
|
48
|
+
k max_k hybrid reranker rerank_candidates rerank_top_n
|
|
49
|
+
rerank_max_document_chars rerank_timeout on_rerank_failure max_total_tokens
|
|
50
|
+
].freeze
|
|
51
|
+
|
|
52
|
+
# Defaults applied to an option the profile does not set.
|
|
53
|
+
DEFAULTS = {
|
|
54
|
+
k: 10,
|
|
55
|
+
max_k: 20,
|
|
56
|
+
hybrid: nil,
|
|
57
|
+
rerank_candidates: 30,
|
|
58
|
+
rerank_top_n: nil,
|
|
59
|
+
rerank_max_document_chars: 4_000,
|
|
60
|
+
rerank_timeout: 10,
|
|
61
|
+
on_rerank_failure: :fallback,
|
|
62
|
+
max_total_tokens: nil,
|
|
63
|
+
}.freeze
|
|
64
|
+
|
|
65
|
+
# Hard ceilings no profile may exceed, so a misconfigured profile still
|
|
66
|
+
# cannot fan out unbounded provider work.
|
|
67
|
+
MAX_RERANK_CANDIDATES = 100
|
|
68
|
+
# semantic_search returns at most this many documents, so a larger
|
|
69
|
+
# max_k could never take effect; it is refused rather than clamped.
|
|
70
|
+
MAX_K = 20
|
|
71
|
+
MAX_RERANK_DOCUMENT_CHARS = 32_000
|
|
72
|
+
|
|
73
|
+
FAILURE_MODES = %i[fallback raise].freeze
|
|
74
|
+
# Options that only have an effect when the profile names a reranker.
|
|
75
|
+
RERANK_OPTIONS = %i[rerank_candidates rerank_top_n rerank_max_document_chars rerank_timeout on_rerank_failure].freeze
|
|
76
|
+
HYBRID_KEYS = %i[lexical vector fusion].freeze
|
|
77
|
+
NAME_RE = /\A[a-z][a-z0-9_]{0,39}\z/.freeze
|
|
78
|
+
|
|
79
|
+
@registry = {}
|
|
80
|
+
@mutex = Mutex.new
|
|
81
|
+
|
|
82
|
+
class << self
|
|
83
|
+
# Register (or replace) a profile.
|
|
84
|
+
#
|
|
85
|
+
# @param name [Symbol, String] lowercase identifier.
|
|
86
|
+
# @param options [Hash] see {ALLOWED_OPTIONS}.
|
|
87
|
+
# * `k`, `max_k` [Integer]: default and maximum results.
|
|
88
|
+
# * `hybrid` [true, Hash]: run lexical + vector fusion; a Hash
|
|
89
|
+
# carries server-side `lexical:`, `vector:`, `fusion:` settings.
|
|
90
|
+
# * `reranker` [Symbol, String]: a name registered with
|
|
91
|
+
# {Parse::Retrieval.register_reranker}.
|
|
92
|
+
# * `rerank_candidates` [Integer]: documents retrieved and sent to
|
|
93
|
+
# the reranker (capped at {MAX_RERANK_CANDIDATES}).
|
|
94
|
+
# * `rerank_top_n` [Integer, nil]: documents kept after reranking
|
|
95
|
+
# (defaults to the effective `k`).
|
|
96
|
+
# * `rerank_max_document_chars` [Integer]: each document's text is
|
|
97
|
+
# cut to this length before it is sent to the reranker.
|
|
98
|
+
# * `rerank_timeout` [Numeric]: seconds allowed for reranking.
|
|
99
|
+
# * `on_rerank_failure` [:fallback, :raise]: on timeout or provider
|
|
100
|
+
# failure, keep the retrieval order (observable) or fail the call.
|
|
101
|
+
# * `max_total_tokens` [Integer, nil]: default response budget.
|
|
102
|
+
# @return [Profile]
|
|
103
|
+
# @raise [ArgumentError] on any invalid option.
|
|
104
|
+
def register(name, **options)
|
|
105
|
+
profile = build(name, options)
|
|
106
|
+
@mutex.synchronize { @registry[profile.name] = profile }
|
|
107
|
+
profile
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# @return [Profile] the registered profile.
|
|
111
|
+
# @raise [Parse::Agent::ValidationError] for an unknown name, listing
|
|
112
|
+
# the registered ones.
|
|
113
|
+
def fetch!(name)
|
|
114
|
+
key = name.to_s
|
|
115
|
+
profile = @mutex.synchronize { @registry[key] }
|
|
116
|
+
return profile if profile
|
|
117
|
+
|
|
118
|
+
raise Parse::Agent::ValidationError,
|
|
119
|
+
"Unknown retrieval profile #{key.inspect}. Available: #{names.inspect}."
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# @return [Array<String>] registered profile names, sorted.
|
|
123
|
+
def names
|
|
124
|
+
@mutex.synchronize { @registry.keys.sort }
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def unregister(name)
|
|
128
|
+
@mutex.synchronize { @registry.delete(name.to_s) }
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def reset!
|
|
132
|
+
@mutex.synchronize { @registry.clear }
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
private
|
|
136
|
+
|
|
137
|
+
def build(name, options)
|
|
138
|
+
key = name.to_s
|
|
139
|
+
unless key.match?(NAME_RE)
|
|
140
|
+
raise ArgumentError, "Retrieval profile name #{name.inspect} must match #{NAME_RE.inspect}."
|
|
141
|
+
end
|
|
142
|
+
unknown = options.keys.map(&:to_sym) - ALLOWED_OPTIONS
|
|
143
|
+
unless unknown.empty?
|
|
144
|
+
raise ArgumentError,
|
|
145
|
+
"Retrieval profile #{key.inspect}: unknown option(s) #{unknown.inspect}. " \
|
|
146
|
+
"Allowed: #{ALLOWED_OPTIONS.inspect}."
|
|
147
|
+
end
|
|
148
|
+
given = options.transform_keys(&:to_sym)
|
|
149
|
+
rerank_only = given.keys & RERANK_OPTIONS
|
|
150
|
+
if given[:reranker].nil? && !rerank_only.empty?
|
|
151
|
+
raise ArgumentError,
|
|
152
|
+
"Retrieval profile #{key.inspect}: #{rerank_only.inspect} only apply with a reranker:; " \
|
|
153
|
+
"set reranker: or remove them."
|
|
154
|
+
end
|
|
155
|
+
opts = DEFAULTS.merge(given)
|
|
156
|
+
|
|
157
|
+
k = positive_int!(key, :k, opts[:k])
|
|
158
|
+
max_k = positive_int!(key, :max_k, opts[:max_k])
|
|
159
|
+
raise ArgumentError, "Retrieval profile #{key.inspect}: k (#{k}) exceeds max_k (#{max_k})." if k > max_k
|
|
160
|
+
if max_k > MAX_K
|
|
161
|
+
raise ArgumentError,
|
|
162
|
+
"Retrieval profile #{key.inspect}: max_k #{max_k} exceeds the semantic_search maximum (#{MAX_K})."
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
reranker = nil
|
|
166
|
+
unless opts[:reranker].nil?
|
|
167
|
+
reranker = opts[:reranker].to_s
|
|
168
|
+
unless Parse::Retrieval.reranker_registered?(reranker)
|
|
169
|
+
raise ArgumentError,
|
|
170
|
+
"Retrieval profile #{key.inspect}: reranker #{reranker.inspect} is not registered. " \
|
|
171
|
+
"Register it with Parse::Retrieval.register_reranker before the profile."
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
candidates = positive_int!(key, :rerank_candidates, opts[:rerank_candidates])
|
|
176
|
+
if candidates > MAX_RERANK_CANDIDATES
|
|
177
|
+
raise ArgumentError,
|
|
178
|
+
"Retrieval profile #{key.inspect}: rerank_candidates #{candidates} exceeds #{MAX_RERANK_CANDIDATES}."
|
|
179
|
+
end
|
|
180
|
+
top_n = opts[:rerank_top_n].nil? ? nil : positive_int!(key, :rerank_top_n, opts[:rerank_top_n])
|
|
181
|
+
if top_n && top_n > candidates
|
|
182
|
+
raise ArgumentError,
|
|
183
|
+
"Retrieval profile #{key.inspect}: rerank_top_n (#{top_n}) exceeds rerank_candidates (#{candidates})."
|
|
184
|
+
end
|
|
185
|
+
doc_chars = positive_int!(key, :rerank_max_document_chars, opts[:rerank_max_document_chars])
|
|
186
|
+
if doc_chars > MAX_RERANK_DOCUMENT_CHARS
|
|
187
|
+
raise ArgumentError,
|
|
188
|
+
"Retrieval profile #{key.inspect}: rerank_max_document_chars #{doc_chars} exceeds " \
|
|
189
|
+
"#{MAX_RERANK_DOCUMENT_CHARS}."
|
|
190
|
+
end
|
|
191
|
+
timeout = opts[:rerank_timeout]
|
|
192
|
+
unless timeout.is_a?(Numeric) && timeout.positive?
|
|
193
|
+
raise ArgumentError, "Retrieval profile #{key.inspect}: rerank_timeout must be a positive number."
|
|
194
|
+
end
|
|
195
|
+
failure = opts[:on_rerank_failure].to_s.to_sym
|
|
196
|
+
unless FAILURE_MODES.include?(failure)
|
|
197
|
+
raise ArgumentError,
|
|
198
|
+
"Retrieval profile #{key.inspect}: on_rerank_failure must be one of #{FAILURE_MODES.inspect}."
|
|
199
|
+
end
|
|
200
|
+
budget = opts[:max_total_tokens].nil? ? nil : positive_int!(key, :max_total_tokens, opts[:max_total_tokens])
|
|
201
|
+
|
|
202
|
+
Profile.new(
|
|
203
|
+
name: key, k: k, max_k: max_k, hybrid: normalize_hybrid!(key, opts[:hybrid]),
|
|
204
|
+
reranker: reranker, rerank_candidates: candidates, rerank_top_n: top_n,
|
|
205
|
+
rerank_max_document_chars: doc_chars, rerank_timeout: timeout,
|
|
206
|
+
on_rerank_failure: failure, max_total_tokens: budget,
|
|
207
|
+
).freeze
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def positive_int!(key, option, value)
|
|
211
|
+
unless value.is_a?(Integer) && value.positive?
|
|
212
|
+
raise ArgumentError,
|
|
213
|
+
"Retrieval profile #{key.inspect}: #{option} must be a positive Integer (got #{value.inspect})."
|
|
214
|
+
end
|
|
215
|
+
value
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def normalize_hybrid!(key, hybrid)
|
|
219
|
+
return nil if hybrid.nil? || hybrid == false
|
|
220
|
+
return {} if hybrid == true
|
|
221
|
+
unless hybrid.is_a?(Hash)
|
|
222
|
+
raise ArgumentError, "Retrieval profile #{key.inspect}: hybrid must be true or a Hash."
|
|
223
|
+
end
|
|
224
|
+
extra = hybrid.keys.map(&:to_sym) - HYBRID_KEYS
|
|
225
|
+
unless extra.empty?
|
|
226
|
+
raise ArgumentError,
|
|
227
|
+
"Retrieval profile #{key.inspect}: unknown hybrid option(s) #{extra.inspect}. " \
|
|
228
|
+
"Allowed: #{HYBRID_KEYS.inspect}."
|
|
229
|
+
end
|
|
230
|
+
Marshal.load(Marshal.dump(hybrid.transform_keys(&:to_sym))).freeze
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Wraps a registered reranker with a profile's budgets. Every document
|
|
236
|
+
# is cut to `rerank_max_document_chars`, the call is charged to the
|
|
237
|
+
# caller's spend budget through `charge`, and the provider call is
|
|
238
|
+
# bounded by `rerank_timeout`. On a timeout or provider failure it
|
|
239
|
+
# either returns nil (the retriever then keeps the retrieval order) and
|
|
240
|
+
# records the reason, or re-raises, per `on_rerank_failure`.
|
|
241
|
+
#
|
|
242
|
+
# The text it receives has already passed the `semantic_search` text
|
|
243
|
+
# source check, so it never contains a field outside the agent's
|
|
244
|
+
# effective `agent_fields`.
|
|
245
|
+
class BudgetedReranker
|
|
246
|
+
attr_reader :stats
|
|
247
|
+
|
|
248
|
+
# @param inner [#rerank] the registered reranker.
|
|
249
|
+
# @param profile [Profiles::Profile]
|
|
250
|
+
# @param charge [Proc, nil] `charge.call(tokens)`; may raise to refuse.
|
|
251
|
+
def initialize(inner, profile, charge: nil)
|
|
252
|
+
@inner = inner
|
|
253
|
+
@profile = profile
|
|
254
|
+
@charge = charge
|
|
255
|
+
@stats = { used: false, documents: 0, chars: 0, tokens_estimated: 0,
|
|
256
|
+
duration_ms: 0.0, fallback: false, fallback_reason: nil }
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
# Longest query sent to a reranker. The query is paired with every
|
|
260
|
+
# document in the provider call, so an unbounded query multiplies the
|
|
261
|
+
# cost of every rerank.
|
|
262
|
+
MAX_QUERY_CHARS = 2_000
|
|
263
|
+
|
|
264
|
+
def rerank(query:, documents:, top_n: nil)
|
|
265
|
+
limit = @profile.rerank_max_document_chars
|
|
266
|
+
query = query.to_s[0, MAX_QUERY_CHARS]
|
|
267
|
+
docs = Array(documents).map { |d| d.to_s[0, limit] }
|
|
268
|
+
tokens = Parse::Embeddings::SpendCap.estimate_tokens(query) * [docs.length, 1].max +
|
|
269
|
+
docs.sum { |d| Parse::Embeddings::SpendCap.estimate_tokens(d) }
|
|
270
|
+
@stats.merge!(used: true, documents: docs.length, chars: docs.sum(&:length), tokens_estimated: tokens)
|
|
271
|
+
@charge&.call(tokens)
|
|
272
|
+
|
|
273
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
274
|
+
begin
|
|
275
|
+
Timeout.timeout(@profile.rerank_timeout) do
|
|
276
|
+
@inner.rerank(query: query, documents: docs, top_n: top_n)
|
|
277
|
+
end
|
|
278
|
+
rescue Timeout::Error, StandardError => e
|
|
279
|
+
raise if @profile.on_rerank_failure == :raise
|
|
280
|
+
@stats[:fallback] = true
|
|
281
|
+
@stats[:fallback_reason] = e.is_a?(Timeout::Error) ? "timeout" : e.class.name
|
|
282
|
+
nil
|
|
283
|
+
ensure
|
|
284
|
+
@stats[:duration_ms] = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round(1)
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
@rerankers = {}
|
|
290
|
+
@rerankers_mutex = Mutex.new
|
|
291
|
+
|
|
292
|
+
class << self
|
|
293
|
+
# Register a reranker under a name that retrieval profiles reference.
|
|
294
|
+
# Profiles name rerankers instead of embedding them so provider
|
|
295
|
+
# objects (and their credentials) stay server-side.
|
|
296
|
+
#
|
|
297
|
+
# @param name [Symbol, String]
|
|
298
|
+
# @param reranker [#rerank]
|
|
299
|
+
def register_reranker(name, reranker)
|
|
300
|
+
unless reranker.respond_to?(:rerank)
|
|
301
|
+
raise ArgumentError, "Parse::Retrieval.register_reranker: #{reranker.class} does not respond to #rerank."
|
|
302
|
+
end
|
|
303
|
+
@rerankers_mutex.synchronize { @rerankers[name.to_s] = reranker }
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# @return [#rerank, nil]
|
|
307
|
+
def reranker(name)
|
|
308
|
+
@rerankers_mutex.synchronize { @rerankers[name.to_s] }
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
def reranker_registered?(name)
|
|
312
|
+
!reranker(name).nil?
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def reset_rerankers!
|
|
316
|
+
@rerankers_mutex.synchronize { @rerankers.clear }
|
|
317
|
+
end
|
|
318
|
+
end
|
|
319
|
+
end
|
|
320
|
+
end
|
|
@@ -342,7 +342,11 @@ module Parse
|
|
|
342
342
|
sym = field.to_sym
|
|
343
343
|
fmap = klass.respond_to?(:field_map) ? klass.field_map : {}
|
|
344
344
|
mapped = fmap[sym]
|
|
345
|
-
|
|
345
|
+
return mapped.to_s if mapped
|
|
346
|
+
# An exact declared server name (`title_exact`, `PublicText`) is kept
|
|
347
|
+
# as written, matching Parse::Agent::MetadataRegistry.wire_field_names.
|
|
348
|
+
return field.to_s if fmap.values.any? { |v| v.to_s == field.to_s }
|
|
349
|
+
sym.to_s.columnize
|
|
346
350
|
end
|
|
347
351
|
|
|
348
352
|
# @!visibility private
|
|
@@ -359,9 +363,14 @@ module Parse
|
|
|
359
363
|
# document's presentation text (the same `text_field` used for
|
|
360
364
|
# chunking). Index alignment between `documents` and `raw_hits` is
|
|
361
365
|
# preserved so the returned `index` maps back to the right hit.
|
|
366
|
+
#
|
|
367
|
+
# A reranker that returns nil (Parse::Retrieval::BudgetedReranker after
|
|
368
|
+
# a timeout or provider failure under `on_rerank_failure: :fallback`)
|
|
369
|
+
# keeps the retrieval order, trimmed to `top_n`.
|
|
362
370
|
def apply_rerank(reranker, query, raw_hits, text_wire, top_n)
|
|
363
371
|
documents = raw_hits.map { |doc| fetch_field(doc, text_wire, text_wire).to_s }
|
|
364
372
|
results = reranker.rerank(query: query, documents: documents, top_n: top_n)
|
|
373
|
+
return top_n ? raw_hits.first(top_n) : raw_hits if results.nil?
|
|
365
374
|
results.map do |r|
|
|
366
375
|
hit = raw_hits[r.index]
|
|
367
376
|
next nil if hit.nil?
|
data/lib/parse/retrieval.rb
CHANGED
|
@@ -67,7 +67,7 @@ module Parse
|
|
|
67
67
|
def plan
|
|
68
68
|
coll = collection_name
|
|
69
69
|
existing, available = fetch_existing_indexes(coll)
|
|
70
|
-
declared =
|
|
70
|
+
declared = all_declarations.map { |d| effective_declaration(d) }
|
|
71
71
|
|
|
72
72
|
existing_by_name = existing.each_with_object({}) do |idx, h|
|
|
73
73
|
name = (idx["name"] || idx[:name]).to_s
|
|
@@ -131,6 +131,7 @@ module Parse
|
|
|
131
131
|
# - `:wait_results` — Hash{name => :ready|:failed|:timeout} when
|
|
132
132
|
# `wait: true`; empty otherwise.
|
|
133
133
|
def apply!(update: false, drop: false, wait: false, timeout: 600)
|
|
134
|
+
load_atlas_search!
|
|
134
135
|
p = plan
|
|
135
136
|
coll = p[:collection]
|
|
136
137
|
wait_results = {}
|
|
@@ -189,6 +190,17 @@ module Parse
|
|
|
189
190
|
|
|
190
191
|
private
|
|
191
192
|
|
|
193
|
+
# Hand-written `mongo_search_index` declarations plus definitions
|
|
194
|
+
# generated by `vector_search_index` (computed now, from the model's
|
|
195
|
+
# current declarations).
|
|
196
|
+
def all_declarations
|
|
197
|
+
declared = @model_class.mongo_search_index_declarations.dup
|
|
198
|
+
if @model_class.respond_to?(:generated_vector_search_index_declarations)
|
|
199
|
+
declared.concat(@model_class.generated_vector_search_index_declarations)
|
|
200
|
+
end
|
|
201
|
+
declared
|
|
202
|
+
end
|
|
203
|
+
|
|
192
204
|
# Augment a `vectorSearch` declaration with the model's registered
|
|
193
205
|
# `agent_tenant_scope` field as a `type: "filter"` path when the
|
|
194
206
|
# declaration doesn't already carry it. Tenant-scoped retrieval
|
|
@@ -242,15 +254,21 @@ module Parse
|
|
|
242
254
|
# Search support) — the migrator degrades gracefully and treats
|
|
243
255
|
# the absence as "no indexes yet".
|
|
244
256
|
def fetch_existing_indexes(coll)
|
|
245
|
-
unless defined?(Parse::AtlasSearch::IndexManager)
|
|
246
|
-
return [[], false]
|
|
247
|
-
end
|
|
248
257
|
return [[], false] unless mongodb_enabled?
|
|
258
|
+
load_atlas_search!
|
|
249
259
|
[Parse::AtlasSearch::IndexManager.list_indexes(coll, force_refresh: true), true]
|
|
250
|
-
rescue Parse::AtlasSearch::NotAvailable
|
|
260
|
+
rescue StandardError # includes Parse::AtlasSearch::NotAvailable
|
|
251
261
|
[[], false]
|
|
252
262
|
end
|
|
253
263
|
|
|
264
|
+
# Parse::AtlasSearch is not loaded with the SDK. Load it here instead
|
|
265
|
+
# of testing `defined?(Parse::AtlasSearch...)`: the namespace can
|
|
266
|
+
# already exist (protected_paths.rb opens it early) without the
|
|
267
|
+
# IndexManager or the error classes the migrator needs.
|
|
268
|
+
def load_atlas_search!
|
|
269
|
+
require_relative "../atlas_search"
|
|
270
|
+
end
|
|
271
|
+
|
|
254
272
|
def mongodb_enabled?
|
|
255
273
|
defined?(Parse::MongoDB) &&
|
|
256
274
|
Parse::MongoDB.respond_to?(:enabled?) &&
|
data/lib/parse/schema.rb
CHANGED
|
@@ -53,7 +53,9 @@ module Parse
|
|
|
53
53
|
# Parse field type mappings to Ruby types
|
|
54
54
|
TYPE_MAP = {
|
|
55
55
|
"String" => :string,
|
|
56
|
-
|
|
56
|
+
# A Number column holds integral and fractional values; :number keeps
|
|
57
|
+
# integers as Integer and fractional values as Float.
|
|
58
|
+
"Number" => :number,
|
|
57
59
|
"Boolean" => :boolean,
|
|
58
60
|
"Date" => :date,
|
|
59
61
|
"File" => :file,
|
|
@@ -71,6 +73,7 @@ module Parse
|
|
|
71
73
|
string: "String",
|
|
72
74
|
integer: "Number",
|
|
73
75
|
float: "Number",
|
|
76
|
+
number: "Number",
|
|
74
77
|
boolean: "Boolean",
|
|
75
78
|
date: "Date",
|
|
76
79
|
file: "File",
|
|
@@ -83,8 +86,51 @@ module Parse
|
|
|
83
86
|
relation: "Relation",
|
|
84
87
|
bytes: "Bytes",
|
|
85
88
|
acl: "ACL",
|
|
89
|
+
# SDK-only property types with no native Parse column type. They are
|
|
90
|
+
# stored in the column type Parse Server actually holds them as.
|
|
91
|
+
vector: "Array",
|
|
92
|
+
timezone: "String",
|
|
93
|
+
time_zone: "String",
|
|
94
|
+
phone: "String",
|
|
95
|
+
email: "String",
|
|
86
96
|
}.freeze
|
|
87
97
|
|
|
98
|
+
# Resolve the local type of a model field for schema comparison and
|
|
99
|
+
# migration. `fields` has no entry for a `has_many through: :relation`
|
|
100
|
+
# association, so a field that only appears in `relations` is reported
|
|
101
|
+
# as `:relation`.
|
|
102
|
+
# @param model_class [Class] a Parse::Object subclass
|
|
103
|
+
# @param name [Symbol] the canonical field name (a `field_map` key)
|
|
104
|
+
# @return [Symbol, nil]
|
|
105
|
+
def self.local_field_type(model_class, name)
|
|
106
|
+
type = model_class.fields[name]
|
|
107
|
+
return type.to_sym unless type.nil?
|
|
108
|
+
return :relation if model_class.respond_to?(:relations) && model_class.relations.key?(name.to_sym)
|
|
109
|
+
nil
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Build the server column definition for a model field, including the
|
|
113
|
+
# `targetClass` a Pointer or Relation column requires.
|
|
114
|
+
# @param model_class [Class] a Parse::Object subclass
|
|
115
|
+
# @param name [Symbol] the canonical field name (a `field_map` key)
|
|
116
|
+
# @param wire [String, Symbol] the server column name
|
|
117
|
+
# @return [Hash] the column definition, e.g. `{ "type" => "Pointer", "targetClass" => "Author" }`
|
|
118
|
+
def self.field_definition_for(model_class, name, wire)
|
|
119
|
+
type = local_field_type(model_class, name) || :string
|
|
120
|
+
parse_type = REVERSE_TYPE_MAP[type] || "String"
|
|
121
|
+
definition = { "type" => parse_type }
|
|
122
|
+
case parse_type
|
|
123
|
+
when "Pointer"
|
|
124
|
+
target = model_class.references[wire.to_s] || model_class.references[wire.to_sym] ||
|
|
125
|
+
model_class.references[name]
|
|
126
|
+
definition["targetClass"] = target.to_s if target
|
|
127
|
+
when "Relation"
|
|
128
|
+
target = model_class.relations[name.to_sym] || model_class.relations[wire.to_sym]
|
|
129
|
+
definition["targetClass"] = target.to_s if target
|
|
130
|
+
end
|
|
131
|
+
definition
|
|
132
|
+
end
|
|
133
|
+
|
|
88
134
|
class << self
|
|
89
135
|
# Fetch all schemas from the Parse Server.
|
|
90
136
|
# @param client [Parse::Client] optional client to use
|
|
@@ -247,7 +293,7 @@ module Parse
|
|
|
247
293
|
@model_class.field_map.each do |name, wire|
|
|
248
294
|
next if core_field?(name)
|
|
249
295
|
next if server.include?(wire.to_s)
|
|
250
|
-
missing[name] = @model_class
|
|
296
|
+
missing[name] = Parse::Schema.local_field_type(@model_class, name)
|
|
251
297
|
end
|
|
252
298
|
missing
|
|
253
299
|
end
|
|
@@ -263,7 +309,7 @@ module Parse
|
|
|
263
309
|
server.each do |name, info|
|
|
264
310
|
# Skip core fields
|
|
265
311
|
next if %w[objectId createdAt updatedAt ACL].include?(name)
|
|
266
|
-
missing[name] = info[:type] unless local.include?(name) || local.include?(name.underscore
|
|
312
|
+
missing[name] = info[:type] unless local.include?(name) || local.include?(name.underscore)
|
|
267
313
|
end
|
|
268
314
|
missing
|
|
269
315
|
end
|
|
@@ -285,7 +331,7 @@ module Parse
|
|
|
285
331
|
mismatches = {}
|
|
286
332
|
@model_class.field_map.each do |name, wire|
|
|
287
333
|
next if core_field?(name)
|
|
288
|
-
local_type = @model_class
|
|
334
|
+
local_type = Parse::Schema.local_field_type(@model_class, name)
|
|
289
335
|
next if local_type.nil?
|
|
290
336
|
server_type = @server_schema.field_type(wire.to_s)
|
|
291
337
|
next unless server_type
|
|
@@ -359,8 +405,15 @@ module Parse
|
|
|
359
405
|
@model_class.fields.reject { |k, _| core_field?(k) }
|
|
360
406
|
end
|
|
361
407
|
|
|
408
|
+
# Every local name a server column may match: field keys plus the
|
|
409
|
+
# canonical and wire names in `field_map` (which also covers
|
|
410
|
+
# `has_many through: :relation` columns, absent from `fields`).
|
|
362
411
|
def local_field_names
|
|
363
|
-
local_fields.keys.map(&:to_s)
|
|
412
|
+
names = local_fields.keys.map(&:to_s)
|
|
413
|
+
@model_class.field_map.each do |name, wire|
|
|
414
|
+
names << name.to_s << wire.to_s
|
|
415
|
+
end
|
|
416
|
+
names.uniq
|
|
364
417
|
end
|
|
365
418
|
|
|
366
419
|
def server_field_names
|
|
@@ -371,10 +424,16 @@ module Parse
|
|
|
371
424
|
%i[id object_id created_at updated_at acl objectId createdAt updatedAt ACL].include?(name.to_sym)
|
|
372
425
|
end
|
|
373
426
|
|
|
427
|
+
# Types that compare equal to the server column they are stored in:
|
|
428
|
+
# a `:vector` is an Array column, and `:timezone`, `:phone`, and
|
|
429
|
+
# `:email` are String columns.
|
|
374
430
|
def normalize_type(type)
|
|
375
431
|
case type.to_sym
|
|
376
432
|
when :integer, :float, :number then :number
|
|
377
433
|
when :geo_point then :geopoint
|
|
434
|
+
when :geo_polygon then :polygon
|
|
435
|
+
when :vector then :array
|
|
436
|
+
when :timezone, :time_zone, :phone, :email then :string
|
|
378
437
|
else type.to_sym
|
|
379
438
|
end
|
|
380
439
|
end
|
|
@@ -414,11 +473,11 @@ module Parse
|
|
|
414
473
|
end
|
|
415
474
|
|
|
416
475
|
@diff.missing_on_server.each do |name, type|
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
476
|
+
wire = @model_class.field_map[name]
|
|
477
|
+
definition = Parse::Schema.field_definition_for(@model_class, name, wire)
|
|
478
|
+
op = { action: :add_field, field: wire.to_s, type: definition["type"] }
|
|
479
|
+
op[:target_class] = definition["targetClass"] if definition["targetClass"]
|
|
480
|
+
ops << op
|
|
422
481
|
end
|
|
423
482
|
|
|
424
483
|
ops
|
|
@@ -435,7 +494,8 @@ module Parse
|
|
|
435
494
|
when :create_class
|
|
436
495
|
lines << " CREATE CLASS #{op[:class_name]}"
|
|
437
496
|
when :add_field
|
|
438
|
-
|
|
497
|
+
target = op[:target_class] ? " -> #{op[:target_class]}" : ""
|
|
498
|
+
lines << " ADD FIELD #{op[:field]} (#{op[:type]}#{target})"
|
|
439
499
|
end
|
|
440
500
|
end
|
|
441
501
|
lines.join("\n")
|
|
@@ -469,7 +529,8 @@ module Parse
|
|
|
469
529
|
# Add missing fields
|
|
470
530
|
@diff.missing_on_server.each do |name, type|
|
|
471
531
|
field_name = @model_class.field_map[name].to_s
|
|
472
|
-
|
|
532
|
+
definition = Parse::Schema.field_definition_for(@model_class, name, field_name)
|
|
533
|
+
field_schema = { "fields" => { field_name => definition } }
|
|
473
534
|
|
|
474
535
|
response = @client.update_schema(@model_class.parse_class, field_schema)
|
|
475
536
|
if response.success?
|
|
@@ -494,7 +555,7 @@ module Parse
|
|
|
494
555
|
# multi-word or custom-`field:` property.
|
|
495
556
|
@model_class.field_map.each do |name, wire|
|
|
496
557
|
next if %i[id object_id created_at updated_at acl objectId createdAt updatedAt ACL].include?(name)
|
|
497
|
-
fields[wire.to_s] =
|
|
558
|
+
fields[wire.to_s] = Parse::Schema.field_definition_for(@model_class, name, wire)
|
|
498
559
|
end
|
|
499
560
|
|
|
500
561
|
# Add pointer targets. `references` is keyed by the wire column name
|
|
@@ -533,11 +594,6 @@ module Parse
|
|
|
533
594
|
end
|
|
534
595
|
Parse::Schema.default_class_level_permissions
|
|
535
596
|
end
|
|
536
|
-
|
|
537
|
-
def field_definition(type)
|
|
538
|
-
parse_type = REVERSE_TYPE_MAP[type.to_sym] || "String"
|
|
539
|
-
{ "type" => parse_type }
|
|
540
|
-
end
|
|
541
597
|
end
|
|
542
598
|
end
|
|
543
599
|
end
|
data/lib/parse/stack/tasks.rb
CHANGED
|
@@ -21,8 +21,10 @@ module Parse
|
|
|
21
21
|
|
|
22
22
|
# Installs the rake tasks.
|
|
23
23
|
def install_tasks
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
# `Rails` alone is not enough: gems such as rails-html-sanitizer
|
|
25
|
+
# define a bare `Rails` module with no `root` or `application`.
|
|
26
|
+
if defined?(::Rails) && ::Rails.respond_to?(:root)
|
|
27
|
+
unless Rake::Task.task_defined?("db:seed") || ::Rails.root.blank?
|
|
26
28
|
namespace :db do
|
|
27
29
|
desc "Seeds your database with by loading db/seeds.rb"
|
|
28
30
|
task :seed => "parse:env" do
|
|
@@ -36,8 +38,8 @@ module Parse
|
|
|
36
38
|
task :env do
|
|
37
39
|
if Rake::Task.task_defined?("environment")
|
|
38
40
|
Rake::Task["environment"].invoke
|
|
39
|
-
if defined?(::Rails)
|
|
40
|
-
Rails.application.eager_load! if Rails.application.present?
|
|
41
|
+
if defined?(::Rails) && ::Rails.respond_to?(:application)
|
|
42
|
+
::Rails.application.eager_load! if ::Rails.application.present?
|
|
41
43
|
end
|
|
42
44
|
end
|
|
43
45
|
end
|