parse-stack-next 5.6.0 → 5.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +409 -0
- data/README.md +378 -6
- data/docs/caching.md +748 -0
- data/examples/basic_client.rb +3 -3
- data/examples/basic_server.rb +3 -3
- data/examples/live_query_listener.rb +12 -12
- data/examples/rag_chatbot.rb +14 -14
- data/examples/transaction_example.rb +44 -45
- data/examples/webhook_server.rb +3 -3
- data/lib/parse/access.rb +493 -0
- data/lib/parse/acl_scope.rb +125 -52
- data/lib/parse/agent/approval_gate.rb +0 -0
- data/lib/parse/agent/cancellation_token.rb +3 -3
- data/lib/parse/agent/constraint_translator.rb +4 -4
- data/lib/parse/agent/describe.rb +34 -34
- data/lib/parse/agent/errors.rb +9 -9
- data/lib/parse/agent/mcp_client.rb +61 -59
- data/lib/parse/agent/mcp_dispatcher.rb +89 -101
- data/lib/parse/agent/mcp_rack_app.rb +92 -93
- data/lib/parse/agent/mcp_server.rb +1 -1
- data/lib/parse/agent/mcp_subscriptions.rb +22 -22
- data/lib/parse/agent/metadata_audit.rb +1 -2
- data/lib/parse/agent/metadata_dsl.rb +7 -7
- data/lib/parse/agent/metadata_registry.rb +12 -12
- data/lib/parse/agent/prompt_hardening.rb +4 -4
- data/lib/parse/agent/prompts.rb +10 -10
- data/lib/parse/agent/result_formatter.rb +4 -4
- data/lib/parse/agent/tools.rb +493 -400
- data/lib/parse/agent.rb +157 -116
- data/lib/parse/api/hooks.rb +1 -1
- data/lib/parse/api/server.rb +2 -2
- data/lib/parse/api/users.rb +2 -2
- data/lib/parse/atlas_search/index_manager.rb +1 -1
- data/lib/parse/atlas_search/session.rb +40 -218
- data/lib/parse/atlas_search.rb +181 -60
- data/lib/parse/authorization.rb +466 -0
- data/lib/parse/cache/invalidation.rb +167 -0
- data/lib/parse/cache/keyspace.rb +306 -0
- data/lib/parse/cache/moneta_surface.rb +126 -0
- data/lib/parse/cache/pool.rb +48 -5
- data/lib/parse/cache/redis.rb +425 -10
- data/lib/parse/cache/scoped_view.rb +443 -0
- data/lib/parse/cache/sub_cache.rb +264 -0
- data/lib/parse/cache/upstream_roles.rb +230 -0
- data/lib/parse/client/authentication.rb +1 -1
- data/lib/parse/client/body_builder.rb +1 -2
- data/lib/parse/client/caching.rb +84 -5
- data/lib/parse/client.rb +327 -189
- data/lib/parse/clp_scope.rb +225 -28
- data/lib/parse/console.rb +3 -3
- data/lib/parse/embeddings/batch_embedder.rb +3 -3
- data/lib/parse/embeddings/cache.rb +17 -17
- data/lib/parse/embeddings/cohere.rb +21 -22
- data/lib/parse/embeddings/image_fetch.rb +22 -21
- data/lib/parse/embeddings/jina.rb +21 -21
- data/lib/parse/embeddings/local_http.rb +6 -7
- data/lib/parse/embeddings/media_file.rb +5 -4
- data/lib/parse/embeddings/openai.rb +5 -5
- data/lib/parse/embeddings/provider.rb +1 -1
- data/lib/parse/embeddings/qwen.rb +10 -10
- data/lib/parse/embeddings/spend_cap.rb +2 -3
- data/lib/parse/embeddings/streaming_body.rb +11 -11
- data/lib/parse/embeddings/video_source.rb +7 -6
- data/lib/parse/embeddings/voyage.rb +59 -61
- data/lib/parse/embeddings.rb +28 -27
- data/lib/parse/graphql/type_generator.rb +1 -1
- data/lib/parse/graphql.rb +5 -5
- data/lib/parse/live_query/client.rb +1 -1
- data/lib/parse/live_query.rb +1 -1
- data/lib/parse/lock.rb +15 -18
- data/lib/parse/lock_backend.rb +1 -1
- data/lib/parse/lookup_rewriter.rb +6 -6
- data/lib/parse/model/classes/role.rb +227 -28
- data/lib/parse/model/classes/user.rb +60 -1
- data/lib/parse/model/clp.rb +4 -4
- data/lib/parse/model/core/actions.rb +97 -15
- data/lib/parse/model/core/create_lock.rb +0 -2
- data/lib/parse/model/core/describe.rb +57 -57
- data/lib/parse/model/core/embed_managed.rb +21 -23
- data/lib/parse/model/core/fetching.rb +2 -0
- data/lib/parse/model/core/indexing.rb +14 -14
- data/lib/parse/model/core/parse_reference.rb +1 -1
- data/lib/parse/model/core/properties.rb +2 -2
- data/lib/parse/model/core/querying.rb +1 -1
- data/lib/parse/model/core/schema.rb +2 -2
- data/lib/parse/model/core/search_indexing.rb +2 -2
- data/lib/parse/model/core/vector_searchable.rb +4 -5
- data/lib/parse/model/file.rb +21 -15
- data/lib/parse/model/geojson.rb +2 -2
- data/lib/parse/model/geopoint.rb +1 -0
- data/lib/parse/model/object.rb +132 -66
- data/lib/parse/model/pointer.rb +2 -0
- data/lib/parse/model/polygon.rb +3 -6
- data/lib/parse/model/push.rb +2 -2
- data/lib/parse/model/vector.rb +3 -1
- data/lib/parse/mongodb.rb +512 -282
- data/lib/parse/pipeline_security.rb +3 -0
- data/lib/parse/query/constraints.rb +46 -46
- data/lib/parse/query.rb +146 -59
- data/lib/parse/retrieval/agent_tool.rb +34 -20
- data/lib/parse/retrieval/chunk.rb +1 -0
- data/lib/parse/retrieval/reranker/cohere.rb +11 -11
- data/lib/parse/retrieval/reranker.rb +3 -4
- data/lib/parse/retrieval/retriever.rb +4 -6
- data/lib/parse/schema/index_migrator.rb +26 -26
- data/lib/parse/schema/search_index_migrator.rb +19 -19
- data/lib/parse/stack/tasks.rb +6 -6
- data/lib/parse/stack/version.rb +1 -1
- data/lib/parse/stack.rb +10 -10
- data/lib/parse/vector_search/hybrid.rb +66 -41
- data/lib/parse/vector_search.rb +14 -11
- data/lib/parse/webhooks/payload.rb +1 -0
- data/lib/parse/webhooks/registration.rb +4 -4
- data/lib/parse/webhooks/trigger_audit.rb +50 -49
- data/lib/parse/webhooks.rb +22 -4
- data/parse-stack-next.gemspec +6 -6
- metadata +10 -1
|
@@ -63,36 +63,36 @@ module Parse
|
|
|
63
63
|
class RateLimitError < Error; end
|
|
64
64
|
class TransientError < Error; end
|
|
65
65
|
|
|
66
|
-
DEFAULT_BASE_URL
|
|
67
|
-
DEFAULT_MODEL
|
|
68
|
-
DEFAULT_TIMEOUT
|
|
66
|
+
DEFAULT_BASE_URL = "https://api.jina.ai/v1"
|
|
67
|
+
DEFAULT_MODEL = "jina-embeddings-v3"
|
|
68
|
+
DEFAULT_TIMEOUT = 30
|
|
69
69
|
DEFAULT_OPEN_TIMEOUT = 5
|
|
70
70
|
DEFAULT_MAX_RETRIES = 3
|
|
71
|
-
DEFAULT_BATCH_SIZE
|
|
72
|
-
MAX_RESPONSE_BYTES
|
|
71
|
+
DEFAULT_BATCH_SIZE = 100
|
|
72
|
+
MAX_RESPONSE_BYTES = 16 * 1024 * 1024
|
|
73
73
|
|
|
74
74
|
# Native vector widths. The Matryoshka-capable rows allow the
|
|
75
75
|
# caller to truncate via the `dimensions:` kwarg.
|
|
76
76
|
MODEL_DEFAULT_DIMENSIONS = {
|
|
77
77
|
"jina-embeddings-v5-omni-small" => 1024,
|
|
78
|
-
"jina-embeddings-v5-omni-nano"
|
|
78
|
+
"jina-embeddings-v5-omni-nano" => 512,
|
|
79
79
|
"jina-embeddings-v5-text-small" => 1024,
|
|
80
|
-
"jina-embeddings-v5-text-nano"
|
|
81
|
-
"jina-embeddings-v4"
|
|
82
|
-
"jina-embeddings-v3"
|
|
83
|
-
"jina-code-embeddings-1.5b"
|
|
84
|
-
"jina-code-embeddings-0.5b"
|
|
80
|
+
"jina-embeddings-v5-text-nano" => 512,
|
|
81
|
+
"jina-embeddings-v4" => 2048,
|
|
82
|
+
"jina-embeddings-v3" => 1024,
|
|
83
|
+
"jina-code-embeddings-1.5b" => 1024,
|
|
84
|
+
"jina-code-embeddings-0.5b" => 1024,
|
|
85
85
|
}.freeze
|
|
86
86
|
|
|
87
87
|
MODEL_MAX_INPUT_TOKENS = {
|
|
88
88
|
"jina-embeddings-v5-omni-small" => 32_000,
|
|
89
|
-
"jina-embeddings-v5-omni-nano"
|
|
89
|
+
"jina-embeddings-v5-omni-nano" => 32_000,
|
|
90
90
|
"jina-embeddings-v5-text-small" => 32_000,
|
|
91
|
-
"jina-embeddings-v5-text-nano"
|
|
92
|
-
"jina-embeddings-v4"
|
|
93
|
-
"jina-embeddings-v3"
|
|
94
|
-
"jina-code-embeddings-1.5b"
|
|
95
|
-
"jina-code-embeddings-0.5b"
|
|
91
|
+
"jina-embeddings-v5-text-nano" => 32_000,
|
|
92
|
+
"jina-embeddings-v4" => 32_000,
|
|
93
|
+
"jina-embeddings-v3" => 8_192,
|
|
94
|
+
"jina-code-embeddings-1.5b" => 32_000,
|
|
95
|
+
"jina-code-embeddings-0.5b" => 32_000,
|
|
96
96
|
}.freeze
|
|
97
97
|
|
|
98
98
|
# Models that accept the Matryoshka `dimensions` field. Other
|
|
@@ -108,10 +108,10 @@ module Parse
|
|
|
108
108
|
|
|
109
109
|
# Map SDK-canonical input_type symbols to Jina `task` strings.
|
|
110
110
|
INPUT_TYPE_WIRE_VALUES = {
|
|
111
|
-
search_query:
|
|
111
|
+
search_query: "retrieval.query",
|
|
112
112
|
search_document: "retrieval.passage",
|
|
113
|
-
classification:
|
|
114
|
-
clustering:
|
|
113
|
+
classification: "classification",
|
|
114
|
+
clustering: "separation",
|
|
115
115
|
}.freeze
|
|
116
116
|
|
|
117
117
|
# @param api_key [String] required. Sent as `Authorization: Bearer …`.
|
|
@@ -352,7 +352,7 @@ module Parse
|
|
|
352
352
|
end
|
|
353
353
|
|
|
354
354
|
def backoff_seconds(attempt)
|
|
355
|
-
[0.5 * (2**(attempt - 1)), 30.0].min
|
|
355
|
+
[0.5 * (2 ** (attempt - 1)), 30.0].min
|
|
356
356
|
end
|
|
357
357
|
|
|
358
358
|
def retry_after_seconds(response)
|
|
@@ -78,11 +78,11 @@ module Parse
|
|
|
78
78
|
class RateLimitError < Error; end
|
|
79
79
|
class TransientError < Error; end
|
|
80
80
|
|
|
81
|
-
DEFAULT_TIMEOUT
|
|
81
|
+
DEFAULT_TIMEOUT = 30
|
|
82
82
|
DEFAULT_OPEN_TIMEOUT = 5
|
|
83
|
-
DEFAULT_MAX_RETRIES
|
|
84
|
-
DEFAULT_BATCH_SIZE
|
|
85
|
-
MAX_RESPONSE_BYTES
|
|
83
|
+
DEFAULT_MAX_RETRIES = 3
|
|
84
|
+
DEFAULT_BATCH_SIZE = 32
|
|
85
|
+
MAX_RESPONSE_BYTES = 16 * 1024 * 1024
|
|
86
86
|
|
|
87
87
|
# @param base_url [String] required. Must be http(s):// with a host.
|
|
88
88
|
# @param model [String] required. Identifier the local server expects
|
|
@@ -353,7 +353,7 @@ module Parse
|
|
|
353
353
|
end
|
|
354
354
|
|
|
355
355
|
def backoff_seconds(attempt)
|
|
356
|
-
[0.5 * (2**(attempt - 1)), 30.0].min
|
|
356
|
+
[0.5 * (2 ** (attempt - 1)), 30.0].min
|
|
357
357
|
end
|
|
358
358
|
|
|
359
359
|
def retry_after_seconds(response)
|
|
@@ -414,8 +414,7 @@ module Parse
|
|
|
414
414
|
# Empty-resolution under allow_private_endpoint is treated as
|
|
415
415
|
# private for the http:// scheme gate below, since the operator
|
|
416
416
|
# has already asserted local-class trust.
|
|
417
|
-
is_private =
|
|
418
|
-
if resolved.empty?
|
|
417
|
+
is_private = if resolved.empty?
|
|
419
418
|
allow_private_endpoint
|
|
420
419
|
else
|
|
421
420
|
resolved.any? { |ip| Parse::File::BLOCKED_CIDRS.any? { |cidr| cidr.include?(ip) } }
|
|
@@ -55,14 +55,14 @@ module Parse
|
|
|
55
55
|
mime = ImageFetch.sniff_mime(header)
|
|
56
56
|
if mime.nil?
|
|
57
57
|
raise ImageFetch::InvalidImageType.new(:unknown_magic,
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
"Parse::Embeddings::MediaFile.image: #{path} matches no supported image " \
|
|
59
|
+
"format (JPEG/PNG/GIF/WebP).")
|
|
60
60
|
end
|
|
61
61
|
allowed = Parse::Embeddings.allowed_image_types
|
|
62
62
|
unless allowed.include?(mime)
|
|
63
63
|
raise ImageFetch::InvalidImageType.new(:type_not_allowed,
|
|
64
|
-
|
|
65
|
-
|
|
64
|
+
"Parse::Embeddings::MediaFile.image: sniffed type #{mime.inspect} is not in " \
|
|
65
|
+
"Parse::Embeddings.allowed_image_types (#{allowed.inspect}).")
|
|
66
66
|
end
|
|
67
67
|
new(path: path, mime_type: mime, kind: :image)
|
|
68
68
|
end
|
|
@@ -130,6 +130,7 @@ module Parse
|
|
|
130
130
|
"#<Parse::Embeddings::MediaFile kind=#{@kind} mime_type=#{@mime_type.inspect} " \
|
|
131
131
|
"bytes=#{@byte_size} path=#{@path.inspect}>"
|
|
132
132
|
end
|
|
133
|
+
|
|
133
134
|
alias_method :to_s, :inspect
|
|
134
135
|
end
|
|
135
136
|
end
|
|
@@ -49,12 +49,12 @@ module Parse
|
|
|
49
49
|
class RateLimitError < Error; end
|
|
50
50
|
class TransientError < Error; end
|
|
51
51
|
|
|
52
|
-
DEFAULT_BASE_URL
|
|
53
|
-
DEFAULT_MODEL
|
|
54
|
-
DEFAULT_TIMEOUT
|
|
52
|
+
DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
|
53
|
+
DEFAULT_MODEL = "text-embedding-3-small"
|
|
54
|
+
DEFAULT_TIMEOUT = 30
|
|
55
55
|
DEFAULT_OPEN_TIMEOUT = 5
|
|
56
56
|
DEFAULT_MAX_RETRIES = 3
|
|
57
|
-
DEFAULT_BATCH_SIZE
|
|
57
|
+
DEFAULT_BATCH_SIZE = 100
|
|
58
58
|
|
|
59
59
|
# Hard ceiling on the response body we'll parse. A legitimate
|
|
60
60
|
# OpenAI embeddings response for the worst-case configuration
|
|
@@ -397,7 +397,7 @@ module Parse
|
|
|
397
397
|
# synchronize the retry storm exponentially.
|
|
398
398
|
def backoff_seconds(attempt)
|
|
399
399
|
# 0.5, 1.0, 2.0, 4.0, 8.0 … capped at 30s
|
|
400
|
-
[0.5 * (2**(attempt - 1)), 30.0].min
|
|
400
|
+
[0.5 * (2 ** (attempt - 1)), 30.0].min
|
|
401
401
|
end
|
|
402
402
|
|
|
403
403
|
def retry_after_seconds(response)
|
|
@@ -60,27 +60,27 @@ module Parse
|
|
|
60
60
|
# Default to the international compatible-mode host. Operators
|
|
61
61
|
# in mainland China should override to
|
|
62
62
|
# `https://dashscope.aliyuncs.com/compatible-mode/v1`.
|
|
63
|
-
DEFAULT_BASE_URL
|
|
64
|
-
DEFAULT_MODEL
|
|
65
|
-
DEFAULT_TIMEOUT
|
|
63
|
+
DEFAULT_BASE_URL = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
|
|
64
|
+
DEFAULT_MODEL = "qwen3-embedding-8b"
|
|
65
|
+
DEFAULT_TIMEOUT = 30
|
|
66
66
|
DEFAULT_OPEN_TIMEOUT = 5
|
|
67
67
|
DEFAULT_MAX_RETRIES = 3
|
|
68
68
|
# DashScope's compatible endpoint caps embedding requests at 25
|
|
69
69
|
# inputs per call (smaller than OpenAI's 2048). Default below
|
|
70
70
|
# the cap so callers don't have to tune.
|
|
71
|
-
DEFAULT_BATCH_SIZE
|
|
72
|
-
MAX_RESPONSE_BYTES
|
|
71
|
+
DEFAULT_BATCH_SIZE = 10
|
|
72
|
+
MAX_RESPONSE_BYTES = 16 * 1024 * 1024
|
|
73
73
|
|
|
74
74
|
MODEL_DEFAULT_DIMENSIONS = {
|
|
75
75
|
"qwen3-embedding-0.6b" => 1024,
|
|
76
|
-
"qwen3-embedding-4b"
|
|
77
|
-
"qwen3-embedding-8b"
|
|
76
|
+
"qwen3-embedding-4b" => 2560,
|
|
77
|
+
"qwen3-embedding-8b" => 4096,
|
|
78
78
|
}.freeze
|
|
79
79
|
|
|
80
80
|
MODEL_MAX_INPUT_TOKENS = {
|
|
81
81
|
"qwen3-embedding-0.6b" => 32_000,
|
|
82
|
-
"qwen3-embedding-4b"
|
|
83
|
-
"qwen3-embedding-8b"
|
|
82
|
+
"qwen3-embedding-4b" => 32_000,
|
|
83
|
+
"qwen3-embedding-8b" => 32_000,
|
|
84
84
|
}.freeze
|
|
85
85
|
|
|
86
86
|
# Every Qwen3-Embedding row is Matryoshka-capable. Kept as an
|
|
@@ -329,7 +329,7 @@ module Parse
|
|
|
329
329
|
end
|
|
330
330
|
|
|
331
331
|
def backoff_seconds(attempt)
|
|
332
|
-
[0.5 * (2**(attempt - 1)), 30.0].min
|
|
332
|
+
[0.5 * (2 ** (attempt - 1)), 30.0].min
|
|
333
333
|
end
|
|
334
334
|
|
|
335
335
|
def retry_after_seconds(response)
|
|
@@ -118,8 +118,7 @@ module Parse
|
|
|
118
118
|
end
|
|
119
119
|
end
|
|
120
120
|
mutex.synchronize do
|
|
121
|
-
limits[key] =
|
|
122
|
-
if limit_tokens.nil?
|
|
121
|
+
limits[key] = if limit_tokens.nil?
|
|
123
122
|
nil
|
|
124
123
|
else
|
|
125
124
|
cfg = { limit: Integer(limit_tokens), window: w }
|
|
@@ -357,7 +356,7 @@ module Parse
|
|
|
357
356
|
# must not serialize every other tenant's charge).
|
|
358
357
|
def emit_soft_cap_warning(payload)
|
|
359
358
|
return unless defined?(ActiveSupport::Notifications)
|
|
360
|
-
ActiveSupport::Notifications.instrument(AS_NOTIFICATION_NAME, payload) {}
|
|
359
|
+
ActiveSupport::Notifications.instrument(AS_NOTIFICATION_NAME, payload) { }
|
|
361
360
|
rescue StandardError
|
|
362
361
|
# A raising subscriber must not turn a successful (admitted)
|
|
363
362
|
# charge into a caller-visible failure.
|
|
@@ -69,6 +69,7 @@ module Parse
|
|
|
69
69
|
seg.is_a?(String) ? seg.bytesize : 4 * ((seg[:size] + 2) / 3)
|
|
70
70
|
end
|
|
71
71
|
end
|
|
72
|
+
|
|
72
73
|
alias_method :length, :size
|
|
73
74
|
|
|
74
75
|
# @param len [Integer, nil] bytes wanted; nil reads to the end
|
|
@@ -83,8 +84,7 @@ module Parse
|
|
|
83
84
|
return len.nil? ? "" : nil
|
|
84
85
|
end
|
|
85
86
|
|
|
86
|
-
chunk =
|
|
87
|
-
if len.nil?
|
|
87
|
+
chunk = if len.nil?
|
|
88
88
|
b = @buffer
|
|
89
89
|
@buffer = +""
|
|
90
90
|
b
|
|
@@ -137,16 +137,16 @@ module Parse
|
|
|
137
137
|
end
|
|
138
138
|
|
|
139
139
|
@io ||= begin
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
140
|
+
f = ::File.open(seg[:path], "rb")
|
|
141
|
+
actual = f.size
|
|
142
|
+
if actual != seg[:size]
|
|
143
|
+
f.close
|
|
144
|
+
raise SizeMismatch,
|
|
145
|
+
"Parse::Embeddings::StreamingBody: #{seg[:path]} is #{actual} bytes but " \
|
|
146
|
+
"#{seg[:size]} was declared; the file changed underneath the request."
|
|
147
|
+
end
|
|
148
|
+
f
|
|
147
149
|
end
|
|
148
|
-
f
|
|
149
|
-
end
|
|
150
150
|
|
|
151
151
|
data = @io.read(READ_CHUNK)
|
|
152
152
|
if data.nil? || data.empty?
|
|
@@ -31,6 +31,7 @@ module Parse
|
|
|
31
31
|
class InvalidVideoType < Parse::Embeddings::Error
|
|
32
32
|
# @return [Symbol] failure-mode tag.
|
|
33
33
|
attr_reader :reason
|
|
34
|
+
|
|
34
35
|
def initialize(reason, message)
|
|
35
36
|
@reason = reason
|
|
36
37
|
super(message)
|
|
@@ -98,20 +99,20 @@ module Parse
|
|
|
98
99
|
def verify!(bytes)
|
|
99
100
|
if bytes.nil? || bytes.empty?
|
|
100
101
|
raise InvalidVideoType.new(:empty,
|
|
101
|
-
|
|
102
|
+
"Parse::Embeddings::VideoSource: video payload is empty.")
|
|
102
103
|
end
|
|
103
104
|
mime = sniff_mime(bytes)
|
|
104
105
|
if mime.nil?
|
|
105
106
|
raise InvalidVideoType.new(:unknown_magic,
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
107
|
+
"Parse::Embeddings::VideoSource: leading bytes match no supported video " \
|
|
108
|
+
"container (MP4/QuickTime/WebM). The Content-Type header is not consulted — " \
|
|
109
|
+
"unrecognized content is refused outright.")
|
|
109
110
|
end
|
|
110
111
|
allowed = Parse::Embeddings.allowed_video_types
|
|
111
112
|
unless allowed.include?(mime)
|
|
112
113
|
raise InvalidVideoType.new(:type_not_allowed,
|
|
113
|
-
|
|
114
|
-
|
|
114
|
+
"Parse::Embeddings::VideoSource: sniffed type #{mime.inspect} is not in " \
|
|
115
|
+
"Parse::Embeddings.allowed_video_types (#{allowed.inspect}).")
|
|
115
116
|
end
|
|
116
117
|
mime
|
|
117
118
|
end
|
|
@@ -109,23 +109,23 @@ module Parse
|
|
|
109
109
|
class RateLimitError < Error; end
|
|
110
110
|
class TransientError < Error; end
|
|
111
111
|
|
|
112
|
-
DEFAULT_BASE_URL
|
|
112
|
+
DEFAULT_BASE_URL = "https://api.voyageai.com/v1"
|
|
113
113
|
# MongoDB's Atlas Embedding and Reranking API re-exposes the same
|
|
114
114
|
# Voyage models under a MongoDB-operated host. The wire contract
|
|
115
115
|
# (request envelopes, response envelopes, error shapes) is
|
|
116
116
|
# identical — only the host and the credential differ.
|
|
117
|
-
ATLAS_BASE_URL
|
|
117
|
+
ATLAS_BASE_URL = "https://ai.mongodb.com/v1"
|
|
118
118
|
# Bumped from `voyage-3` in 5.6.0: that model is retired from the
|
|
119
119
|
# Atlas endpoint, so an Atlas key used without naming a model
|
|
120
120
|
# failed at construction. `voyage-3.5` is served by both
|
|
121
121
|
# endpoints and shares the 1024 native width.
|
|
122
|
-
DEFAULT_MODEL
|
|
123
|
-
DEFAULT_TIMEOUT
|
|
122
|
+
DEFAULT_MODEL = "voyage-3.5"
|
|
123
|
+
DEFAULT_TIMEOUT = 30
|
|
124
124
|
DEFAULT_OPEN_TIMEOUT = 5
|
|
125
125
|
DEFAULT_MAX_RETRIES = 3
|
|
126
126
|
# Voyage's documented per-request cap is 128 inputs.
|
|
127
|
-
DEFAULT_BATCH_SIZE
|
|
128
|
-
MAX_RESPONSE_BYTES
|
|
127
|
+
DEFAULT_BATCH_SIZE = 128
|
|
128
|
+
MAX_RESPONSE_BYTES = 16 * 1024 * 1024
|
|
129
129
|
|
|
130
130
|
# Default (native) vector width per model — the width returned
|
|
131
131
|
# when `output_dimension` is omitted from the request.
|
|
@@ -138,20 +138,20 @@ module Parse
|
|
|
138
138
|
# Verified against the live API for every model reachable
|
|
139
139
|
# through {ATLAS_BASE_URL}; see {MODEL_SUPPORTED_DIMENSIONS}.
|
|
140
140
|
MODEL_DEFAULT_DIMENSIONS = {
|
|
141
|
-
"voyage-4-large"
|
|
142
|
-
"voyage-4"
|
|
143
|
-
"voyage-4-lite"
|
|
144
|
-
"voyage-4-nano"
|
|
145
|
-
"voyage-3-large"
|
|
146
|
-
"voyage-3.5"
|
|
147
|
-
"voyage-3.5-lite"
|
|
148
|
-
"voyage-3"
|
|
149
|
-
"voyage-3-lite"
|
|
150
|
-
"voyage-code-3"
|
|
151
|
-
"voyage-code-2"
|
|
152
|
-
"voyage-finance-2"
|
|
153
|
-
"voyage-law-2"
|
|
154
|
-
"voyage-multimodal-3"
|
|
141
|
+
"voyage-4-large" => 1024,
|
|
142
|
+
"voyage-4" => 1024,
|
|
143
|
+
"voyage-4-lite" => 1024,
|
|
144
|
+
"voyage-4-nano" => 1024,
|
|
145
|
+
"voyage-3-large" => 1024,
|
|
146
|
+
"voyage-3.5" => 1024,
|
|
147
|
+
"voyage-3.5-lite" => 1024,
|
|
148
|
+
"voyage-3" => 1024,
|
|
149
|
+
"voyage-3-lite" => 512,
|
|
150
|
+
"voyage-code-3" => 1024,
|
|
151
|
+
"voyage-code-2" => 1536,
|
|
152
|
+
"voyage-finance-2" => 1024,
|
|
153
|
+
"voyage-law-2" => 1024,
|
|
154
|
+
"voyage-multimodal-3" => 1024,
|
|
155
155
|
"voyage-multimodal-3.5" => 1024,
|
|
156
156
|
}.freeze
|
|
157
157
|
|
|
@@ -165,20 +165,20 @@ module Parse
|
|
|
165
165
|
# 256/512/1024/2048 ladder, and `voyage-multimodal-3.5` accepts
|
|
166
166
|
# it too while `voyage-multimodal-3` does not.
|
|
167
167
|
MODEL_SUPPORTED_DIMENSIONS = {
|
|
168
|
-
"voyage-4-large"
|
|
169
|
-
"voyage-4"
|
|
170
|
-
"voyage-4-lite"
|
|
171
|
-
"voyage-4-nano"
|
|
172
|
-
"voyage-3-large"
|
|
173
|
-
"voyage-3.5"
|
|
174
|
-
"voyage-3.5-lite"
|
|
175
|
-
"voyage-3"
|
|
176
|
-
"voyage-3-lite"
|
|
177
|
-
"voyage-code-3"
|
|
178
|
-
"voyage-code-2"
|
|
179
|
-
"voyage-finance-2"
|
|
180
|
-
"voyage-law-2"
|
|
181
|
-
"voyage-multimodal-3"
|
|
168
|
+
"voyage-4-large" => [256, 512, 1024, 2048],
|
|
169
|
+
"voyage-4" => [256, 512, 1024, 2048],
|
|
170
|
+
"voyage-4-lite" => [256, 512, 1024, 2048],
|
|
171
|
+
"voyage-4-nano" => [256, 512, 1024, 2048],
|
|
172
|
+
"voyage-3-large" => [256, 512, 1024, 2048],
|
|
173
|
+
"voyage-3.5" => [256, 512, 1024, 2048],
|
|
174
|
+
"voyage-3.5-lite" => [256, 512, 1024, 2048],
|
|
175
|
+
"voyage-3" => [1024],
|
|
176
|
+
"voyage-3-lite" => [512],
|
|
177
|
+
"voyage-code-3" => [256, 512, 1024, 2048],
|
|
178
|
+
"voyage-code-2" => [1536],
|
|
179
|
+
"voyage-finance-2" => [1024],
|
|
180
|
+
"voyage-law-2" => [1024],
|
|
181
|
+
"voyage-multimodal-3" => [1024],
|
|
182
182
|
"voyage-multimodal-3.5" => [256, 512, 1024, 2048],
|
|
183
183
|
}.freeze
|
|
184
184
|
|
|
@@ -189,20 +189,20 @@ module Parse
|
|
|
189
189
|
MODEL_SUPPORTED_DIMENSIONS.select { |_m, dims| dims.length > 1 }.keys.freeze
|
|
190
190
|
|
|
191
191
|
MODEL_MAX_INPUT_TOKENS = {
|
|
192
|
-
"voyage-4-large"
|
|
193
|
-
"voyage-4"
|
|
194
|
-
"voyage-4-lite"
|
|
195
|
-
"voyage-4-nano"
|
|
196
|
-
"voyage-3-large"
|
|
197
|
-
"voyage-3.5"
|
|
198
|
-
"voyage-3.5-lite"
|
|
199
|
-
"voyage-3"
|
|
200
|
-
"voyage-3-lite"
|
|
201
|
-
"voyage-code-3"
|
|
202
|
-
"voyage-code-2"
|
|
203
|
-
"voyage-finance-2"
|
|
204
|
-
"voyage-law-2"
|
|
205
|
-
"voyage-multimodal-3"
|
|
192
|
+
"voyage-4-large" => 32_000,
|
|
193
|
+
"voyage-4" => 32_000,
|
|
194
|
+
"voyage-4-lite" => 32_000,
|
|
195
|
+
"voyage-4-nano" => 32_000,
|
|
196
|
+
"voyage-3-large" => 32_000,
|
|
197
|
+
"voyage-3.5" => 32_000,
|
|
198
|
+
"voyage-3.5-lite" => 32_000,
|
|
199
|
+
"voyage-3" => 32_000,
|
|
200
|
+
"voyage-3-lite" => 32_000,
|
|
201
|
+
"voyage-code-3" => 32_000,
|
|
202
|
+
"voyage-code-2" => 16_000,
|
|
203
|
+
"voyage-finance-2" => 32_000,
|
|
204
|
+
"voyage-law-2" => 16_000,
|
|
205
|
+
"voyage-multimodal-3" => 32_000,
|
|
206
206
|
"voyage-multimodal-3.5" => 32_000,
|
|
207
207
|
}.freeze
|
|
208
208
|
|
|
@@ -251,10 +251,10 @@ module Parse
|
|
|
251
251
|
# Voyage only distinguishes retrieval halves — other intents
|
|
252
252
|
# should receive the unconditioned vector.
|
|
253
253
|
INPUT_TYPE_WIRE_VALUES = {
|
|
254
|
-
search_query:
|
|
254
|
+
search_query: "query",
|
|
255
255
|
search_document: "document",
|
|
256
|
-
classification:
|
|
257
|
-
clustering:
|
|
256
|
+
classification: nil,
|
|
257
|
+
clustering: nil,
|
|
258
258
|
}.freeze
|
|
259
259
|
|
|
260
260
|
# @param api_key [String] required. Sent as `Authorization: Bearer …`.
|
|
@@ -400,8 +400,7 @@ module Parse
|
|
|
400
400
|
# different request envelope. The response envelope shape is
|
|
401
401
|
# the same (`{ data: [{ embedding, index }], usage: {...} }`)
|
|
402
402
|
# so `extract_vectors!` is reused as-is.
|
|
403
|
-
body =
|
|
404
|
-
if MULTIMODAL_MODELS.include?(@model)
|
|
403
|
+
body = if MULTIMODAL_MODELS.include?(@model)
|
|
405
404
|
build_multimodal_body(strings, wire_input_type)
|
|
406
405
|
else
|
|
407
406
|
build_text_body(strings, wire_input_type)
|
|
@@ -630,7 +629,7 @@ module Parse
|
|
|
630
629
|
end
|
|
631
630
|
|
|
632
631
|
rows = build_media_rows(
|
|
633
|
-
sources, kind: kind, allow_insecure: allow_insecure, caller_name: caller_name
|
|
632
|
+
sources, kind: kind, allow_insecure: allow_insecure, caller_name: caller_name,
|
|
634
633
|
)
|
|
635
634
|
wire_input_type = INPUT_TYPE_WIRE_VALUES[input_type]
|
|
636
635
|
|
|
@@ -907,7 +906,7 @@ module Parse
|
|
|
907
906
|
end
|
|
908
907
|
|
|
909
908
|
def backoff_seconds(attempt)
|
|
910
|
-
[0.5 * (2**(attempt - 1)), 30.0].min
|
|
909
|
+
[0.5 * (2 ** (attempt - 1)), 30.0].min
|
|
911
910
|
end
|
|
912
911
|
|
|
913
912
|
def retry_after_seconds(response)
|
|
@@ -966,12 +965,11 @@ module Parse
|
|
|
966
965
|
|
|
967
966
|
if base_url
|
|
968
967
|
host = begin
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
inferred =
|
|
974
|
-
case host
|
|
968
|
+
URI.parse(base_url).host
|
|
969
|
+
rescue URI::InvalidURIError
|
|
970
|
+
nil
|
|
971
|
+
end
|
|
972
|
+
inferred = case host
|
|
975
973
|
when URI.parse(ATLAS_BASE_URL).host then :atlas
|
|
976
974
|
when URI.parse(DEFAULT_BASE_URL).host then :voyage
|
|
977
975
|
else :custom
|
data/lib/parse/embeddings.rb
CHANGED
|
@@ -104,6 +104,7 @@ module Parse
|
|
|
104
104
|
class InvalidImageURL < Error
|
|
105
105
|
# @return [Symbol] failure-mode tag.
|
|
106
106
|
attr_reader :reason
|
|
107
|
+
|
|
107
108
|
def initialize(reason, message)
|
|
108
109
|
@reason = reason
|
|
109
110
|
super(message)
|
|
@@ -132,6 +133,7 @@ module Parse
|
|
|
132
133
|
end
|
|
133
134
|
super(name.to_sym, provider)
|
|
134
135
|
end
|
|
136
|
+
|
|
135
137
|
alias_method :store, :[]=
|
|
136
138
|
end
|
|
137
139
|
|
|
@@ -447,8 +449,7 @@ module Parse
|
|
|
447
449
|
# @raise [InvalidImageURL] on any other validation failure.
|
|
448
450
|
def validate_image_url!(url, allow_insecure: false, mode: :forward)
|
|
449
451
|
unless mode == :fetch || trust_provider_url_fetch?
|
|
450
|
-
hint =
|
|
451
|
-
if allowed_image_hosts.empty?
|
|
452
|
+
hint = if allowed_image_hosts.empty?
|
|
452
453
|
" First populate Parse::Embeddings.allowed_image_hosts with the CDN " \
|
|
453
454
|
"hostnames you trust (currently empty — every host would be denied " \
|
|
454
455
|
"even after the sentinel is set)."
|
|
@@ -463,30 +464,30 @@ module Parse
|
|
|
463
464
|
|
|
464
465
|
unless url.is_a?(String) && !url.empty?
|
|
465
466
|
raise InvalidImageURL.new(:parse,
|
|
466
|
-
|
|
467
|
-
|
|
467
|
+
"Parse::Embeddings.validate_image_url!: url must be a non-empty String " \
|
|
468
|
+
"(got #{url.class}).")
|
|
468
469
|
end
|
|
469
470
|
|
|
470
471
|
uri = begin
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
472
|
+
URI.parse(url)
|
|
473
|
+
rescue URI::InvalidURIError => e
|
|
474
|
+
raise InvalidImageURL.new(:parse,
|
|
475
|
+
"Parse::Embeddings.validate_image_url!: invalid URL (#{e.message}).")
|
|
476
|
+
end
|
|
476
477
|
|
|
477
478
|
valid_schemes = allow_insecure ? %w[http https] : %w[https]
|
|
478
479
|
unless valid_schemes.include?(uri.scheme)
|
|
479
480
|
raise InvalidImageURL.new(:scheme,
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
481
|
+
"Parse::Embeddings.validate_image_url!: scheme must be #{valid_schemes.join(" or ")} " \
|
|
482
|
+
"(got #{uri.scheme.inspect}). Forwarding non-HTTPS image URLs to a provider " \
|
|
483
|
+
"leaks any embedded query-string secrets in cleartext.")
|
|
483
484
|
end
|
|
484
485
|
|
|
485
486
|
if uri.userinfo
|
|
486
487
|
raise InvalidImageURL.new(:userinfo,
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
488
|
+
"Parse::Embeddings.validate_image_url!: URL must not include userinfo " \
|
|
489
|
+
"credentials. Embedding providers will forward the full URL in their fetch " \
|
|
490
|
+
"and may log it.")
|
|
490
491
|
end
|
|
491
492
|
|
|
492
493
|
# `uri.hostname` returns the IDNA-decoded form WITHOUT IPv6
|
|
@@ -497,7 +498,7 @@ module Parse
|
|
|
497
498
|
host = uri.hostname
|
|
498
499
|
if host.nil? || host.empty?
|
|
499
500
|
raise InvalidImageURL.new(:parse,
|
|
500
|
-
|
|
501
|
+
"Parse::Embeddings.validate_image_url!: URL is missing a host.")
|
|
501
502
|
end
|
|
502
503
|
|
|
503
504
|
# Reject non-canonical IPv4 forms (decimal `2130706433`,
|
|
@@ -514,9 +515,9 @@ module Parse
|
|
|
514
515
|
# character).
|
|
515
516
|
if ip_shaped_but_not_canonical?(host)
|
|
516
517
|
raise InvalidImageURL.new(:host_blocked,
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
518
|
+
"Parse::Embeddings.validate_image_url!: host #{host.inspect} is an obfuscated " \
|
|
519
|
+
"or non-canonical IP literal. Use dotted-quad IPv4 (a.b.c.d) or canonical IPv6. " \
|
|
520
|
+
"Decimal/octal/hex IP forms are refused to prevent localhost-bypass attempts.")
|
|
520
521
|
end
|
|
521
522
|
|
|
522
523
|
# **Image-host allowlist runs BEFORE the resolver hop.** Round-2
|
|
@@ -528,9 +529,9 @@ module Parse
|
|
|
528
529
|
allowed = allowed_image_hosts
|
|
529
530
|
if allowed.empty?
|
|
530
531
|
raise InvalidImageURL.new(:host_not_allowlisted,
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
532
|
+
"Parse::Embeddings.validate_image_url!: Parse::Embeddings.allowed_image_hosts " \
|
|
533
|
+
"is empty — every image URL is denied. Add the CDN hostnames you trust before " \
|
|
534
|
+
"forwarding image URLs to a provider.")
|
|
534
535
|
end
|
|
535
536
|
permitted = allowed.any? do |entry|
|
|
536
537
|
if entry.start_with?(".")
|
|
@@ -542,8 +543,8 @@ module Parse
|
|
|
542
543
|
end
|
|
543
544
|
unless permitted
|
|
544
545
|
raise InvalidImageURL.new(:host_not_allowlisted,
|
|
545
|
-
|
|
546
|
-
|
|
546
|
+
"Parse::Embeddings.validate_image_url!: host #{host.inspect} not in " \
|
|
547
|
+
"Parse::Embeddings.allowed_image_hosts (#{allowed.inspect}).")
|
|
547
548
|
end
|
|
548
549
|
|
|
549
550
|
# Port allowlist runs after the host allowlist (cheap string
|
|
@@ -553,8 +554,8 @@ module Parse
|
|
|
553
554
|
require_relative "model/file"
|
|
554
555
|
unless Parse::File.allowed_remote_ports.include?(port)
|
|
555
556
|
raise InvalidImageURL.new(:port,
|
|
556
|
-
|
|
557
|
-
|
|
557
|
+
"Parse::Embeddings.validate_image_url!: port #{port} not in " \
|
|
558
|
+
"Parse::File.allowed_remote_ports.")
|
|
558
559
|
end
|
|
559
560
|
|
|
560
561
|
# CIDR + DNS resolution last — most expensive (syscall). An
|
|
@@ -567,7 +568,7 @@ module Parse
|
|
|
567
568
|
rescue ArgumentError => e
|
|
568
569
|
tag = e.message.include?("private/internal address") ? :host_blocked : :parse
|
|
569
570
|
raise InvalidImageURL.new(tag,
|
|
570
|
-
|
|
571
|
+
"Parse::Embeddings.validate_image_url!: #{e.message}")
|
|
571
572
|
end
|
|
572
573
|
|
|
573
574
|
# Return the canonicalized URL so callers store/forward
|