cloakapi 1.4.1-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +160 -0
- data/LICENSE +202 -0
- data/NOTICE +18 -0
- data/README.md +407 -0
- data/THIRD-PARTY-NOTICES.md +22 -0
- data/TRADEMARKS.md +19 -0
- data/examples/fake_gateway.rb +78 -0
- data/examples/hello.rb +64 -0
- data/lib/cloakapi/_lib/linux-x86_64/libcloak_engine.so +0 -0
- data/lib/cloakapi/client.rb +1272 -0
- data/lib/cloakapi/degraded.rb +77 -0
- data/lib/cloakapi/disclosure.rb +252 -0
- data/lib/cloakapi/engine.rb +702 -0
- data/lib/cloakapi/errors.rb +65 -0
- data/lib/cloakapi/pii/bundled_rules.rb +670 -0
- data/lib/cloakapi/pii/card_spans.rb +300 -0
- data/lib/cloakapi/pii/detector.rb +1925 -0
- data/lib/cloakapi/pii/patterns.json +2238 -0
- data/lib/cloakapi/pii/pii_match.rb +7 -0
- data/lib/cloakapi/pii/pii_type.rb +7 -0
- data/lib/cloakapi/pii/ruleset.rb +65 -0
- data/lib/cloakapi/pii/tokenizer.rb +411 -0
- data/lib/cloakapi/posture.rb +82 -0
- data/lib/cloakapi/receipts.rb +406 -0
- data/lib/cloakapi/resources/chat.rb +82 -0
- data/lib/cloakapi/resources/messages.rb +90 -0
- data/lib/cloakapi/resources/models.rb +25 -0
- data/lib/cloakapi/resources/receipts.rb +162 -0
- data/lib/cloakapi/resources/reconstruct.rb +120 -0
- data/lib/cloakapi/resources/request_strings.rb +485 -0
- data/lib/cloakapi/resources/reviewable.rb +163 -0
- data/lib/cloakapi/resources/tokens.rb +28 -0
- data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
- data/lib/cloakapi/routing.rb +500 -0
- data/lib/cloakapi/streaming.rb +196 -0
- data/lib/cloakapi/surrogate_mode.rb +233 -0
- data/lib/cloakapi/version.rb +15 -0
- data/lib/cloakapi/warnings.rb +137 -0
- data/lib/cloakapi.rb +37 -0
- data/licenses/Apache-2.0.txt +202 -0
- data/licenses/CC-BY-4.0.txt +104 -0
- data/licenses/MIT-Faker.txt +21 -0
- data/licenses/MIT.txt +21 -0
- metadata +150 -0
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module CloakAPI
|
|
6
|
+
module Resources
|
|
7
|
+
# Strings on the messages/chat create path that message-content
|
|
8
|
+
# tokenisation never walks: a top-level system prompt (string or text
|
|
9
|
+
# blocks), metadata values (the request's own AND each message's own),
|
|
10
|
+
# and tool-call arguments (OpenAI JSON string or Anthropic input object).
|
|
11
|
+
# Detected PII is tokenised and the call continues — a detection is not
|
|
12
|
+
# a refusal.
|
|
13
|
+
module RequestStrings
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def cloak_create_body(client, messages, kwargs, seed_hex)
|
|
17
|
+
reject_marker_collision!(messages)
|
|
18
|
+
reject_marker_collision!(kwargs)
|
|
19
|
+
[replace_detected_keys(client, sanitize(client, messages, seed_hex, "messages", message_list: true,
|
|
20
|
+
objects_only: true),
|
|
21
|
+
seed_hex, ["messages"]),
|
|
22
|
+
replace_detected_keys(client, sanitize(client, kwargs, seed_hex, top_level: true), seed_hex, [])]
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
TOOL_USE_TYPES = %w[tool_use server_tool_use mcp_tool_use].freeze
|
|
26
|
+
|
|
27
|
+
# A detected value used as an object key in free-form data (user, fields
|
|
28
|
+
# this SDK does not know) is replaced by its stand-in where it stands.
|
|
29
|
+
# Metadata keys are never replaced (owner decision F-068, 2026-11-14);
|
|
30
|
+
# the value under the key is still tokenised by sanitize, and the send
|
|
31
|
+
# is not stopped. Keys inside tool-call arguments and tool definitions
|
|
32
|
+
# follow the tool's contract and stay as written (owner decision
|
|
33
|
+
# 2026-11-13). A stand-in that would collide with a sibling key is not
|
|
34
|
+
# applied. +trail+ is the path from the body root, with "[]" for an
|
|
35
|
+
# array index.
|
|
36
|
+
def replace_detected_keys(client, value, seed_hex, trail)
|
|
37
|
+
case value
|
|
38
|
+
when Array
|
|
39
|
+
value.map { |child| replace_detected_keys(client, child, seed_hex, trail + ["[]"]) }
|
|
40
|
+
when Hash
|
|
41
|
+
value.each_with_object({}) do |(key, child), out|
|
|
42
|
+
name = key.to_s
|
|
43
|
+
if tool_argument_place?(trail, name, value) || (trail.empty? && name == "tools")
|
|
44
|
+
note_argument_keys(client, child) if tool_argument_place?(trail, name, value)
|
|
45
|
+
out[key] = child
|
|
46
|
+
next
|
|
47
|
+
end
|
|
48
|
+
child_trail = trail + [name]
|
|
49
|
+
if trail.include?("metadata")
|
|
50
|
+
out[key] = replace_detected_keys(client, child, seed_hex, child_trail)
|
|
51
|
+
next
|
|
52
|
+
end
|
|
53
|
+
out[detected_key_stand_in(client, key, value, out, seed_hex)] =
|
|
54
|
+
replace_detected_keys(client, child, seed_hex, child_trail)
|
|
55
|
+
end
|
|
56
|
+
else
|
|
57
|
+
value
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Tool-call argument data sits only in a message: the arguments of
|
|
62
|
+
# messages[].tool_calls[].function or legacy messages[].function_call,
|
|
63
|
+
# and the input of a tool_use block in message content (also inside
|
|
64
|
+
# tool_result content). The same names elsewhere are free-form data.
|
|
65
|
+
def tool_argument_place?(trail, name, holder)
|
|
66
|
+
return false unless trail[0] == "messages" && trail[1] == "[]"
|
|
67
|
+
|
|
68
|
+
rest = trail[2..]
|
|
69
|
+
return rest == %w[tool_calls [] function] || rest == %w[function_call] if name == "arguments"
|
|
70
|
+
return false unless name == "input"
|
|
71
|
+
|
|
72
|
+
type = holder["type"] || holder[:type]
|
|
73
|
+
TOOL_USE_TYPES.include?(type.to_s) && rest.length >= 2 && rest.length.even? &&
|
|
74
|
+
rest.each_with_index.all? { |segment, index| segment == (index.even? ? "content" : "[]") }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def detected_key_stand_in(client, key, siblings, written, seed_hex)
|
|
78
|
+
name = key.to_s
|
|
79
|
+
return key if name.empty? || !looks_like_value?(name)
|
|
80
|
+
|
|
81
|
+
renamed = cloak_string(client, name, seed_hex)
|
|
82
|
+
return key if renamed == name
|
|
83
|
+
candidate = key.is_a?(Symbol) ? renamed.to_sym : renamed
|
|
84
|
+
taken = [siblings, written].any? { |hash| hash.key?(renamed) || hash.key?(renamed.to_sym) }
|
|
85
|
+
taken ? key : candidate
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Treat the complete JSON-shaped request as the security boundary. Older
|
|
89
|
+
# walkers knew a handful of fields and copied every other string verbatim
|
|
90
|
+
# (notably tool descriptions, stop sequences and future API fields).
|
|
91
|
+
# Only protocol structure is exempt from tokenisation; unknown fields are
|
|
92
|
+
# recursively tokenised and malformed block shapes fail closed.
|
|
93
|
+
STRUCTURAL_FIELDS = %w[model id tool_use_id].freeze
|
|
94
|
+
ROLE_VALUES = %w[user assistant system developer].freeze
|
|
95
|
+
KNOWN_BLOCK_TYPES = %w[text image image_url tool_use tool_result document file
|
|
96
|
+
input_text input_image input_audio input_file output_text refusal
|
|
97
|
+
thinking redacted_thinking server_tool_use mcp_tool_use audio
|
|
98
|
+
output_audio tool_call function_call].freeze
|
|
99
|
+
# The model's own text, signed by the provider: sent back exactly as the model wrote it.
|
|
100
|
+
THINKING_BLOCK_TYPES = %w[thinking redacted_thinking].freeze
|
|
101
|
+
# Provider blobs and cache hints inside a content block: sent as written.
|
|
102
|
+
OPAQUE_BLOCK_KEYS = %w[signature encrypted_content encrypted_index cache_control].freeze
|
|
103
|
+
MARKER = /<<CLOAK:[A-Z_]+:[0-9a-f]{6}>>/
|
|
104
|
+
# Content blocks whose own `content` is a list of content blocks.
|
|
105
|
+
CONTENT_CARRIERS = %w[tool_result search_result web_search_tool_result mcp_tool_result].freeze
|
|
106
|
+
|
|
107
|
+
# CONTENT-AS-DATA: `content` holds content blocks only on a message and inside a block
|
|
108
|
+
# that carries content (CONTENT_CARRIERS); `system` only at the top level. Anywhere
|
|
109
|
+
# else (tool input, metadata, any free JSON) the value is data: walked and tokenised
|
|
110
|
+
# like other values, never refused. Until 2026-11-15 a key with one of these names
|
|
111
|
+
# was read as content blocks at any depth, so {"content": ["text"]} in tool input
|
|
112
|
+
# raised "content blocks must be objects".
|
|
113
|
+
def content_list?(name, child, in_message, in_content, block_type, top_level)
|
|
114
|
+
return in_message || (in_content && CONTENT_CARRIERS.include?(block_type)) if name == "content"
|
|
115
|
+
|
|
116
|
+
name == "system" && child.is_a?(Array) && top_level
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Lists whose entries must be objects: the request's messages and tools, a message's
|
|
120
|
+
# tool calls. A list under one of these names inside data is data.
|
|
121
|
+
def objects_list?(name, in_message, top_level)
|
|
122
|
+
(top_level && %w[messages tools].include?(name)) || (in_message && name == "tool_calls")
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def sanitize(client, value, seed_hex, key = nil, in_content: false, in_message: false,
|
|
126
|
+
message_list: false, top_level: false, hold_name: false, objects_only: false)
|
|
127
|
+
case value
|
|
128
|
+
when String
|
|
129
|
+
if key.to_s == "tool_choice" || (hold_name && key.to_s == "name")
|
|
130
|
+
note(client, :identifier) if looks_like_value?(value)
|
|
131
|
+
return value
|
|
132
|
+
end
|
|
133
|
+
return value if STRUCTURAL_FIELDS.include?(key.to_s)
|
|
134
|
+
# A content block's own type is checked where the block is read (the
|
|
135
|
+
# Hash branch below, in_content). Anywhere else "type" is ordinary
|
|
136
|
+
# request data: a JSON Schema keyword in a tool's parameters or
|
|
137
|
+
# input_schema ("object", "string"), tool_choice ("auto", "tool"),
|
|
138
|
+
# response_format ("json_object"), a metadata field. It is not a
|
|
139
|
+
# content block, so it is handled like any other string.
|
|
140
|
+
return value if key.to_s == "type" && (KNOWN_BLOCK_TYPES.include?(value) || value == "function")
|
|
141
|
+
cloak_string(client, value, seed_hex)
|
|
142
|
+
when Hash
|
|
143
|
+
type_key = field_key(value, "type")
|
|
144
|
+
block_type = type_key ? value[type_key].to_s : ""
|
|
145
|
+
# THINKING-AS-WRITTEN: a thinking or redacted_thinking block is the model's own text,
|
|
146
|
+
# written from tokenised input and signed by the provider. It goes back exactly as the
|
|
147
|
+
# model wrote it: never tokenised here and never restored in a reply, so its signature
|
|
148
|
+
# still matches and it holds stand-ins only. Same rule as the local proxy.
|
|
149
|
+
return value if in_content && THINKING_BLOCK_TYPES.include?(block_type)
|
|
150
|
+
# UNKNOWN-BLOCK-WALK: a content block of a type this SDK does not name is
|
|
151
|
+
# walked like the rest of the request below (every string is tokenised,
|
|
152
|
+
# protocol fields stay as written). It is reported; it does not stop the send.
|
|
153
|
+
note(client, :unknown_block) if in_content && !KNOWN_BLOCK_TYPES.include?(block_type)
|
|
154
|
+
# Inline media and media references are sent as provided, with a
|
|
155
|
+
# notice (owner decision O1, 2026-11-12; value-placement rule, place
|
|
156
|
+
# 12). They are never handed to the detector: rewriting inside
|
|
157
|
+
# base64 would corrupt the file. Until 1.4.0 they stopped the request.
|
|
158
|
+
# Media is a content-block notion: inside data a key named `file` or `image_url`
|
|
159
|
+
# is ordinary data and is tokenised.
|
|
160
|
+
media = in_content ? media_fields(value) : []
|
|
161
|
+
note(client, :media, media.map { |media_key| value[media_key] }) if in_content && (!media.empty? || media_block?(value))
|
|
162
|
+
value.each_with_object({}) do |(child_key, child), out|
|
|
163
|
+
name = child_key.to_s
|
|
164
|
+
child_hold = hold_name || %w[tools tool_choice function function_call].include?(key.to_s) ||
|
|
165
|
+
(TOOL_USE_TYPES.include?((value["type"] || value[:type]).to_s) && name == "name")
|
|
166
|
+
out[child_key] = if media.include?(child_key)
|
|
167
|
+
child
|
|
168
|
+
elsif in_content && (OPAQUE_BLOCK_KEYS.include?(name) ||
|
|
169
|
+
(name == "data" && block_type == "redacted_thinking"))
|
|
170
|
+
child
|
|
171
|
+
elsif in_message && name == "role" && child.is_a?(String)
|
|
172
|
+
raise ArgumentError, "unknown message role" unless ROLE_VALUES.include?(child)
|
|
173
|
+
child
|
|
174
|
+
elsif name == "arguments" && child.is_a?(String)
|
|
175
|
+
cloak_arguments(client, child, seed_hex)
|
|
176
|
+
else
|
|
177
|
+
sanitize(client, child, seed_hex, name,
|
|
178
|
+
in_content: content_list?(name, child, in_message, in_content, block_type, top_level),
|
|
179
|
+
objects_only: objects_list?(name, in_message, top_level),
|
|
180
|
+
message_list: top_level && name == "messages",
|
|
181
|
+
hold_name: child_hold && name == "name")
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
when Array
|
|
185
|
+
value.map do |child|
|
|
186
|
+
if (in_content || objects_only) && !child.is_a?(Hash)
|
|
187
|
+
raise ArgumentError, "content blocks must be objects"
|
|
188
|
+
end
|
|
189
|
+
sanitize(client, child, seed_hex, key, in_content: in_content,
|
|
190
|
+
in_message: message_list && child.is_a?(Hash))
|
|
191
|
+
end
|
|
192
|
+
when Integer
|
|
193
|
+
# A number stays a number (value-placement rule, place 11). When it
|
|
194
|
+
# looks like personal data the request says so; it is not replaced.
|
|
195
|
+
note(client, :number) if value.abs.to_s.length.between?(6, 16) && looks_like_value?(value.abs.to_s)
|
|
196
|
+
value
|
|
197
|
+
when NilClass, TrueClass, FalseClass, Numeric
|
|
198
|
+
value
|
|
199
|
+
else
|
|
200
|
+
raise ArgumentError, "unsupported request value; refusing to send unreviewed structure"
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
MEDIA_KEYS = %w[file file_data file_url file_id data_url image_url source].freeze
|
|
205
|
+
MEDIA_TYPES = %w[image file image_url].freeze
|
|
206
|
+
# A source of these types is text, not media: a plain-text document (`data`) or a
|
|
207
|
+
# document made of its own content blocks (`content`).
|
|
208
|
+
TEXT_SOURCE_TYPES = %w[text content].freeze
|
|
209
|
+
|
|
210
|
+
# Keys of +hash+ whose value is media or a reference to media.
|
|
211
|
+
def media_fields(hash)
|
|
212
|
+
hash.keys.select do |key|
|
|
213
|
+
value = hash[key]
|
|
214
|
+
name = key.to_s.downcase
|
|
215
|
+
# A string source is the citation source of a search result: text, not media.
|
|
216
|
+
next false if name == "source" && value.is_a?(String)
|
|
217
|
+
# TEXT-SOURCE: a plain-text document's source is walked like other text.
|
|
218
|
+
next false if name == "source" && value.is_a?(Hash) && TEXT_SOURCE_TYPES.include?((value["type"] || value[:type]).to_s)
|
|
219
|
+
|
|
220
|
+
MEDIA_KEYS.include?(name) && ((value.is_a?(String) && !value.empty?) || value.is_a?(Hash))
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def media_block?(hash)
|
|
225
|
+
type = hash["type"] || hash[:type]
|
|
226
|
+
MEDIA_TYPES.include?(type.to_s)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def looks_like_value?(text)
|
|
230
|
+
!CloakAPI::Pii::Detector.new.detect(text).empty?
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Record on the request being prepared that a kind of place holds
|
|
234
|
+
# something that was not replaced. A client that keeps no such record
|
|
235
|
+
# (a test double) is left alone.
|
|
236
|
+
def note_argument_keys(client, value)
|
|
237
|
+
case value
|
|
238
|
+
when Hash
|
|
239
|
+
value.each do |key, child|
|
|
240
|
+
note(client, :tool_argument_key) if looks_like_value?(key.to_s)
|
|
241
|
+
note_argument_keys(client, child)
|
|
242
|
+
end
|
|
243
|
+
when Array
|
|
244
|
+
value.each { |child| note_argument_keys(client, child) }
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def note(client, place, values = [])
|
|
249
|
+
client.send(:note_placement, place, values) if client.respond_to?(:note_placement, true)
|
|
250
|
+
end
|
|
251
|
+
private_class_method :content_list?, :objects_list?
|
|
252
|
+
private_class_method :media_fields, :media_block?, :looks_like_value?, :note, :note_argument_keys,
|
|
253
|
+
:replace_detected_keys, :tool_argument_place?, :detected_key_stand_in
|
|
254
|
+
|
|
255
|
+
def reject_marker_collision!(value)
|
|
256
|
+
case value
|
|
257
|
+
when String
|
|
258
|
+
raise ArgumentError, "reserved token marker in request; refusing ambiguous restoration" if value.match?(MARKER)
|
|
259
|
+
when Hash
|
|
260
|
+
# Thinking holds the stand-ins the model received; it goes back as written.
|
|
261
|
+
return if THINKING_BLOCK_TYPES.include?((value["type"] || value[:type]).to_s)
|
|
262
|
+
|
|
263
|
+
value.each_value { |child| reject_marker_collision!(child) }
|
|
264
|
+
when Array
|
|
265
|
+
value.each { |child| reject_marker_collision!(child) }
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def reject_media_fields!(hash)
|
|
270
|
+
hash.each do |key, value|
|
|
271
|
+
name = key.to_s.downcase
|
|
272
|
+
if %w[file file_data file_url file_id data_url image_url source].include?(name) &&
|
|
273
|
+
((value.is_a?(String) && !value.empty?) || value.is_a?(Hash))
|
|
274
|
+
raise ArgumentError, "file or image payload requires local file review"
|
|
275
|
+
end
|
|
276
|
+
if name == "type" && %w[image file image_url].include?(value.to_s)
|
|
277
|
+
raise ArgumentError, "file or image payload requires local file review"
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
private_class_method :sanitize, :reject_marker_collision!, :reject_media_fields!
|
|
282
|
+
|
|
283
|
+
def cloak_message(client, message, seed_hex)
|
|
284
|
+
return message unless message.is_a?(Hash)
|
|
285
|
+
|
|
286
|
+
copy = message.dup
|
|
287
|
+
key = field_key(copy, "content")
|
|
288
|
+
copy[key] = cloak_content(client, copy[key], seed_hex) if key
|
|
289
|
+
meta = field_key(copy, "metadata")
|
|
290
|
+
copy[meta] = cloak_leaves(client, copy[meta], seed_hex) if meta
|
|
291
|
+
calls = field_key(copy, "tool_calls")
|
|
292
|
+
copy[calls] = cloak_tool_calls(client, copy[calls], seed_hex) if calls
|
|
293
|
+
copy
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def cloak_system!(client, kwargs, seed_hex)
|
|
297
|
+
key = field_key(kwargs, "system")
|
|
298
|
+
return unless key
|
|
299
|
+
|
|
300
|
+
sys = kwargs[key]
|
|
301
|
+
cloaked = if sys.is_a?(String)
|
|
302
|
+
cloak_string(client, sys, seed_hex)
|
|
303
|
+
elsif sys.is_a?(Array)
|
|
304
|
+
sys.map { |part| cloak_system_part(client, part, seed_hex) }
|
|
305
|
+
end
|
|
306
|
+
return if cloaked.nil?
|
|
307
|
+
|
|
308
|
+
kwargs.delete(:system)
|
|
309
|
+
kwargs.delete("system")
|
|
310
|
+
kwargs[:system] = cloaked
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def cloak_metadata!(client, kwargs, seed_hex)
|
|
314
|
+
key = field_key(kwargs, "metadata")
|
|
315
|
+
return unless key
|
|
316
|
+
|
|
317
|
+
meta = kwargs[key]
|
|
318
|
+
return unless meta.is_a?(Hash)
|
|
319
|
+
|
|
320
|
+
kwargs.delete(:metadata)
|
|
321
|
+
kwargs.delete("metadata")
|
|
322
|
+
kwargs[:metadata] = cloak_leaves(client, meta, seed_hex)
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def cloak_content(client, content, seed_hex)
|
|
326
|
+
return cloak_string(client, content, seed_hex) if content.is_a?(String)
|
|
327
|
+
return content unless content.is_a?(Array)
|
|
328
|
+
|
|
329
|
+
content.map do |part|
|
|
330
|
+
next part unless part.is_a?(Hash)
|
|
331
|
+
|
|
332
|
+
reject_unreviewed_image!(part)
|
|
333
|
+
reject_unreviewed_file!(part)
|
|
334
|
+
|
|
335
|
+
copy = part.dup
|
|
336
|
+
text = field_key(copy, "text")
|
|
337
|
+
copy[text] = cloak_string(client, copy[text], seed_hex) if text && copy[text].is_a?(String)
|
|
338
|
+
input = field_key(copy, "input")
|
|
339
|
+
copy[input] = cloak_leaves(client, copy[input], seed_hex) if input && copy[input].is_a?(Hash)
|
|
340
|
+
nested = field_key(copy, "content")
|
|
341
|
+
copy[nested] = cloak_content(client, copy[nested], seed_hex) if nested
|
|
342
|
+
copy
|
|
343
|
+
end
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# Image bytes and remote image URLs must pass through tokenize_file and
|
|
347
|
+
# local review before they can enter a relay body. Text tokenisation does
|
|
348
|
+
# not inspect image payloads, so silently preserving these slots is unsafe.
|
|
349
|
+
def reject_unreviewed_image!(part)
|
|
350
|
+
type = part["type"] || part[:type]
|
|
351
|
+
image_url = part["image_url"] || part[:image_url]
|
|
352
|
+
return unless type.to_s == "image" || type.to_s == "image_url" || image_url.is_a?(Hash)
|
|
353
|
+
|
|
354
|
+
source = part["source"] || part[:source]
|
|
355
|
+
source_data = source.is_a?(Hash) && (source["data"] || source[:data])
|
|
356
|
+
source_url = source.is_a?(Hash) && (source["url"] || source[:url])
|
|
357
|
+
image_url_value = image_url.is_a?(Hash) && (image_url["url"] || image_url[:url])
|
|
358
|
+
if [source_data, source_url, image_url_value].any? { |value| value.is_a?(String) && !value.empty? }
|
|
359
|
+
raise ArgumentError, "inline image bytes or URLs require local file review"
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# File payloads need the same on-device tokenize_file + local review path
|
|
364
|
+
# as images. This request API has no reviewed attachment representation,
|
|
365
|
+
# so fail closed before the preview can be approved or relayed.
|
|
366
|
+
def reject_unreviewed_file!(part)
|
|
367
|
+
type = part["type"] || part[:type]
|
|
368
|
+
file = part["file"] || part[:file]
|
|
369
|
+
fields = %w[file_data file_url file_id]
|
|
370
|
+
has_file_field = fields.any? do |field|
|
|
371
|
+
value = part[field] || part[field.to_sym]
|
|
372
|
+
value.is_a?(String) && !value.empty?
|
|
373
|
+
end
|
|
374
|
+
nested_file_field = file.is_a?(Hash) && fields.any? do |field|
|
|
375
|
+
value = file[field] || file[field.to_sym]
|
|
376
|
+
value.is_a?(String) && !value.empty?
|
|
377
|
+
end
|
|
378
|
+
return unless type.to_s == "file" || has_file_field || nested_file_field
|
|
379
|
+
|
|
380
|
+
raise ArgumentError, "file bytes or references require local file review"
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
def cloak_system_part(client, part, seed_hex)
|
|
384
|
+
return part unless part.is_a?(Hash)
|
|
385
|
+
|
|
386
|
+
copy = part.dup
|
|
387
|
+
text = field_key(copy, "text")
|
|
388
|
+
copy[text] = cloak_string(client, copy[text], seed_hex) if text && copy[text].is_a?(String)
|
|
389
|
+
copy
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
def cloak_tool_calls(client, calls, seed_hex)
|
|
393
|
+
return calls unless calls.is_a?(Array)
|
|
394
|
+
|
|
395
|
+
calls.map do |call|
|
|
396
|
+
next call unless call.is_a?(Hash)
|
|
397
|
+
|
|
398
|
+
copy = call.dup
|
|
399
|
+
fn_key = field_key(copy, "function")
|
|
400
|
+
next copy unless fn_key && copy[fn_key].is_a?(Hash)
|
|
401
|
+
|
|
402
|
+
fn = copy[fn_key].dup
|
|
403
|
+
args = field_key(fn, "arguments")
|
|
404
|
+
if args && (fn[args].is_a?(String) || fn[args].is_a?(Hash) || fn[args].is_a?(Array))
|
|
405
|
+
fn[args] = cloak_arguments(client, fn[args], seed_hex)
|
|
406
|
+
end
|
|
407
|
+
copy[fn_key] = fn
|
|
408
|
+
copy
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
def cloak_leaves(client, value, seed_hex)
|
|
413
|
+
case value
|
|
414
|
+
when String
|
|
415
|
+
cloak_string(client, value, seed_hex)
|
|
416
|
+
when Array
|
|
417
|
+
value.map { |v| cloak_leaves(client, v, seed_hex) }
|
|
418
|
+
when Hash
|
|
419
|
+
value.each_with_object({}) do |(k, v), out|
|
|
420
|
+
out[k] = cloak_leaves(client, v, seed_hex)
|
|
421
|
+
end
|
|
422
|
+
else
|
|
423
|
+
value
|
|
424
|
+
end
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
# OpenAI tool-call arguments are normally a JSON string. Tokenising that
|
|
428
|
+
# raw string misses characters encoded with JSON escapes (for example an
|
|
429
|
+
# e-mail's `@` as `\u0040`). The receiving tool decodes those escapes, so
|
|
430
|
+
# the PII would still reach it. Decode each JSON string VALUE before
|
|
431
|
+
# tokenising it, while preserving keys and every unchanged byte of the
|
|
432
|
+
# caller's JSON text. Non-JSON strings retain the historical plain-text
|
|
433
|
+
# behaviour.
|
|
434
|
+
def cloak_arguments(client, arguments, seed_hex)
|
|
435
|
+
return cloak_leaves(client, arguments, seed_hex) unless arguments.is_a?(String)
|
|
436
|
+
|
|
437
|
+
begin
|
|
438
|
+
parsed = JSON.parse(arguments, max_nesting: false)
|
|
439
|
+
rescue JSON::ParserError, EncodingError
|
|
440
|
+
return cloak_string(client, arguments, seed_hex)
|
|
441
|
+
end
|
|
442
|
+
# A root JSON number, true, false or null is not a string. The bytes
|
|
443
|
+
# below stay the caller's. A detected root number is the same notice
|
|
444
|
+
# as a native number.
|
|
445
|
+
if parsed.is_a?(Integer) && parsed.abs.to_s.length.between?(6, 16) && looks_like_value?(parsed.abs.to_s)
|
|
446
|
+
note(client, :number)
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
changed = false
|
|
450
|
+
out = arguments.gsub(/"(?:[^"\\]|\\.)*"/m) do |literal|
|
|
451
|
+
after_literal = Regexp.last_match.end(0)
|
|
452
|
+
if /\A\s*:/.match?(arguments[after_literal..].to_s)
|
|
453
|
+
key_text = JSON.parse(literal) rescue nil
|
|
454
|
+
note(client, :tool_argument_key) if key_text.is_a?(String) && looks_like_value?(key_text)
|
|
455
|
+
next literal
|
|
456
|
+
end
|
|
457
|
+
|
|
458
|
+
value = JSON.parse(literal)
|
|
459
|
+
tokenised = cloak_string(client, value, seed_hex)
|
|
460
|
+
if tokenised == value
|
|
461
|
+
literal
|
|
462
|
+
else
|
|
463
|
+
changed = true
|
|
464
|
+
JSON.generate(tokenised)
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
changed ? out : arguments
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
def cloak_string(client, text, seed_hex)
|
|
472
|
+
return text if text.nil? || text.empty?
|
|
473
|
+
|
|
474
|
+
client.tokenize_outbound(text, seed_hex: seed_hex)
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
def field_key(hash, name)
|
|
478
|
+
return name.to_sym if hash.key?(name.to_sym)
|
|
479
|
+
return name.to_s if hash.key?(name.to_s)
|
|
480
|
+
|
|
481
|
+
nil
|
|
482
|
+
end
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "request_strings"
|
|
4
|
+
|
|
5
|
+
module CloakAPI
|
|
6
|
+
module Resources
|
|
7
|
+
# The one review pattern, shared by +messages+ and +chat.completions+.
|
|
8
|
+
#
|
|
9
|
+
# Step form:
|
|
10
|
+
#
|
|
11
|
+
# prepared = resource.prepare(model: ..., messages: ...)
|
|
12
|
+
# # show prepared.text, prepared.coverage, prepared.warnings
|
|
13
|
+
# approved = client.approve(prepared) # or prepared.approve
|
|
14
|
+
# reply = resource.create(approved) # send_approved is the same call
|
|
15
|
+
#
|
|
16
|
+
# Callback form (one call):
|
|
17
|
+
#
|
|
18
|
+
# client = CloakAPI::Client.new(key, review: ->(text, info) { show(text); true })
|
|
19
|
+
# reply = client.messages.create(model: ..., messages: ...)
|
|
20
|
+
#
|
|
21
|
+
# The callback receives the exact outgoing body and a CloakAPI::ReviewInfo
|
|
22
|
+
# (state, coverage, warnings). A true value approves. A false or nil value,
|
|
23
|
+
# or no callback at all, raises CloakAPI::ReviewRequiredError before any
|
|
24
|
+
# network call.
|
|
25
|
+
module Reviewable
|
|
26
|
+
# Tokenise every string of the request on this device and return the
|
|
27
|
+
# prepared request for review. Nothing is sent.
|
|
28
|
+
def prepare(model:, messages:, end_user: nil, seed_hex: nil, stream: false, **kwargs)
|
|
29
|
+
c = @client
|
|
30
|
+
# Under the explicit strict option this raises before anything is
|
|
31
|
+
# tokenised; otherwise it only writes the notice.
|
|
32
|
+
c.validate_engine_identity!(allow_rules_only: true)
|
|
33
|
+
body = nil
|
|
34
|
+
collected = c.send(:collect_request) do
|
|
35
|
+
body_messages = messages
|
|
36
|
+
if c.tokenize
|
|
37
|
+
body_messages, kwargs = RequestStrings.cloak_create_body(c, messages, kwargs, seed_hex)
|
|
38
|
+
end
|
|
39
|
+
body = { model: model, messages: body_messages }.merge(kwargs)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Raises only under the explicit strict option; otherwise the engine
|
|
43
|
+
# state becomes coverage and warnings on the prepared request.
|
|
44
|
+
engine_identity, identity_warning = c.send(:checked_engine_identity)
|
|
45
|
+
engine_in_use = !engine_identity[1].empty?
|
|
46
|
+
coverage = collected[:coverage]
|
|
47
|
+
warnings = CloakAPI::Warnings.merge(collected[:warnings], identity_warning)
|
|
48
|
+
# Places where something is sent as it is (value-placement rule): the
|
|
49
|
+
# sentence names the kind of place, never the value.
|
|
50
|
+
if collected[:places].include?(:number)
|
|
51
|
+
warnings = CloakAPI::Warnings.merge(warnings, CloakAPI::Warnings.unchecked("value_not_replaced", CloakAPI::Warnings::NUMBER_MESSAGE))
|
|
52
|
+
end
|
|
53
|
+
if collected[:places].include?(:identifier)
|
|
54
|
+
warnings = CloakAPI::Warnings.merge(warnings, CloakAPI::Warnings.unchecked("value_not_replaced", CloakAPI::Warnings::IDENTIFIER_MESSAGE))
|
|
55
|
+
end
|
|
56
|
+
if collected[:places].include?(:tool_argument_key)
|
|
57
|
+
warnings = CloakAPI::Warnings.merge(warnings, CloakAPI::Warnings.unchecked("value_not_replaced", CloakAPI::Warnings::ARGUMENT_KEY_MESSAGE))
|
|
58
|
+
end
|
|
59
|
+
if collected[:places].include?(:media)
|
|
60
|
+
warnings = CloakAPI::Warnings.merge(warnings, CloakAPI::Warnings.unchecked("media_not_checked", CloakAPI::Warnings::MEDIA_MESSAGE))
|
|
61
|
+
end
|
|
62
|
+
if collected[:places].include?(:unknown_block)
|
|
63
|
+
warnings = CloakAPI::Warnings.merge(warnings, CloakAPI::Warnings.unchecked("unrecognised_block", CloakAPI::Warnings::UNKNOWN_BLOCK_MESSAGE))
|
|
64
|
+
end
|
|
65
|
+
# No string was tokenised (for example an empty message): the request
|
|
66
|
+
# still states what this device could have checked.
|
|
67
|
+
coverage ||= engine_in_use ? CloakAPI::Coverage::FULL : CloakAPI::Coverage::RULES_ONLY
|
|
68
|
+
# Engine identity is stated only when the engine handled every string.
|
|
69
|
+
if collected[:rules_fallback] || !engine_in_use
|
|
70
|
+
engine_identity = [CloakAPI::Engine::RULESET_VERSION, ""]
|
|
71
|
+
coverage = CloakAPI::Coverage::RULES_ONLY
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
c.send(:prepare_review, relay_path, body,
|
|
75
|
+
token_map: c.session_token_map,
|
|
76
|
+
streaming: stream,
|
|
77
|
+
headers: c.end_user_header(end_user),
|
|
78
|
+
engine_identity: engine_identity,
|
|
79
|
+
coverage: coverage,
|
|
80
|
+
warnings: warnings,
|
|
81
|
+
unread: collected[:media])
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Send an approved request, or (with keyword arguments and a review
|
|
85
|
+
# callback) prepare, review and send in one call.
|
|
86
|
+
def create(reviewed = nil, end_user: nil, review: nil, **kwargs)
|
|
87
|
+
if reviewed.nil? && !kwargs.empty?
|
|
88
|
+
reviewed = review_with_callback(review, stream: false, end_user: end_user, **kwargs)
|
|
89
|
+
kwargs = {}
|
|
90
|
+
end
|
|
91
|
+
unless reviewed.is_a?(CloakAPI::Client::ReviewedRequest)
|
|
92
|
+
raise CloakAPI::ReviewRequiredError,
|
|
93
|
+
"explicit review required: prepare the request, inspect preview, approve it, then pass ReviewedRequest " \
|
|
94
|
+
"(or give the client a review callback)"
|
|
95
|
+
end
|
|
96
|
+
raise ArgumentError, "request body cannot change after review" unless kwargs.empty?
|
|
97
|
+
|
|
98
|
+
token_map = @client.send(:reviewed_token_map, reviewed)
|
|
99
|
+
resp = @client.post_reviewed(reviewed, path: relay_path,
|
|
100
|
+
extra_headers: @client.end_user_header(end_user))
|
|
101
|
+
detokenise_response(@client, resp, token_map)
|
|
102
|
+
end
|
|
103
|
+
alias send_approved create
|
|
104
|
+
|
|
105
|
+
def create_stream(reviewed = nil, end_user: nil, review: nil, **kwargs, &handler)
|
|
106
|
+
if reviewed.nil? && !kwargs.empty?
|
|
107
|
+
reviewed = review_with_callback(review, stream: true, end_user: end_user, **kwargs)
|
|
108
|
+
end
|
|
109
|
+
unless reviewed.is_a?(CloakAPI::Client::ReviewedRequest)
|
|
110
|
+
raise CloakAPI::ReviewRequiredError,
|
|
111
|
+
"explicit review required: prepare a streaming request, inspect preview, approve it, then pass ReviewedRequest " \
|
|
112
|
+
"(or give the client a review callback)"
|
|
113
|
+
end
|
|
114
|
+
detok = CloakAPI::Streaming::Detokenizer.new(@client.send(:reviewed_token_map, reviewed))
|
|
115
|
+
rescued = nil
|
|
116
|
+
result = nil
|
|
117
|
+
begin
|
|
118
|
+
result = @client.post_reviewed(reviewed, path: relay_path, streaming: true,
|
|
119
|
+
extra_headers: @client.end_user_header(end_user)) do |event|
|
|
120
|
+
next if event.nil?
|
|
121
|
+
|
|
122
|
+
handler.call(CloakAPI::Streaming.detokenise_event(event, detok)) if handler
|
|
123
|
+
end
|
|
124
|
+
rescue StandardError => error
|
|
125
|
+
rescued = error
|
|
126
|
+
ensure
|
|
127
|
+
begin
|
|
128
|
+
tail = CloakAPI::Streaming.flush_event(detok, stream_format)
|
|
129
|
+
handler.call(tail) if tail && handler
|
|
130
|
+
rescue StandardError
|
|
131
|
+
# Preserve the original failure if the final tail callback also
|
|
132
|
+
# fails. On the success path the tail callback error propagates.
|
|
133
|
+
raise unless rescued
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
raise rescued if rescued
|
|
137
|
+
|
|
138
|
+
result
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
private
|
|
142
|
+
|
|
143
|
+
# Callback form: prepare, hand (text, info) to the callback, approve on a
|
|
144
|
+
# true value. Anything else is review_required, before any network call.
|
|
145
|
+
def review_with_callback(per_call, stream:, end_user:, **request)
|
|
146
|
+
callback = per_call || @client.review
|
|
147
|
+
if callback.nil?
|
|
148
|
+
raise CloakAPI::ReviewRequiredError,
|
|
149
|
+
"explicit review required: give the client a review callback (review: ->(text, info) { ... }) " \
|
|
150
|
+
"or prepare the request, approve it, then pass it to create"
|
|
151
|
+
end
|
|
152
|
+
raise ArgumentError, "review must respond to call(text, info)" unless callback.respond_to?(:call)
|
|
153
|
+
|
|
154
|
+
prepared = prepare(stream: stream, end_user: end_user, **request)
|
|
155
|
+
unless callback.call(prepared.text, prepared.info)
|
|
156
|
+
raise CloakAPI::ReviewRequiredError, "explicit review required: the review callback declined the request"
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
@client.approve(prepared)
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "request_strings"
|
|
4
|
+
|
|
5
|
+
module CloakAPI
|
|
6
|
+
module Resources
|
|
7
|
+
# Token counting is unavailable. Sending messages to a gateway for counting
|
|
8
|
+
# would violate the content-blind boundary; callers should estimate locally.
|
|
9
|
+
#
|
|
10
|
+
# N9hg, 2026-08-04 — the old note said the opposite, in as many words
|
|
11
|
+
# (kept verbatim, R2):
|
|
12
|
+
#
|
|
13
|
+
# NOTE: this is a SERVER-SIDE estimate — the gateway sees the message text for
|
|
14
|
+
# counting (a server-side convenience). For a fully client-side count use the
|
|
15
|
+
# bundled engine. Kept explicit so callers choose the trade-off knowingly.
|
|
16
|
+
#
|
|
17
|
+
class Tokens
|
|
18
|
+
def initialize(client)
|
|
19
|
+
@client = client
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def count(provider:, model:, messages:, all_providers: false)
|
|
23
|
+
raise CloakAPI::TokensCountUnavailableError,
|
|
24
|
+
"tokens.count is unavailable: server-side token counting is disabled; estimate locally with your provider tokenizer"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|