cloakapi 1.4.1-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +160 -0
- data/LICENSE +202 -0
- data/NOTICE +18 -0
- data/README.md +407 -0
- data/THIRD-PARTY-NOTICES.md +22 -0
- data/TRADEMARKS.md +19 -0
- data/examples/fake_gateway.rb +78 -0
- data/examples/hello.rb +64 -0
- data/lib/cloakapi/_lib/linux-x86_64/libcloak_engine.so +0 -0
- data/lib/cloakapi/client.rb +1272 -0
- data/lib/cloakapi/degraded.rb +77 -0
- data/lib/cloakapi/disclosure.rb +252 -0
- data/lib/cloakapi/engine.rb +702 -0
- data/lib/cloakapi/errors.rb +65 -0
- data/lib/cloakapi/pii/bundled_rules.rb +670 -0
- data/lib/cloakapi/pii/card_spans.rb +300 -0
- data/lib/cloakapi/pii/detector.rb +1925 -0
- data/lib/cloakapi/pii/patterns.json +2238 -0
- data/lib/cloakapi/pii/pii_match.rb +7 -0
- data/lib/cloakapi/pii/pii_type.rb +7 -0
- data/lib/cloakapi/pii/ruleset.rb +65 -0
- data/lib/cloakapi/pii/tokenizer.rb +411 -0
- data/lib/cloakapi/posture.rb +82 -0
- data/lib/cloakapi/receipts.rb +406 -0
- data/lib/cloakapi/resources/chat.rb +82 -0
- data/lib/cloakapi/resources/messages.rb +90 -0
- data/lib/cloakapi/resources/models.rb +25 -0
- data/lib/cloakapi/resources/receipts.rb +162 -0
- data/lib/cloakapi/resources/reconstruct.rb +120 -0
- data/lib/cloakapi/resources/request_strings.rb +485 -0
- data/lib/cloakapi/resources/reviewable.rb +163 -0
- data/lib/cloakapi/resources/tokens.rb +28 -0
- data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
- data/lib/cloakapi/routing.rb +500 -0
- data/lib/cloakapi/streaming.rb +196 -0
- data/lib/cloakapi/surrogate_mode.rb +233 -0
- data/lib/cloakapi/version.rb +15 -0
- data/lib/cloakapi/warnings.rb +137 -0
- data/lib/cloakapi.rb +37 -0
- data/licenses/Apache-2.0.txt +202 -0
- data/licenses/CC-BY-4.0.txt +104 -0
- data/licenses/MIT-Faker.txt +21 -0
- data/licenses/MIT.txt +21 -0
- metadata +150 -0
|
@@ -0,0 +1,702 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
# fiddle comes with Ruby up to 3.4 and is a bundled gem from 3.5. Without it
|
|
5
|
+
# (a Ruby built without libffi, or an application bundle on Ruby 3.5+ that
|
|
6
|
+
# does not list it) the SDK still loads and works on its built-in rules.
|
|
7
|
+
begin
|
|
8
|
+
require "fiddle"
|
|
9
|
+
rescue LoadError
|
|
10
|
+
nil
|
|
11
|
+
end
|
|
12
|
+
require "fileutils"
|
|
13
|
+
require "json"
|
|
14
|
+
require "rbconfig"
|
|
15
|
+
require "tmpdir"
|
|
16
|
+
|
|
17
|
+
require_relative "warnings"
|
|
18
|
+
require_relative "degraded"
|
|
19
|
+
require_relative "pii/ruleset"
|
|
20
|
+
require_relative "surrogate_mode"
|
|
21
|
+
|
|
22
|
+
module CloakAPI
|
|
23
|
+
# Client-side engine binding — Fiddle (stdlib) over the shared CloakAPI C ABI.
|
|
24
|
+
#
|
|
25
|
+
# This is the Ruby end of the "shared Rust engine, embedded per language via
|
|
26
|
+
# FFI" design. It loads +libcloak_engine+ (built from
|
|
27
|
+
# +packages/cloak-engine-ffi+, a cdylib over the SAME file-pipeline +
|
|
28
|
+
# pii-engine crates the desktop app and local proxy use) and turns **raw file
|
|
29
|
+
# bytes + MIME into tokenised markdown ENTIRELY on this machine** — so a
|
|
30
|
+
# document attachment (or a prompt) is protected before a single byte egresses
|
|
31
|
+
# to the gateway. Mirrors the reference implementation in
|
|
32
|
+
# +sdk/python/cloakapi/engine.py+.
|
|
33
|
+
#
|
|
34
|
+
# +fiddle+ is deliberate: it comes with Ruby, so there is no build step and
|
|
35
|
+
# no compiler. The only runtime artifact is the shared library, taken from
|
|
36
|
+
# (in order):
|
|
37
|
+
#
|
|
38
|
+
# 1. the path in +CLOAKAPI_ENGINE_LIB+, used only when
|
|
39
|
+
# +CLOAKAPI_ENGINE_SHA256_PIN+ equals the SHA-256 of that file;
|
|
40
|
+
# 2. a file shipped inside the gem (+cloakapi/_lib/+), used only when its
|
|
41
|
+
# SHA-256 is the one this release names for the platform.
|
|
42
|
+
#
|
|
43
|
+
# When no engine is used, {available?} is false, {status} says why, and text
|
|
44
|
+
# is handled by the SDK's built-in rules with a notice. A wrong or missing
|
|
45
|
+
# engine never stops a send.
|
|
46
|
+
module Engine
|
|
47
|
+
# The local engine was present but refused to process this input
|
|
48
|
+
# (unsupported/encrypted/undecodable file). The caller must NOT then fall back
|
|
49
|
+
# to sending the raw text/bytes — fail closed.
|
|
50
|
+
#
|
|
51
|
+
# NOTE: secrets (API keys/cards/IBANs/national-IDs) are NOT an EngineError —
|
|
52
|
+
# the policy is TOKENISE-AND-FLAG, not block. A credential is tokenised to a
|
|
53
|
+
# +<<CLOAK:CRED_KEY:…>>+ token like any other entity (golden vector
|
|
54
|
+
# +text-secret-apikey+) and the call returns normally.
|
|
55
|
+
class EngineError < StandardError; end
|
|
56
|
+
|
|
57
|
+
# Canonical shared-ruleset (fasit) semver — the ALWAYS-present
|
|
58
|
+
# X-Cloak-Engine-Version fallback used when no engine binary is loaded to
|
|
59
|
+
# report its own. Tracks shared/pii-patterns/patterns.json "version" and
|
|
60
|
+
# mirrors the Go/TS/Rust bindings' RULESET_VERSION (ADR-0014 §3). The strict
|
|
61
|
+
# gateway 428s a relay that presents no version under
|
|
62
|
+
# missing_identity_mode=strict, so this header is never dropped.
|
|
63
|
+
# DERIVED, never hand-typed: read from the ruleset the SDK actually ships
|
|
64
|
+
# (lib/cloakapi/pii/patterns.json), which spec/ruleset_conformance_spec.rb
|
|
65
|
+
# byte-locks to shared/pii-patterns/patterns.json. A hand-typed literal here
|
|
66
|
+
# had already drifted (1.4.29 while canonical was 1.6.0), i.e. the SDK
|
|
67
|
+
# declared a ruleset it was not running.
|
|
68
|
+
RULESET_VERSION = CloakAPI::Pii::Ruleset.version
|
|
69
|
+
|
|
70
|
+
# Matches the engine's own token grammar: <<CLOAK:TYPE:6-hex>>.
|
|
71
|
+
TOKEN_RE = /<<CLOAK:[A-Z_]+:([0-9a-f]{6})>>/.freeze
|
|
72
|
+
# Type label extracted from each emitted token, for category reporting.
|
|
73
|
+
TOKEN_LABEL_RE = /<<CLOAK:([A-Z_]+):[0-9a-f]{6}>>/.freeze
|
|
74
|
+
|
|
75
|
+
# Outcome of a fully on-device *text* tokenisation (the prompt path).
|
|
76
|
+
# text — tokenised text; the ONLY field meant to leave the machine.
|
|
77
|
+
# token_map — REPLACEMENT STRING => original; kept LOCAL to restore the
|
|
78
|
+
# reply. Built from the engine's +replacements+ table, NOT
|
|
79
|
+
# from the legacy hash-keyed +mappings+: in BALANCED the
|
|
80
|
+
# replacement is not a token at all, so a hash-keyed map
|
|
81
|
+
# cannot express it and a BALANCED reply could not be
|
|
82
|
+
# restored from it.
|
|
83
|
+
# categories — cloud-token labels that fired (e.g. ["CONTACT_EM","ID_GOV"]).
|
|
84
|
+
# MAX-only signal: BALANCED emits realistic strings, not
|
|
85
|
+
# <<CLOAK:TYPE:hash>> tokens, so no label is recoverable
|
|
86
|
+
# from the bytes. A REPORTING difference, never a detection
|
|
87
|
+
# difference — the span set is identical in both modes.
|
|
88
|
+
# surrogate_mode — REQUIRED {CloakAPI::SurrogateMode::Report}: what this pass
|
|
89
|
+
# ACTUALLY did. Data.define makes it a required keyword, so
|
|
90
|
+
# a hand-written fake result raises at construction.
|
|
91
|
+
# bindings — the conversation ledger, ADVANCED by this call.
|
|
92
|
+
# RE-IDENTIFICATION MATERIAL: it maps stand-ins back to real
|
|
93
|
+
# values. CLIENT-SIDE ONLY — never egress it, never log it,
|
|
94
|
+
# never put it in a receipt, exactly like the token map.
|
|
95
|
+
LocalTextResult = Data.define(:text, :token_map, :categories, :surrogate_mode, :bindings, :meta) do
|
|
96
|
+
def initialize(text:, token_map:, categories:, surrogate_mode:, bindings:, meta: {})
|
|
97
|
+
super(text: text, token_map: token_map, categories: categories,
|
|
98
|
+
surrogate_mode: surrogate_mode, bindings: bindings, meta: meta)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def inspect
|
|
102
|
+
"#<#{self.class} sensitive fields redacted>"
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
alias to_s inspect
|
|
106
|
+
|
|
107
|
+
def marshal_dump
|
|
108
|
+
raise TypeError, "CloakAPI tokenisation results cannot be marshalled"
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Outcome of a fully on-device *file* tokenisation. Same field contract as
|
|
113
|
+
# {LocalTextResult}; +markdown+ replaces +text+ and +meta+ carries the
|
|
114
|
+
# diagnostics (resolved mime, extractor tag, pii_count, chars…).
|
|
115
|
+
LocalTokeniseResult = Data.define(:markdown, :token_map, :meta, :surrogate_mode, :bindings) do
|
|
116
|
+
def inspect
|
|
117
|
+
"#<#{self.class} sensitive fields redacted>"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
alias to_s inspect
|
|
121
|
+
|
|
122
|
+
def marshal_dump
|
|
123
|
+
raise TypeError, "CloakAPI tokenisation results cannot be marshalled"
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Thin, safe wrapper over the +cloak_engine+ C ABI.
|
|
128
|
+
class LocalEngine
|
|
129
|
+
# +lib_hash+ (optional) is the SHA-256 of the exact bytes the +handle+ was
|
|
130
|
+
# loaded from. {Engine.load_engine} passes it so the identity is never
|
|
131
|
+
# recomputed from a path that can change after the library was mapped.
|
|
132
|
+
def initialize(handle, lib_path = nil, lib_hash = nil)
|
|
133
|
+
@handle = handle
|
|
134
|
+
@lib_path = lib_path
|
|
135
|
+
@lib_hash = lib_hash
|
|
136
|
+
@fn_tokenise = Fiddle::Function.new(
|
|
137
|
+
handle["cloak_tokenise_file"],
|
|
138
|
+
[
|
|
139
|
+
Fiddle::TYPE_VOIDP, # const uint8_t* bytes
|
|
140
|
+
Fiddle::TYPE_SIZE_T, # size_t len
|
|
141
|
+
Fiddle::TYPE_VOIDP, # const char* mime
|
|
142
|
+
Fiddle::TYPE_VOIDP, # const char* opts_json
|
|
143
|
+
Fiddle::TYPE_VOIDP, # char** out_json
|
|
144
|
+
Fiddle::TYPE_VOIDP # char** err
|
|
145
|
+
],
|
|
146
|
+
Fiddle::TYPE_INT
|
|
147
|
+
)
|
|
148
|
+
@fn_free = Fiddle::Function.new(
|
|
149
|
+
handle["cloak_string_free"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOID
|
|
150
|
+
)
|
|
151
|
+
@fn_version = Fiddle::Function.new(
|
|
152
|
+
handle["cloak_engine_version"], [], Fiddle::TYPE_VOIDP
|
|
153
|
+
)
|
|
154
|
+
# ADR-0014 engine-identity symbol (locked 2026-07-12). Bound OPTIONALLY:
|
|
155
|
+
# an older engine build may predate it, in which case the header falls
|
|
156
|
+
# back to the RULESET_VERSION constant.
|
|
157
|
+
# cloak_ruleset_version() -> X-Cloak-Engine-Version (fasit semver)
|
|
158
|
+
# cloak_engine_hash() -> SHA-256 of the exact shared ruleset bytes
|
|
159
|
+
# NOTE: X-Cloak-Engine-Hash is NOT sourced from an FFI symbol — the
|
|
160
|
+
# +cloak_engine_hash()+ export returns the RULESET content digest, which
|
|
161
|
+
# identifies the ruleset, NOT the binary that ran. The gateway
|
|
162
|
+
# attestation must identify the ENGINE BINARY, so {engine_hash} hashes
|
|
163
|
+
# the loaded shared library itself (mirrors Python's binary_hash()).
|
|
164
|
+
@fn_ruleset_version = optional_static_fn(handle, "cloak_ruleset_version")
|
|
165
|
+
@fn_ruleset_hash = optional_static_fn(handle, "cloak_engine_hash")
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Static engine version string (never freed — the ABI owns it).
|
|
169
|
+
def version
|
|
170
|
+
static_string(@fn_version)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Canonical fasit ruleset semver — the X-Cloak-Engine-Version identity the
|
|
174
|
+
# strict gateway's EnforceEngineFloor keys on (a missing version 428s under
|
|
175
|
+
# missing_identity_mode=strict). "" if the loaded engine predates the symbol.
|
|
176
|
+
def ruleset_version
|
|
177
|
+
static_string(@fn_ruleset_version)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Exact shared-ruleset digest reported by the loaded engine.
|
|
181
|
+
def ruleset_hash
|
|
182
|
+
static_string(@fn_ruleset_hash)
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Canonical X-Cloak-Engine-Hash — SHA-256 of the loaded shared-library
|
|
186
|
+
# binary, the authoritative id of the engine that actually tokenised
|
|
187
|
+
# on-device (mirrors Python's LocalEngine#binary_hash and the MCP server
|
|
188
|
+
# hashing its WASM binary). This is the ENGINE-BINARY sha, NOT the ruleset
|
|
189
|
+
# content digest — the gateway attestation/receipt must identify the binary
|
|
190
|
+
# that ran. {Engine.load_engine} hashes the bytes it maps BEFORE loading and
|
|
191
|
+
# loads from those same bytes, so this value is the hash of the loaded
|
|
192
|
+
# code, not of whatever sits at the path later. Only a handle built without
|
|
193
|
+
# that hash (a preloaded handle, a spec) falls back to reading +lib_path+.
|
|
194
|
+
# Memoised; "" when the loaded library's path is unknown or unreadable — an
|
|
195
|
+
# honest "no binary to attest", never a fabricated hash.
|
|
196
|
+
def engine_hash
|
|
197
|
+
return @lib_hash unless @lib_hash.nil?
|
|
198
|
+
return "" if @lib_path.nil? || @lib_path.to_s.empty?
|
|
199
|
+
|
|
200
|
+
@lib_hash = begin
|
|
201
|
+
Digest::SHA256.hexdigest(File.binread(@lib_path))
|
|
202
|
+
rescue SystemCallError, IOError
|
|
203
|
+
""
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Convert +data+ (binary String) to tokenised markdown ENTIRELY on-device.
|
|
208
|
+
#
|
|
209
|
+
# Raises EngineError if the file cannot be converted/tokenised here — the
|
|
210
|
+
# caller must NOT then fall back to sending the raw bytes silently.
|
|
211
|
+
# @param mode [String, nil] "balanced" (default) or "max". This is an
|
|
212
|
+
# EXPLICIT CALL-SITE argument, so it goes through
|
|
213
|
+
# {CloakAPI::SurrogateMode.resolve}: a bad literal RAISES
|
|
214
|
+
# {CloakAPI::SurrogateMode::Error} rather than being silently corrected.
|
|
215
|
+
# Stored values should be run through +SurrogateMode.coerce+ first.
|
|
216
|
+
# **The mode selects STYLE ONLY** — it can never gate whether
|
|
217
|
+
# tokenisation happens.
|
|
218
|
+
# @param bindings [Hash, nil] the conversation ledger from a previous
|
|
219
|
+
# result. Supplying it (even as +{}+) puts the call in CONVERSATION
|
|
220
|
+
# scope, so the same real value keeps the same stand-in across turns and
|
|
221
|
+
# the result reports <tt>continuity: "conversation"</tt>. Omit it and the
|
|
222
|
+
# result honestly reports <tt>"single-message"</tt>.
|
|
223
|
+
def tokenise_file(data, mime: nil, filename: nil, names: nil, seed_hex: nil,
|
|
224
|
+
mode: nil, bindings: nil, accepted: nil)
|
|
225
|
+
opts = {}
|
|
226
|
+
opts["filename"] = filename if filename && !filename.empty?
|
|
227
|
+
opts["names"] = names if names && !names.empty?
|
|
228
|
+
opts["seed_hex"] = seed_hex if seed_hex && !seed_hex.to_s.empty?
|
|
229
|
+
opts["mode"] = CloakAPI::SurrogateMode.resolve(mode)
|
|
230
|
+
# Presence of the ledger is what puts the call in CONVERSATION scope, so
|
|
231
|
+
# {} and nil are DIFFERENT and must stay different.
|
|
232
|
+
opts["bindings"] = bindings unless bindings.nil?
|
|
233
|
+
# PHP LocalEngine::tokeniseFile sets opts['accepted'] = Degraded::allowed().
|
|
234
|
+
# Java LocalEngine.putReducedCoverageAcceptance does the same.
|
|
235
|
+
# Python LocalEngine.tokenise_file sets opts["accepted"] = True.
|
|
236
|
+
# TypeScript tokeniseText sets opts.accepted the same way; file bytes go
|
|
237
|
+
# through the same C ABI, which returns code 5 unless the key is present.
|
|
238
|
+
# Standard policy sends true and continues with a warning. Strict sends false.
|
|
239
|
+
opts["accepted"] = CloakAPI::Degraded.allowed?(accepted)
|
|
240
|
+
begin
|
|
241
|
+
payload = invoke(data.b, mime.to_s, JSON.generate(opts))
|
|
242
|
+
rescue EngineError => e
|
|
243
|
+
# Code 5 is ERR_REDUCED_COVERAGE. Strict handling refuses here.
|
|
244
|
+
# When the policy allows it, the engine returns a body instead of code 5;
|
|
245
|
+
# a code 5 that still arrives is a refusal we do not turn into a raw send.
|
|
246
|
+
if e.message.include?("engine code 5")
|
|
247
|
+
CloakAPI::Degraded.guard!("tokenise on-device", accepted, CloakAPI::Degraded::KIND_NO_NER)
|
|
248
|
+
end
|
|
249
|
+
raise
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
markdown = payload.fetch("markdown")
|
|
253
|
+
CloakAPI::Degraded.assert_full_coverage!(payload["meta"], accepted: accepted)
|
|
254
|
+
meta = payload["meta"]
|
|
255
|
+
if meta.is_a?(Hash) && meta["ner_enabled"] == false
|
|
256
|
+
CloakAPI::Warnings.emit(
|
|
257
|
+
CloakAPI::Warnings.reduced("model_missing", CloakAPI::Warnings::MODEL_MISSING_MESSAGE)
|
|
258
|
+
)
|
|
259
|
+
end
|
|
260
|
+
token_map = detok_table(markdown, payload)
|
|
261
|
+
LocalTokeniseResult.new(
|
|
262
|
+
markdown: markdown,
|
|
263
|
+
token_map: token_map,
|
|
264
|
+
meta: payload["meta"] || {},
|
|
265
|
+
surrogate_mode: CloakAPI::SurrogateMode::Report.from_payload(payload["surrogate_mode"]),
|
|
266
|
+
bindings: payload["bindings"] || {}
|
|
267
|
+
)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# Tokenise a plain-text prompt ENTIRELY on-device through the SAME shared
|
|
271
|
+
# engine (fasit ruleset + checksum validators + native NER) the drop-in
|
|
272
|
+
# proxy / desktop app / pii-engine mode use — so a prompt is protected
|
|
273
|
+
# identically on every surface (masterplan §11a/§12a, one engine
|
|
274
|
+
# everywhere). Text is fed as +text/plain+ through the file-pipeline.
|
|
275
|
+
#
|
|
276
|
+
# Raises EngineError only if the engine genuinely cannot process the input
|
|
277
|
+
# (there is no such case for plain text today) — the caller must NOT then
|
|
278
|
+
# fall back to sending the raw text. Secrets are tokenised+flagged, not
|
|
279
|
+
# blocked (a credential returns a +<<CLOAK:CRED_KEY:…>>+ token, never a raise).
|
|
280
|
+
# +mode+ / +bindings+ behave exactly as documented on {#tokenise_file}.
|
|
281
|
+
def tokenise_text(text, names: nil, seed_hex: nil, mode: nil, bindings: nil, accepted: nil)
|
|
282
|
+
opts = { "filename" => "prompt.txt" }
|
|
283
|
+
opts["names"] = names if names && !names.empty?
|
|
284
|
+
# C21b — continuity seed for cross-session stable stand-ins (forward-safe:
|
|
285
|
+
# the engine ignores unknown keys on an older build).
|
|
286
|
+
opts["seed_hex"] = seed_hex if seed_hex && !seed_hex.to_s.empty?
|
|
287
|
+
# EXPLICIT call-site argument -> resolve (raises on a bad literal).
|
|
288
|
+
opts["mode"] = CloakAPI::SurrogateMode.resolve(mode)
|
|
289
|
+
opts["bindings"] = bindings unless bindings.nil?
|
|
290
|
+
# The C ABI requires explicit acceptance before it returns a rules-only
|
|
291
|
+
# text result. Standard policy accepts with a warning; strict settings
|
|
292
|
+
# still send false and remain a non-egress refusal.
|
|
293
|
+
opts["accepted"] = CloakAPI::Degraded.allowed?(accepted)
|
|
294
|
+
payload = invoke(text.to_s.b, "text/plain", JSON.generate(opts))
|
|
295
|
+
CloakAPI::Degraded.assert_full_coverage!(payload["meta"], accepted: accepted)
|
|
296
|
+
|
|
297
|
+
tokenised = payload.fetch("markdown")
|
|
298
|
+
token_map = detok_table(tokenised, payload)
|
|
299
|
+
categories = tokenised.scan(TOKEN_LABEL_RE).flatten.uniq.sort
|
|
300
|
+
LocalTextResult.new(
|
|
301
|
+
text: tokenised,
|
|
302
|
+
token_map: token_map,
|
|
303
|
+
categories: categories,
|
|
304
|
+
surrogate_mode: CloakAPI::SurrogateMode::Report.from_payload(payload["surrogate_mode"]),
|
|
305
|
+
bindings: payload["bindings"] || {},
|
|
306
|
+
meta: payload["meta"] || {}
|
|
307
|
+
)
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
private
|
|
311
|
+
|
|
312
|
+
# One round-trip across the C ABI. Allocates the two out-parameter slots,
|
|
313
|
+
# calls the engine, and on a non-zero code reads + frees the error string
|
|
314
|
+
# then raises (fail-closed). On success parses + frees the JSON string.
|
|
315
|
+
def invoke(bytes, mime, opts_json)
|
|
316
|
+
out = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP, Fiddle::RUBY_FREE)
|
|
317
|
+
err = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP, Fiddle::RUBY_FREE)
|
|
318
|
+
# Zero the char* slots so a null check is meaningful even if the engine
|
|
319
|
+
# left one untouched.
|
|
320
|
+
zero = "\x00" * Fiddle::SIZEOF_VOIDP
|
|
321
|
+
out[0, Fiddle::SIZEOF_VOIDP] = zero
|
|
322
|
+
err[0, Fiddle::SIZEOF_VOIDP] = zero
|
|
323
|
+
|
|
324
|
+
code = @fn_tokenise.call(bytes, bytes.bytesize, mime, opts_json, out, err)
|
|
325
|
+
|
|
326
|
+
if code != 0
|
|
327
|
+
# The native string is untrusted: it can quote an archive entry name
|
|
328
|
+
# or a piece of the document. Free it, keep the numeric code.
|
|
329
|
+
read_and_free(err)
|
|
330
|
+
raise EngineError, "on-device tokenisation failed (engine code #{code})"
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
json = read_and_free(out)
|
|
334
|
+
raise EngineError, "engine returned empty result" if json.empty?
|
|
335
|
+
|
|
336
|
+
JSON.parse(json)
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
# Dereference a char** slot to the heap char*, copy it into a Ruby String,
|
|
340
|
+
# then hand the C string back to +cloak_string_free+. Returns "" for null.
|
|
341
|
+
def read_and_free(slot)
|
|
342
|
+
str_ptr = slot.ptr # *(char**)slot
|
|
343
|
+
return "" if str_ptr.null?
|
|
344
|
+
|
|
345
|
+
value = str_ptr.to_s.force_encoding("UTF-8")
|
|
346
|
+
@fn_free.call(str_ptr)
|
|
347
|
+
value
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# Bind a static-string C getter if the symbol exists; nil otherwise (so a
|
|
351
|
+
# binding never hard-fails on an older engine that lacks the newer symbol).
|
|
352
|
+
def optional_static_fn(handle, name)
|
|
353
|
+
Fiddle::Function.new(handle[name], [], Fiddle::TYPE_VOIDP)
|
|
354
|
+
rescue Fiddle::DLError, StandardError
|
|
355
|
+
nil
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# Read a 'static NUL-terminated C string returned by a getter (never freed —
|
|
359
|
+
# the ABI owns it). "" for a nil getter or a null pointer.
|
|
360
|
+
def static_string(fn)
|
|
361
|
+
return "" if fn.nil?
|
|
362
|
+
|
|
363
|
+
ptr = fn.call
|
|
364
|
+
ptr = Fiddle::Pointer.new(ptr.to_i) unless ptr.is_a?(Fiddle::Pointer)
|
|
365
|
+
ptr.null? ? "" : ptr.to_s.force_encoding("UTF-8")
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
# The AUTHORITATIVE local detokenise table for these bytes.
|
|
369
|
+
#
|
|
370
|
+
# PRIMARY: the engine's +replacements+ map, keyed by the FULL replacement
|
|
371
|
+
# string. This is the only table that works in BOTH modes — in BALANCED the
|
|
372
|
+
# replacement is a realistic string ("Sandra Hall"), not a
|
|
373
|
+
# <<CLOAK:TYPE:hash>> token, so it has no hash to key on and **a BALANCED
|
|
374
|
+
# reply cannot be restored from the hash-keyed +mappings+**. Before
|
|
375
|
+
# 2026-07-29 this SDK used only the hash rebuild below, so with the engine's
|
|
376
|
+
# default (BALANCED) style every reply silently came back un-restored:
|
|
377
|
+
# +token_map+ was +{}+ and +detokenize+ was a no-op.
|
|
378
|
+
#
|
|
379
|
+
# FALLBACK: the legacy hash-keyed +mappings+ + a scan of the emitted
|
|
380
|
+
# <<CLOAK:...>> tokens, kept so an OLDER engine binary (one that predates
|
|
381
|
+
# +replacements+) still detokenises its MAX-style output correctly.
|
|
382
|
+
def detok_table(text, payload)
|
|
383
|
+
replacements = payload["replacements"]
|
|
384
|
+
return replacements.to_h { |k, v| [k.to_s, v.to_s] } if replacements.is_a?(Hash) && !replacements.empty?
|
|
385
|
+
|
|
386
|
+
rebuild_token_map(text, payload["mappings"] || {})
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
# LEGACY fallback. The engine returns +hash => original+; rebuild the
|
|
390
|
+
# +full-token => original+ map by scanning the tokens actually emitted.
|
|
391
|
+
# Only reachable for a pre-+replacements+ engine build, and only capable of
|
|
392
|
+
# expressing MAX-style output.
|
|
393
|
+
def rebuild_token_map(text, hash_to_original)
|
|
394
|
+
map = {}
|
|
395
|
+
text.scan(TOKEN_RE) do
|
|
396
|
+
m = Regexp.last_match
|
|
397
|
+
full_token = m[0]
|
|
398
|
+
hash_val = m[1]
|
|
399
|
+
map[full_token] = hash_to_original[hash_val] if hash_to_original.key?(hash_val)
|
|
400
|
+
end
|
|
401
|
+
map
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
# What the SDK decided about the on-device engine for this process.
|
|
406
|
+
#
|
|
407
|
+
# used - true when the engine is loaded and will tokenise text
|
|
408
|
+
# reason - "" when used, else one of engine_missing, engine_unloadable,
|
|
409
|
+
# pin_missing, pin_mismatch, ruleset_mismatch
|
|
410
|
+
# message - plain English, safe to show; holds no request text
|
|
411
|
+
# sha256 - SHA-256 of the engine bytes that were loaded ("" when not used)
|
|
412
|
+
# source - "configured" (CLOAKAPI_ENGINE_LIB), "bundled" (inside the gem) or ""
|
|
413
|
+
Status = Data.define(:used, :reason, :message, :sha256, :source) do
|
|
414
|
+
def used? = used
|
|
415
|
+
def warning = used ? nil : CloakAPI::Warnings.reduced(reason, message)
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
ENV_LIB = "CLOAKAPI_ENGINE_LIB"
|
|
419
|
+
ENV_PIN = "CLOAKAPI_ENGINE_SHA256_PIN"
|
|
420
|
+
|
|
421
|
+
# The native engine builds of the release set this gem belongs to
|
|
422
|
+
# (repo/release-set.json, checked by release-set-check.mjs). An engine file
|
|
423
|
+
# shipped INSIDE the gem is accepted without a pin only when its SHA-256 is
|
|
424
|
+
# the one listed here for the platform. An engine named by
|
|
425
|
+
# CLOAKAPI_ENGINE_LIB always needs CLOAKAPI_ENGINE_SHA256_PIN.
|
|
426
|
+
RELEASE_SET_NATIVE = {
|
|
427
|
+
"linux-x86_64" => "b2aec8d6d90ead4328258270c41a79dda4e1f71efa79025f308ce8e09f080744",
|
|
428
|
+
"linux-aarch64" => "a8bfb99346cdaf6e47b13214e7f1f872f95e0c2fe040cf6bb3e6a82f1d26ed30"
|
|
429
|
+
}.freeze
|
|
430
|
+
|
|
431
|
+
# The released native engine is built against this glibc.
|
|
432
|
+
MIN_GLIBC = "2.39"
|
|
433
|
+
|
|
434
|
+
RULES_NOTE = "The SDK's built-in rules are used instead; names and some other personal data " \
|
|
435
|
+
"may be sent as written."
|
|
436
|
+
|
|
437
|
+
@gate_mutex = Mutex.new
|
|
438
|
+
|
|
439
|
+
class << self
|
|
440
|
+
# Platform-appropriate shared-library basenames for the engine.
|
|
441
|
+
def lib_filenames
|
|
442
|
+
case RbConfig::CONFIG["host_os"]
|
|
443
|
+
when /darwin/ then ["libcloak_engine.dylib"]
|
|
444
|
+
when /mswin|mingw|cygwin/ then ["cloak_engine.dll"]
|
|
445
|
+
else ["libcloak_engine.so"]
|
|
446
|
+
end
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
# "linux-x86_64", "linux-aarch64", "macos-aarch64", "windows-x86_64", ...
|
|
450
|
+
def platform_key
|
|
451
|
+
os = case RbConfig::CONFIG["host_os"]
|
|
452
|
+
when /darwin/ then "macos"
|
|
453
|
+
when /mswin|mingw|cygwin/ then "windows"
|
|
454
|
+
when /linux/ then "linux"
|
|
455
|
+
else RbConfig::CONFIG["host_os"].to_s
|
|
456
|
+
end
|
|
457
|
+
cpu = RbConfig::CONFIG["host_cpu"].to_s.downcase
|
|
458
|
+
cpu = "x86_64" if %w[amd64 x64 x86_64].include?(cpu)
|
|
459
|
+
cpu = "aarch64" if %w[arm64 aarch64].include?(cpu)
|
|
460
|
+
"#{os}-#{cpu}"
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
# Where an engine file may come from, in order: the path in
|
|
464
|
+
# CLOAKAPI_ENGINE_LIB, then a file shipped inside the gem. Nothing else
|
|
465
|
+
# is searched: an engine the integrator did not name and the gem does
|
|
466
|
+
# not carry is not used.
|
|
467
|
+
def candidate_paths
|
|
468
|
+
candidates = []
|
|
469
|
+
env = ENV[ENV_LIB]
|
|
470
|
+
candidates << env if env && !env.empty?
|
|
471
|
+
candidates.concat(bundled_paths)
|
|
472
|
+
candidates
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
def bundled_paths
|
|
476
|
+
here = __dir__ # .../lib/cloakapi
|
|
477
|
+
# Only the platform sub-directory: that is where a platform build of the
|
|
478
|
+
# gem carries the engine. A file directly in _lib is not used.
|
|
479
|
+
lib_filenames.map { |name| File.join(here, "_lib", platform_key, name) }
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
# The decision for this process: is the engine used, and if not, why.
|
|
483
|
+
# Made once, on first use, and kept. Never raises.
|
|
484
|
+
def status
|
|
485
|
+
@gate_mutex.synchronize do
|
|
486
|
+
unless defined?(@loaded) && @loaded
|
|
487
|
+
@status, @engine = decide
|
|
488
|
+
@loaded = true
|
|
489
|
+
end
|
|
490
|
+
@status
|
|
491
|
+
end
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
# The process-wide LocalEngine, or nil when the engine is not used (see
|
|
495
|
+
# {status} for the reason). Never raises.
|
|
496
|
+
def get
|
|
497
|
+
status
|
|
498
|
+
@engine
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
# True iff the on-device engine is loaded and used in this process.
|
|
502
|
+
def available?
|
|
503
|
+
status.used?
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
# +[version, hash]+ for the engine identity headers. +hash+ is the SHA-256
|
|
507
|
+
# of the loaded engine file, or "" when no engine is used; the relay then
|
|
508
|
+
# states no engine identity at all. Never raises.
|
|
509
|
+
def identity
|
|
510
|
+
current = status
|
|
511
|
+
return [RULESET_VERSION, ""] unless current.used?
|
|
512
|
+
|
|
513
|
+
[RULESET_VERSION, current.sha256]
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
# Reset the memoised decision (test hook).
|
|
517
|
+
def reset!
|
|
518
|
+
@gate_mutex.synchronize do
|
|
519
|
+
@loaded = false
|
|
520
|
+
@engine = nil
|
|
521
|
+
@status = nil
|
|
522
|
+
end
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
# A short sentence about the C library of this system when it explains
|
|
526
|
+
# why the released engine cannot load here; nil otherwise.
|
|
527
|
+
def libc_note
|
|
528
|
+
unless RbConfig::CONFIG["host_os"].to_s.include?("linux")
|
|
529
|
+
return "Native engine builds exist for Linux (x86-64 and aarch64) only."
|
|
530
|
+
end
|
|
531
|
+
return "The engine needs glibc #{MIN_GLIBC} or newer; this system uses musl, which is not supported." if musl?
|
|
532
|
+
|
|
533
|
+
have = glibc_version
|
|
534
|
+
return nil if have.nil?
|
|
535
|
+
return nil if Gem::Version.new(have) >= Gem::Version.new(MIN_GLIBC)
|
|
536
|
+
|
|
537
|
+
"The engine needs glibc #{MIN_GLIBC} or newer (this system has glibc #{have})."
|
|
538
|
+
rescue StandardError
|
|
539
|
+
nil
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
private
|
|
543
|
+
|
|
544
|
+
def musl?
|
|
545
|
+
%w[/lib /lib64 /usr/lib].any? { |dir| !Dir.glob(File.join(dir, "ld-musl-*.so.1")).empty? }
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
def glibc_version
|
|
549
|
+
return nil unless defined?(::Fiddle)
|
|
550
|
+
|
|
551
|
+
fn = Fiddle::Function.new(Fiddle.dlopen(nil)["gnu_get_libc_version"], [], Fiddle::TYPE_VOIDP)
|
|
552
|
+
ptr = fn.call
|
|
553
|
+
ptr = Fiddle::Pointer.new(ptr.to_i) unless ptr.is_a?(Fiddle::Pointer)
|
|
554
|
+
text = ptr.null? ? "" : ptr.to_s
|
|
555
|
+
text.match?(/\A\d+\.\d+(?:\.\d+)?\z/) ? text : nil
|
|
556
|
+
rescue StandardError
|
|
557
|
+
nil
|
|
558
|
+
end
|
|
559
|
+
|
|
560
|
+
def unavailable(reason, message)
|
|
561
|
+
[Status.new(used: false, reason: reason, message: message, sha256: "", source: ""), nil]
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
def with_note(*parts)
|
|
565
|
+
parts.compact.reject(&:empty?).join(" ")
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
def pin_matches?(pin, digest)
|
|
569
|
+
pin.match?(/\A[0-9a-fA-F]{64}\z/) && pin.casecmp?(digest)
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
# The pin rule (one rule for every CloakAPI SDK). A wrong, missing or
|
|
573
|
+
# unreadable engine never stops a send: the engine is treated as not
|
|
574
|
+
# there and the built-in rules are used, with a notice that names the
|
|
575
|
+
# reason.
|
|
576
|
+
def decide
|
|
577
|
+
# Taken as written: a value with spaces around it is not a SHA-256.
|
|
578
|
+
pin = ENV[ENV_PIN].to_s
|
|
579
|
+
configured = ENV[ENV_LIB].to_s
|
|
580
|
+
return decide_configured(configured, pin) unless configured.empty?
|
|
581
|
+
|
|
582
|
+
bundled = bundled_paths.find { |path| File.file?(path) }
|
|
583
|
+
return decide_bundled(bundled, pin) if bundled
|
|
584
|
+
|
|
585
|
+
if pin.empty?
|
|
586
|
+
unavailable("engine_missing", with_note("The CloakAPI on-device engine was not found.", RULES_NOTE))
|
|
587
|
+
else
|
|
588
|
+
unavailable("engine_missing",
|
|
589
|
+
with_note("The CloakAPI on-device engine was not found, so #{ENV_PIN} could not be checked.",
|
|
590
|
+
RULES_NOTE))
|
|
591
|
+
end
|
|
592
|
+
rescue StandardError
|
|
593
|
+
unavailable("engine_unloadable",
|
|
594
|
+
with_note("The CloakAPI on-device engine could not be set up on this system.", RULES_NOTE))
|
|
595
|
+
end
|
|
596
|
+
|
|
597
|
+
def decide_configured(path, pin)
|
|
598
|
+
where = "the engine at #{ENV_LIB}"
|
|
599
|
+
unless File.file?(path)
|
|
600
|
+
return unavailable("engine_missing",
|
|
601
|
+
with_note("The engine file named by #{ENV_LIB} was not found.", RULES_NOTE))
|
|
602
|
+
end
|
|
603
|
+
bytes = read_engine_bytes(path)
|
|
604
|
+
if bytes.nil?
|
|
605
|
+
return unavailable("engine_unloadable",
|
|
606
|
+
with_note("The engine file named by #{ENV_LIB} could not be read.", RULES_NOTE))
|
|
607
|
+
end
|
|
608
|
+
digest = Digest::SHA256.hexdigest(bytes)
|
|
609
|
+
if pin.empty?
|
|
610
|
+
return unavailable("pin_missing", with_note("#{ENV_PIN} is not set, so #{where} was not used.", RULES_NOTE))
|
|
611
|
+
end
|
|
612
|
+
unless pin_matches?(pin, digest)
|
|
613
|
+
return unavailable("pin_mismatch",
|
|
614
|
+
with_note("#{ENV_PIN} does not match the SHA-256 of #{where}, so that engine was not used.",
|
|
615
|
+
RULES_NOTE))
|
|
616
|
+
end
|
|
617
|
+
load_checked(bytes, digest, File.basename(path), "configured")
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
def decide_bundled(path, pin)
|
|
621
|
+
bytes = read_engine_bytes(path)
|
|
622
|
+
if bytes.nil?
|
|
623
|
+
return unavailable("engine_unloadable",
|
|
624
|
+
with_note("The engine file shipped inside the gem could not be read.", RULES_NOTE))
|
|
625
|
+
end
|
|
626
|
+
digest = Digest::SHA256.hexdigest(bytes)
|
|
627
|
+
released = RELEASE_SET_NATIVE[platform_key] == digest
|
|
628
|
+
pinned = pin_matches?(pin, digest)
|
|
629
|
+
if !pin.empty? && !pinned
|
|
630
|
+
return unavailable("pin_mismatch",
|
|
631
|
+
with_note("#{ENV_PIN} does not match the SHA-256 of the engine file shipped inside the gem, " \
|
|
632
|
+
"so that engine was not used.", RULES_NOTE))
|
|
633
|
+
end
|
|
634
|
+
unless released || pinned
|
|
635
|
+
return unavailable("pin_missing",
|
|
636
|
+
with_note("The engine file shipped inside the gem is not a build this SDK release names " \
|
|
637
|
+
"and #{ENV_PIN} is not set, so that engine was not used.", RULES_NOTE))
|
|
638
|
+
end
|
|
639
|
+
load_checked(bytes, digest, File.basename(path), "bundled")
|
|
640
|
+
end
|
|
641
|
+
|
|
642
|
+
def read_engine_bytes(path)
|
|
643
|
+
File.binread(path)
|
|
644
|
+
rescue SystemCallError, IOError
|
|
645
|
+
nil
|
|
646
|
+
end
|
|
647
|
+
|
|
648
|
+
# Load a private copy of exactly +bytes+ and accept it only when it
|
|
649
|
+
# reports the rule set this SDK was built for.
|
|
650
|
+
def load_checked(bytes, digest, basename, source)
|
|
651
|
+
unless defined?(::Fiddle)
|
|
652
|
+
return unavailable("engine_unloadable",
|
|
653
|
+
with_note("Ruby's fiddle library is not available, so the engine file could not be loaded " \
|
|
654
|
+
"(on Ruby 3.5 or newer add gem \"fiddle\" to the Gemfile).", RULES_NOTE))
|
|
655
|
+
end
|
|
656
|
+
engine = begin
|
|
657
|
+
load_private_copy(bytes, digest, basename)
|
|
658
|
+
rescue LoadError, StandardError
|
|
659
|
+
return unavailable("engine_unloadable",
|
|
660
|
+
with_note("The CloakAPI engine file was found but could not be loaded on this system.",
|
|
661
|
+
libc_note, RULES_NOTE))
|
|
662
|
+
end
|
|
663
|
+
version = engine.ruleset_version.to_s
|
|
664
|
+
ruleset_hash = engine.ruleset_hash.to_s
|
|
665
|
+
unless version == RULESET_VERSION && ruleset_hash == CloakAPI::Pii::Ruleset.sha256
|
|
666
|
+
reported = version.match?(/\A[0-9A-Za-z.+-]{1,32}\z/) ? version : "an unknown version"
|
|
667
|
+
said = version == RULESET_VERSION ? "rule set #{reported} with other contents" : "rule set #{reported}"
|
|
668
|
+
return unavailable("ruleset_mismatch",
|
|
669
|
+
with_note("The engine reports #{said}; this SDK expects rule set #{RULESET_VERSION}, " \
|
|
670
|
+
"so that engine was not used.", RULES_NOTE))
|
|
671
|
+
end
|
|
672
|
+
[Status.new(used: true, reason: "", message: "The CloakAPI on-device engine is loaded.",
|
|
673
|
+
sha256: digest, source: source), engine]
|
|
674
|
+
end
|
|
675
|
+
|
|
676
|
+
# Write the already-hashed bytes to a private copy and +dlopen+ the copy.
|
|
677
|
+
# The pin check therefore covers the code that is actually mapped: a file
|
|
678
|
+
# swapped at the original path before or after the load changes nothing,
|
|
679
|
+
# because the loader never sees that path. The directory is created 0700
|
|
680
|
+
# and the copy exclusively, so it is not shared with other users. The copy
|
|
681
|
+
# is removed once the library is mapped; a platform that cannot delete a
|
|
682
|
+
# mapped file keeps it until process exit.
|
|
683
|
+
def load_private_copy(bytes, digest, basename)
|
|
684
|
+
dir = Dir.mktmpdir("cloakapi-engine-")
|
|
685
|
+
begin
|
|
686
|
+
private_path = File.join(dir, basename)
|
|
687
|
+
File.open(private_path, File::WRONLY | File::CREAT | File::EXCL, 0o700) do |f|
|
|
688
|
+
f.binmode
|
|
689
|
+
f.write(bytes)
|
|
690
|
+
end
|
|
691
|
+
LocalEngine.new(Fiddle.dlopen(private_path), private_path, digest)
|
|
692
|
+
ensure
|
|
693
|
+
begin
|
|
694
|
+
FileUtils.remove_entry(dir)
|
|
695
|
+
rescue SystemCallError
|
|
696
|
+
nil
|
|
697
|
+
end
|
|
698
|
+
end
|
|
699
|
+
end
|
|
700
|
+
end
|
|
701
|
+
end
|
|
702
|
+
end
|