cloakapi 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +160 -0
  3. data/LICENSE +202 -0
  4. data/NOTICE +18 -0
  5. data/README.md +407 -0
  6. data/THIRD-PARTY-NOTICES.md +22 -0
  7. data/TRADEMARKS.md +19 -0
  8. data/examples/fake_gateway.rb +78 -0
  9. data/examples/hello.rb +64 -0
  10. data/lib/cloakapi/client.rb +1272 -0
  11. data/lib/cloakapi/degraded.rb +77 -0
  12. data/lib/cloakapi/disclosure.rb +252 -0
  13. data/lib/cloakapi/engine.rb +702 -0
  14. data/lib/cloakapi/errors.rb +65 -0
  15. data/lib/cloakapi/pii/bundled_rules.rb +670 -0
  16. data/lib/cloakapi/pii/card_spans.rb +300 -0
  17. data/lib/cloakapi/pii/detector.rb +1925 -0
  18. data/lib/cloakapi/pii/patterns.json +2238 -0
  19. data/lib/cloakapi/pii/pii_match.rb +7 -0
  20. data/lib/cloakapi/pii/pii_type.rb +7 -0
  21. data/lib/cloakapi/pii/ruleset.rb +65 -0
  22. data/lib/cloakapi/pii/tokenizer.rb +411 -0
  23. data/lib/cloakapi/posture.rb +82 -0
  24. data/lib/cloakapi/receipts.rb +406 -0
  25. data/lib/cloakapi/resources/chat.rb +82 -0
  26. data/lib/cloakapi/resources/messages.rb +90 -0
  27. data/lib/cloakapi/resources/models.rb +25 -0
  28. data/lib/cloakapi/resources/receipts.rb +162 -0
  29. data/lib/cloakapi/resources/reconstruct.rb +120 -0
  30. data/lib/cloakapi/resources/request_strings.rb +485 -0
  31. data/lib/cloakapi/resources/reviewable.rb +163 -0
  32. data/lib/cloakapi/resources/tokens.rb +28 -0
  33. data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
  34. data/lib/cloakapi/routing.rb +500 -0
  35. data/lib/cloakapi/streaming.rb +196 -0
  36. data/lib/cloakapi/surrogate_mode.rb +233 -0
  37. data/lib/cloakapi/version.rb +15 -0
  38. data/lib/cloakapi/warnings.rb +137 -0
  39. data/lib/cloakapi.rb +37 -0
  40. data/licenses/Apache-2.0.txt +202 -0
  41. data/licenses/CC-BY-4.0.txt +104 -0
  42. data/licenses/MIT-Faker.txt +21 -0
  43. data/licenses/MIT.txt +21 -0
  44. metadata +149 -0
@@ -0,0 +1,702 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ # fiddle comes with Ruby up to 3.4 and is a bundled gem from 3.5. Without it
5
+ # (a Ruby built without libffi, or an application bundle on Ruby 3.5+ that
6
+ # does not list it) the SDK still loads and works on its built-in rules.
7
+ begin
8
+ require "fiddle"
9
+ rescue LoadError
10
+ nil
11
+ end
12
+ require "fileutils"
13
+ require "json"
14
+ require "rbconfig"
15
+ require "tmpdir"
16
+
17
+ require_relative "warnings"
18
+ require_relative "degraded"
19
+ require_relative "pii/ruleset"
20
+ require_relative "surrogate_mode"
21
+
22
+ module CloakAPI
23
+ # Client-side engine binding — Fiddle (stdlib) over the shared CloakAPI C ABI.
24
+ #
25
+ # This is the Ruby end of the "shared Rust engine, embedded per language via
26
+ # FFI" design. It loads +libcloak_engine+ (built from
27
+ # +packages/cloak-engine-ffi+, a cdylib over the SAME file-pipeline +
28
+ # pii-engine crates the desktop app and local proxy use) and turns **raw file
29
+ # bytes + MIME into tokenised markdown ENTIRELY on this machine** — so a
30
+ # document attachment (or a prompt) is protected before a single byte egresses
31
+ # to the gateway. Mirrors the reference implementation in
32
+ # +sdk/python/cloakapi/engine.py+.
33
+ #
34
+ # +fiddle+ is deliberate: it comes with Ruby, so there is no build step and
35
+ # no compiler. The only runtime artifact is the shared library, taken from
36
+ # (in order):
37
+ #
38
+ # 1. the path in +CLOAKAPI_ENGINE_LIB+, used only when
39
+ # +CLOAKAPI_ENGINE_SHA256_PIN+ equals the SHA-256 of that file;
40
+ # 2. a file shipped inside the gem (+cloakapi/_lib/+), used only when its
41
+ # SHA-256 is the one this release names for the platform.
42
+ #
43
+ # When no engine is used, {available?} is false, {status} says why, and text
44
+ # is handled by the SDK's built-in rules with a notice. A wrong or missing
45
+ # engine never stops a send.
46
+ module Engine
47
+ # The local engine was present but refused to process this input
48
+ # (unsupported/encrypted/undecodable file). The caller must NOT then fall back
49
+ # to sending the raw text/bytes — fail closed.
50
+ #
51
+ # NOTE: secrets (API keys/cards/IBANs/national-IDs) are NOT an EngineError —
52
+ # the policy is TOKENISE-AND-FLAG, not block. A credential is tokenised to a
53
+ # +<<CLOAK:CRED_KEY:…>>+ token like any other entity (golden vector
54
+ # +text-secret-apikey+) and the call returns normally.
55
+ class EngineError < StandardError; end
56
+
57
+ # Canonical shared-ruleset (fasit) semver — the ALWAYS-present
58
+ # X-Cloak-Engine-Version fallback used when no engine binary is loaded to
59
+ # report its own. Tracks shared/pii-patterns/patterns.json "version" and
60
+ # mirrors the Go/TS/Rust bindings' RULESET_VERSION (ADR-0014 §3). The strict
61
+ # gateway 428s a relay that presents no version under
62
+ # missing_identity_mode=strict, so this header is never dropped.
63
+ # DERIVED, never hand-typed: read from the ruleset the SDK actually ships
64
+ # (lib/cloakapi/pii/patterns.json), which spec/ruleset_conformance_spec.rb
65
+ # byte-locks to shared/pii-patterns/patterns.json. A hand-typed literal here
66
+ # had already drifted (1.4.29 while canonical was 1.6.0), i.e. the SDK
67
+ # declared a ruleset it was not running.
68
+ RULESET_VERSION = CloakAPI::Pii::Ruleset.version
69
+
70
+ # Matches the engine's own token grammar: <<CLOAK:TYPE:6-hex>>.
71
+ TOKEN_RE = /<<CLOAK:[A-Z_]+:([0-9a-f]{6})>>/.freeze
72
+ # Type label extracted from each emitted token, for category reporting.
73
+ TOKEN_LABEL_RE = /<<CLOAK:([A-Z_]+):[0-9a-f]{6}>>/.freeze
74
+
75
+ # Outcome of a fully on-device *text* tokenisation (the prompt path).
76
+ # text — tokenised text; the ONLY field meant to leave the machine.
77
+ # token_map — REPLACEMENT STRING => original; kept LOCAL to restore the
78
+ # reply. Built from the engine's +replacements+ table, NOT
79
+ # from the legacy hash-keyed +mappings+: in BALANCED the
80
+ # replacement is not a token at all, so a hash-keyed map
81
+ # cannot express it and a BALANCED reply could not be
82
+ # restored from it.
83
+ # categories — cloud-token labels that fired (e.g. ["CONTACT_EM","ID_GOV"]).
84
+ # MAX-only signal: BALANCED emits realistic strings, not
85
+ # <<CLOAK:TYPE:hash>> tokens, so no label is recoverable
86
+ # from the bytes. A REPORTING difference, never a detection
87
+ # difference — the span set is identical in both modes.
88
+ # surrogate_mode — REQUIRED {CloakAPI::SurrogateMode::Report}: what this pass
89
+ # ACTUALLY did. Data.define makes it a required keyword, so
90
+ # a hand-written fake result raises at construction.
91
+ # bindings — the conversation ledger, ADVANCED by this call.
92
+ # RE-IDENTIFICATION MATERIAL: it maps stand-ins back to real
93
+ # values. CLIENT-SIDE ONLY — never egress it, never log it,
94
+ # never put it in a receipt, exactly like the token map.
95
+ LocalTextResult = Data.define(:text, :token_map, :categories, :surrogate_mode, :bindings, :meta) do
96
+ def initialize(text:, token_map:, categories:, surrogate_mode:, bindings:, meta: {})
97
+ super(text: text, token_map: token_map, categories: categories,
98
+ surrogate_mode: surrogate_mode, bindings: bindings, meta: meta)
99
+ end
100
+
101
+ def inspect
102
+ "#<#{self.class} sensitive fields redacted>"
103
+ end
104
+
105
+ alias to_s inspect
106
+
107
+ def marshal_dump
108
+ raise TypeError, "CloakAPI tokenisation results cannot be marshalled"
109
+ end
110
+ end
111
+
112
+ # Outcome of a fully on-device *file* tokenisation. Same field contract as
113
+ # {LocalTextResult}; +markdown+ replaces +text+ and +meta+ carries the
114
+ # diagnostics (resolved mime, extractor tag, pii_count, chars…).
115
+ LocalTokeniseResult = Data.define(:markdown, :token_map, :meta, :surrogate_mode, :bindings) do
116
+ def inspect
117
+ "#<#{self.class} sensitive fields redacted>"
118
+ end
119
+
120
+ alias to_s inspect
121
+
122
+ def marshal_dump
123
+ raise TypeError, "CloakAPI tokenisation results cannot be marshalled"
124
+ end
125
+ end
126
+
127
+ # Thin, safe wrapper over the +cloak_engine+ C ABI.
128
+ class LocalEngine
129
+ # +lib_hash+ (optional) is the SHA-256 of the exact bytes the +handle+ was
130
+ # loaded from. {Engine.load_engine} passes it so the identity is never
131
+ # recomputed from a path that can change after the library was mapped.
132
+ def initialize(handle, lib_path = nil, lib_hash = nil)
133
+ @handle = handle
134
+ @lib_path = lib_path
135
+ @lib_hash = lib_hash
136
+ @fn_tokenise = Fiddle::Function.new(
137
+ handle["cloak_tokenise_file"],
138
+ [
139
+ Fiddle::TYPE_VOIDP, # const uint8_t* bytes
140
+ Fiddle::TYPE_SIZE_T, # size_t len
141
+ Fiddle::TYPE_VOIDP, # const char* mime
142
+ Fiddle::TYPE_VOIDP, # const char* opts_json
143
+ Fiddle::TYPE_VOIDP, # char** out_json
144
+ Fiddle::TYPE_VOIDP # char** err
145
+ ],
146
+ Fiddle::TYPE_INT
147
+ )
148
+ @fn_free = Fiddle::Function.new(
149
+ handle["cloak_string_free"], [Fiddle::TYPE_VOIDP], Fiddle::TYPE_VOID
150
+ )
151
+ @fn_version = Fiddle::Function.new(
152
+ handle["cloak_engine_version"], [], Fiddle::TYPE_VOIDP
153
+ )
154
+ # ADR-0014 engine-identity symbol (locked 2026-07-12). Bound OPTIONALLY:
155
+ # an older engine build may predate it, in which case the header falls
156
+ # back to the RULESET_VERSION constant.
157
+ # cloak_ruleset_version() -> X-Cloak-Engine-Version (fasit semver)
158
+ # cloak_engine_hash() -> SHA-256 of the exact shared ruleset bytes
159
+ # NOTE: X-Cloak-Engine-Hash is NOT sourced from an FFI symbol — the
160
+ # +cloak_engine_hash()+ export returns the RULESET content digest, which
161
+ # identifies the ruleset, NOT the binary that ran. The gateway
162
+ # attestation must identify the ENGINE BINARY, so {engine_hash} hashes
163
+ # the loaded shared library itself (mirrors Python's binary_hash()).
164
+ @fn_ruleset_version = optional_static_fn(handle, "cloak_ruleset_version")
165
+ @fn_ruleset_hash = optional_static_fn(handle, "cloak_engine_hash")
166
+ end
167
+
168
+ # Static engine version string (never freed — the ABI owns it).
169
+ def version
170
+ static_string(@fn_version)
171
+ end
172
+
173
+ # Canonical fasit ruleset semver — the X-Cloak-Engine-Version identity the
174
+ # strict gateway's EnforceEngineFloor keys on (a missing version 428s under
175
+ # missing_identity_mode=strict). "" if the loaded engine predates the symbol.
176
+ def ruleset_version
177
+ static_string(@fn_ruleset_version)
178
+ end
179
+
180
+ # Exact shared-ruleset digest reported by the loaded engine.
181
+ def ruleset_hash
182
+ static_string(@fn_ruleset_hash)
183
+ end
184
+
185
+ # Canonical X-Cloak-Engine-Hash — SHA-256 of the loaded shared-library
186
+ # binary, the authoritative id of the engine that actually tokenised
187
+ # on-device (mirrors Python's LocalEngine#binary_hash and the MCP server
188
+ # hashing its WASM binary). This is the ENGINE-BINARY sha, NOT the ruleset
189
+ # content digest — the gateway attestation/receipt must identify the binary
190
+ # that ran. {Engine.load_engine} hashes the bytes it maps BEFORE loading and
191
+ # loads from those same bytes, so this value is the hash of the loaded
192
+ # code, not of whatever sits at the path later. Only a handle built without
193
+ # that hash (a preloaded handle, a spec) falls back to reading +lib_path+.
194
+ # Memoised; "" when the loaded library's path is unknown or unreadable — an
195
+ # honest "no binary to attest", never a fabricated hash.
196
+ def engine_hash
197
+ return @lib_hash unless @lib_hash.nil?
198
+ return "" if @lib_path.nil? || @lib_path.to_s.empty?
199
+
200
+ @lib_hash = begin
201
+ Digest::SHA256.hexdigest(File.binread(@lib_path))
202
+ rescue SystemCallError, IOError
203
+ ""
204
+ end
205
+ end
206
+
207
+ # Convert +data+ (binary String) to tokenised markdown ENTIRELY on-device.
208
+ #
209
+ # Raises EngineError if the file cannot be converted/tokenised here — the
210
+ # caller must NOT then fall back to sending the raw bytes silently.
211
+ # @param mode [String, nil] "balanced" (default) or "max". This is an
212
+ # EXPLICIT CALL-SITE argument, so it goes through
213
+ # {CloakAPI::SurrogateMode.resolve}: a bad literal RAISES
214
+ # {CloakAPI::SurrogateMode::Error} rather than being silently corrected.
215
+ # Stored values should be run through +SurrogateMode.coerce+ first.
216
+ # **The mode selects STYLE ONLY** — it can never gate whether
217
+ # tokenisation happens.
218
+ # @param bindings [Hash, nil] the conversation ledger from a previous
219
+ # result. Supplying it (even as +{}+) puts the call in CONVERSATION
220
+ # scope, so the same real value keeps the same stand-in across turns and
221
+ # the result reports <tt>continuity: "conversation"</tt>. Omit it and the
222
+ # result honestly reports <tt>"single-message"</tt>.
223
+ def tokenise_file(data, mime: nil, filename: nil, names: nil, seed_hex: nil,
224
+ mode: nil, bindings: nil, accepted: nil)
225
+ opts = {}
226
+ opts["filename"] = filename if filename && !filename.empty?
227
+ opts["names"] = names if names && !names.empty?
228
+ opts["seed_hex"] = seed_hex if seed_hex && !seed_hex.to_s.empty?
229
+ opts["mode"] = CloakAPI::SurrogateMode.resolve(mode)
230
+ # Presence of the ledger is what puts the call in CONVERSATION scope, so
231
+ # {} and nil are DIFFERENT and must stay different.
232
+ opts["bindings"] = bindings unless bindings.nil?
233
+ # PHP LocalEngine::tokeniseFile sets opts['accepted'] = Degraded::allowed().
234
+ # Java LocalEngine.putReducedCoverageAcceptance does the same.
235
+ # Python LocalEngine.tokenise_file sets opts["accepted"] = True.
236
+ # TypeScript tokeniseText sets opts.accepted the same way; file bytes go
237
+ # through the same C ABI, which returns code 5 unless the key is present.
238
+ # Standard policy sends true and continues with a warning. Strict sends false.
239
+ opts["accepted"] = CloakAPI::Degraded.allowed?(accepted)
240
+ begin
241
+ payload = invoke(data.b, mime.to_s, JSON.generate(opts))
242
+ rescue EngineError => e
243
+ # Code 5 is ERR_REDUCED_COVERAGE. Strict handling refuses here.
244
+ # When the policy allows it, the engine returns a body instead of code 5;
245
+ # a code 5 that still arrives is a refusal we do not turn into a raw send.
246
+ if e.message.include?("engine code 5")
247
+ CloakAPI::Degraded.guard!("tokenise on-device", accepted, CloakAPI::Degraded::KIND_NO_NER)
248
+ end
249
+ raise
250
+ end
251
+
252
+ markdown = payload.fetch("markdown")
253
+ CloakAPI::Degraded.assert_full_coverage!(payload["meta"], accepted: accepted)
254
+ meta = payload["meta"]
255
+ if meta.is_a?(Hash) && meta["ner_enabled"] == false
256
+ CloakAPI::Warnings.emit(
257
+ CloakAPI::Warnings.reduced("model_missing", CloakAPI::Warnings::MODEL_MISSING_MESSAGE)
258
+ )
259
+ end
260
+ token_map = detok_table(markdown, payload)
261
+ LocalTokeniseResult.new(
262
+ markdown: markdown,
263
+ token_map: token_map,
264
+ meta: payload["meta"] || {},
265
+ surrogate_mode: CloakAPI::SurrogateMode::Report.from_payload(payload["surrogate_mode"]),
266
+ bindings: payload["bindings"] || {}
267
+ )
268
+ end
269
+
270
+ # Tokenise a plain-text prompt ENTIRELY on-device through the SAME shared
271
+ # engine (fasit ruleset + checksum validators + native NER) the drop-in
272
+ # proxy / desktop app / pii-engine mode use — so a prompt is protected
273
+ # identically on every surface (masterplan §11a/§12a, one engine
274
+ # everywhere). Text is fed as +text/plain+ through the file-pipeline.
275
+ #
276
+ # Raises EngineError only if the engine genuinely cannot process the input
277
+ # (there is no such case for plain text today) — the caller must NOT then
278
+ # fall back to sending the raw text. Secrets are tokenised+flagged, not
279
+ # blocked (a credential returns a +<<CLOAK:CRED_KEY:…>>+ token, never a raise).
280
+ # +mode+ / +bindings+ behave exactly as documented on {#tokenise_file}.
281
+ def tokenise_text(text, names: nil, seed_hex: nil, mode: nil, bindings: nil, accepted: nil)
282
+ opts = { "filename" => "prompt.txt" }
283
+ opts["names"] = names if names && !names.empty?
284
+ # C21b — continuity seed for cross-session stable stand-ins (forward-safe:
285
+ # the engine ignores unknown keys on an older build).
286
+ opts["seed_hex"] = seed_hex if seed_hex && !seed_hex.to_s.empty?
287
+ # EXPLICIT call-site argument -> resolve (raises on a bad literal).
288
+ opts["mode"] = CloakAPI::SurrogateMode.resolve(mode)
289
+ opts["bindings"] = bindings unless bindings.nil?
290
+ # The C ABI requires explicit acceptance before it returns a rules-only
291
+ # text result. Standard policy accepts with a warning; strict settings
292
+ # still send false and remain a non-egress refusal.
293
+ opts["accepted"] = CloakAPI::Degraded.allowed?(accepted)
294
+ payload = invoke(text.to_s.b, "text/plain", JSON.generate(opts))
295
+ CloakAPI::Degraded.assert_full_coverage!(payload["meta"], accepted: accepted)
296
+
297
+ tokenised = payload.fetch("markdown")
298
+ token_map = detok_table(tokenised, payload)
299
+ categories = tokenised.scan(TOKEN_LABEL_RE).flatten.uniq.sort
300
+ LocalTextResult.new(
301
+ text: tokenised,
302
+ token_map: token_map,
303
+ categories: categories,
304
+ surrogate_mode: CloakAPI::SurrogateMode::Report.from_payload(payload["surrogate_mode"]),
305
+ bindings: payload["bindings"] || {},
306
+ meta: payload["meta"] || {}
307
+ )
308
+ end
309
+
310
+ private
311
+
312
+ # One round-trip across the C ABI. Allocates the two out-parameter slots,
313
+ # calls the engine, and on a non-zero code reads + frees the error string
314
+ # then raises (fail-closed). On success parses + frees the JSON string.
315
+ def invoke(bytes, mime, opts_json)
316
+ out = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP, Fiddle::RUBY_FREE)
317
+ err = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP, Fiddle::RUBY_FREE)
318
+ # Zero the char* slots so a null check is meaningful even if the engine
319
+ # left one untouched.
320
+ zero = "\x00" * Fiddle::SIZEOF_VOIDP
321
+ out[0, Fiddle::SIZEOF_VOIDP] = zero
322
+ err[0, Fiddle::SIZEOF_VOIDP] = zero
323
+
324
+ code = @fn_tokenise.call(bytes, bytes.bytesize, mime, opts_json, out, err)
325
+
326
+ if code != 0
327
+ # The native string is untrusted: it can quote an archive entry name
328
+ # or a piece of the document. Free it, keep the numeric code.
329
+ read_and_free(err)
330
+ raise EngineError, "on-device tokenisation failed (engine code #{code})"
331
+ end
332
+
333
+ json = read_and_free(out)
334
+ raise EngineError, "engine returned empty result" if json.empty?
335
+
336
+ JSON.parse(json)
337
+ end
338
+
339
+ # Dereference a char** slot to the heap char*, copy it into a Ruby String,
340
+ # then hand the C string back to +cloak_string_free+. Returns "" for null.
341
+ def read_and_free(slot)
342
+ str_ptr = slot.ptr # *(char**)slot
343
+ return "" if str_ptr.null?
344
+
345
+ value = str_ptr.to_s.force_encoding("UTF-8")
346
+ @fn_free.call(str_ptr)
347
+ value
348
+ end
349
+
350
+ # Bind a static-string C getter if the symbol exists; nil otherwise (so a
351
+ # binding never hard-fails on an older engine that lacks the newer symbol).
352
+ def optional_static_fn(handle, name)
353
+ Fiddle::Function.new(handle[name], [], Fiddle::TYPE_VOIDP)
354
+ rescue Fiddle::DLError, StandardError
355
+ nil
356
+ end
357
+
358
+ # Read a 'static NUL-terminated C string returned by a getter (never freed —
359
+ # the ABI owns it). "" for a nil getter or a null pointer.
360
+ def static_string(fn)
361
+ return "" if fn.nil?
362
+
363
+ ptr = fn.call
364
+ ptr = Fiddle::Pointer.new(ptr.to_i) unless ptr.is_a?(Fiddle::Pointer)
365
+ ptr.null? ? "" : ptr.to_s.force_encoding("UTF-8")
366
+ end
367
+
368
+ # The AUTHORITATIVE local detokenise table for these bytes.
369
+ #
370
+ # PRIMARY: the engine's +replacements+ map, keyed by the FULL replacement
371
+ # string. This is the only table that works in BOTH modes — in BALANCED the
372
+ # replacement is a realistic string ("Sandra Hall"), not a
373
+ # <<CLOAK:TYPE:hash>> token, so it has no hash to key on and **a BALANCED
374
+ # reply cannot be restored from the hash-keyed +mappings+**. Before
375
+ # 2026-07-29 this SDK used only the hash rebuild below, so with the engine's
376
+ # default (BALANCED) style every reply silently came back un-restored:
377
+ # +token_map+ was +{}+ and +detokenize+ was a no-op.
378
+ #
379
+ # FALLBACK: the legacy hash-keyed +mappings+ + a scan of the emitted
380
+ # <<CLOAK:...>> tokens, kept so an OLDER engine binary (one that predates
381
+ # +replacements+) still detokenises its MAX-style output correctly.
382
+ def detok_table(text, payload)
383
+ replacements = payload["replacements"]
384
+ return replacements.to_h { |k, v| [k.to_s, v.to_s] } if replacements.is_a?(Hash) && !replacements.empty?
385
+
386
+ rebuild_token_map(text, payload["mappings"] || {})
387
+ end
388
+
389
+ # LEGACY fallback. The engine returns +hash => original+; rebuild the
390
+ # +full-token => original+ map by scanning the tokens actually emitted.
391
+ # Only reachable for a pre-+replacements+ engine build, and only capable of
392
+ # expressing MAX-style output.
393
+ def rebuild_token_map(text, hash_to_original)
394
+ map = {}
395
+ text.scan(TOKEN_RE) do
396
+ m = Regexp.last_match
397
+ full_token = m[0]
398
+ hash_val = m[1]
399
+ map[full_token] = hash_to_original[hash_val] if hash_to_original.key?(hash_val)
400
+ end
401
+ map
402
+ end
403
+ end
404
+
405
+ # What the SDK decided about the on-device engine for this process.
406
+ #
407
+ # used - true when the engine is loaded and will tokenise text
408
+ # reason - "" when used, else one of engine_missing, engine_unloadable,
409
+ # pin_missing, pin_mismatch, ruleset_mismatch
410
+ # message - plain English, safe to show; holds no request text
411
+ # sha256 - SHA-256 of the engine bytes that were loaded ("" when not used)
412
+ # source - "configured" (CLOAKAPI_ENGINE_LIB), "bundled" (inside the gem) or ""
413
+ Status = Data.define(:used, :reason, :message, :sha256, :source) do
414
+ def used? = used
415
+ def warning = used ? nil : CloakAPI::Warnings.reduced(reason, message)
416
+ end
417
+
418
+ ENV_LIB = "CLOAKAPI_ENGINE_LIB"
419
+ ENV_PIN = "CLOAKAPI_ENGINE_SHA256_PIN"
420
+
421
+ # The native engine builds of the release set this gem belongs to
422
+ # (repo/release-set.json, checked by release-set-check.mjs). An engine file
423
+ # shipped INSIDE the gem is accepted without a pin only when its SHA-256 is
424
+ # the one listed here for the platform. An engine named by
425
+ # CLOAKAPI_ENGINE_LIB always needs CLOAKAPI_ENGINE_SHA256_PIN.
426
+ RELEASE_SET_NATIVE = {
427
+ "linux-x86_64" => "b2aec8d6d90ead4328258270c41a79dda4e1f71efa79025f308ce8e09f080744",
428
+ "linux-aarch64" => "a8bfb99346cdaf6e47b13214e7f1f872f95e0c2fe040cf6bb3e6a82f1d26ed30"
429
+ }.freeze
430
+
431
+ # The released native engine is built against this glibc.
432
+ MIN_GLIBC = "2.39"
433
+
434
+ RULES_NOTE = "The SDK's built-in rules are used instead; names and some other personal data " \
435
+ "may be sent as written."
436
+
437
+ @gate_mutex = Mutex.new
438
+
439
+ class << self
440
+ # Platform-appropriate shared-library basenames for the engine.
441
+ def lib_filenames
442
+ case RbConfig::CONFIG["host_os"]
443
+ when /darwin/ then ["libcloak_engine.dylib"]
444
+ when /mswin|mingw|cygwin/ then ["cloak_engine.dll"]
445
+ else ["libcloak_engine.so"]
446
+ end
447
+ end
448
+
449
+ # "linux-x86_64", "linux-aarch64", "macos-aarch64", "windows-x86_64", ...
450
+ def platform_key
451
+ os = case RbConfig::CONFIG["host_os"]
452
+ when /darwin/ then "macos"
453
+ when /mswin|mingw|cygwin/ then "windows"
454
+ when /linux/ then "linux"
455
+ else RbConfig::CONFIG["host_os"].to_s
456
+ end
457
+ cpu = RbConfig::CONFIG["host_cpu"].to_s.downcase
458
+ cpu = "x86_64" if %w[amd64 x64 x86_64].include?(cpu)
459
+ cpu = "aarch64" if %w[arm64 aarch64].include?(cpu)
460
+ "#{os}-#{cpu}"
461
+ end
462
+
463
+ # Where an engine file may come from, in order: the path in
464
+ # CLOAKAPI_ENGINE_LIB, then a file shipped inside the gem. Nothing else
465
+ # is searched: an engine the integrator did not name and the gem does
466
+ # not carry is not used.
467
+ def candidate_paths
468
+ candidates = []
469
+ env = ENV[ENV_LIB]
470
+ candidates << env if env && !env.empty?
471
+ candidates.concat(bundled_paths)
472
+ candidates
473
+ end
474
+
475
+ def bundled_paths
476
+ here = __dir__ # .../lib/cloakapi
477
+ # Only the platform sub-directory: that is where a platform build of the
478
+ # gem carries the engine. A file directly in _lib is not used.
479
+ lib_filenames.map { |name| File.join(here, "_lib", platform_key, name) }
480
+ end
481
+
482
+ # The decision for this process: is the engine used, and if not, why.
483
+ # Made once, on first use, and kept. Never raises.
484
+ def status
485
+ @gate_mutex.synchronize do
486
+ unless defined?(@loaded) && @loaded
487
+ @status, @engine = decide
488
+ @loaded = true
489
+ end
490
+ @status
491
+ end
492
+ end
493
+
494
+ # The process-wide LocalEngine, or nil when the engine is not used (see
495
+ # {status} for the reason). Never raises.
496
+ def get
497
+ status
498
+ @engine
499
+ end
500
+
501
+ # True iff the on-device engine is loaded and used in this process.
502
+ def available?
503
+ status.used?
504
+ end
505
+
506
+ # +[version, hash]+ for the engine identity headers. +hash+ is the SHA-256
507
+ # of the loaded engine file, or "" when no engine is used; the relay then
508
+ # states no engine identity at all. Never raises.
509
+ def identity
510
+ current = status
511
+ return [RULESET_VERSION, ""] unless current.used?
512
+
513
+ [RULESET_VERSION, current.sha256]
514
+ end
515
+
516
+ # Reset the memoised decision (test hook).
517
+ def reset!
518
+ @gate_mutex.synchronize do
519
+ @loaded = false
520
+ @engine = nil
521
+ @status = nil
522
+ end
523
+ end
524
+
525
+ # A short sentence about the C library of this system when it explains
526
+ # why the released engine cannot load here; nil otherwise.
527
+ def libc_note
528
+ unless RbConfig::CONFIG["host_os"].to_s.include?("linux")
529
+ return "Native engine builds exist for Linux (x86-64 and aarch64) only."
530
+ end
531
+ return "The engine needs glibc #{MIN_GLIBC} or newer; this system uses musl, which is not supported." if musl?
532
+
533
+ have = glibc_version
534
+ return nil if have.nil?
535
+ return nil if Gem::Version.new(have) >= Gem::Version.new(MIN_GLIBC)
536
+
537
+ "The engine needs glibc #{MIN_GLIBC} or newer (this system has glibc #{have})."
538
+ rescue StandardError
539
+ nil
540
+ end
541
+
542
+ private
543
+
544
+ def musl?
545
+ %w[/lib /lib64 /usr/lib].any? { |dir| !Dir.glob(File.join(dir, "ld-musl-*.so.1")).empty? }
546
+ end
547
+
548
+ def glibc_version
549
+ return nil unless defined?(::Fiddle)
550
+
551
+ fn = Fiddle::Function.new(Fiddle.dlopen(nil)["gnu_get_libc_version"], [], Fiddle::TYPE_VOIDP)
552
+ ptr = fn.call
553
+ ptr = Fiddle::Pointer.new(ptr.to_i) unless ptr.is_a?(Fiddle::Pointer)
554
+ text = ptr.null? ? "" : ptr.to_s
555
+ text.match?(/\A\d+\.\d+(?:\.\d+)?\z/) ? text : nil
556
+ rescue StandardError
557
+ nil
558
+ end
559
+
560
+ def unavailable(reason, message)
561
+ [Status.new(used: false, reason: reason, message: message, sha256: "", source: ""), nil]
562
+ end
563
+
564
+ def with_note(*parts)
565
+ parts.compact.reject(&:empty?).join(" ")
566
+ end
567
+
568
+ def pin_matches?(pin, digest)
569
+ pin.match?(/\A[0-9a-fA-F]{64}\z/) && pin.casecmp?(digest)
570
+ end
571
+
572
+ # The pin rule (one rule for every CloakAPI SDK). A wrong, missing or
573
+ # unreadable engine never stops a send: the engine is treated as not
574
+ # there and the built-in rules are used, with a notice that names the
575
+ # reason.
576
+ def decide
577
+ # Taken as written: a value with spaces around it is not a SHA-256.
578
+ pin = ENV[ENV_PIN].to_s
579
+ configured = ENV[ENV_LIB].to_s
580
+ return decide_configured(configured, pin) unless configured.empty?
581
+
582
+ bundled = bundled_paths.find { |path| File.file?(path) }
583
+ return decide_bundled(bundled, pin) if bundled
584
+
585
+ if pin.empty?
586
+ unavailable("engine_missing", with_note("The CloakAPI on-device engine was not found.", RULES_NOTE))
587
+ else
588
+ unavailable("engine_missing",
589
+ with_note("The CloakAPI on-device engine was not found, so #{ENV_PIN} could not be checked.",
590
+ RULES_NOTE))
591
+ end
592
+ rescue StandardError
593
+ unavailable("engine_unloadable",
594
+ with_note("The CloakAPI on-device engine could not be set up on this system.", RULES_NOTE))
595
+ end
596
+
597
+ def decide_configured(path, pin)
598
+ where = "the engine at #{ENV_LIB}"
599
+ unless File.file?(path)
600
+ return unavailable("engine_missing",
601
+ with_note("The engine file named by #{ENV_LIB} was not found.", RULES_NOTE))
602
+ end
603
+ bytes = read_engine_bytes(path)
604
+ if bytes.nil?
605
+ return unavailable("engine_unloadable",
606
+ with_note("The engine file named by #{ENV_LIB} could not be read.", RULES_NOTE))
607
+ end
608
+ digest = Digest::SHA256.hexdigest(bytes)
609
+ if pin.empty?
610
+ return unavailable("pin_missing", with_note("#{ENV_PIN} is not set, so #{where} was not used.", RULES_NOTE))
611
+ end
612
+ unless pin_matches?(pin, digest)
613
+ return unavailable("pin_mismatch",
614
+ with_note("#{ENV_PIN} does not match the SHA-256 of #{where}, so that engine was not used.",
615
+ RULES_NOTE))
616
+ end
617
+ load_checked(bytes, digest, File.basename(path), "configured")
618
+ end
619
+
620
+ def decide_bundled(path, pin)
621
+ bytes = read_engine_bytes(path)
622
+ if bytes.nil?
623
+ return unavailable("engine_unloadable",
624
+ with_note("The engine file shipped inside the gem could not be read.", RULES_NOTE))
625
+ end
626
+ digest = Digest::SHA256.hexdigest(bytes)
627
+ released = RELEASE_SET_NATIVE[platform_key] == digest
628
+ pinned = pin_matches?(pin, digest)
629
+ if !pin.empty? && !pinned
630
+ return unavailable("pin_mismatch",
631
+ with_note("#{ENV_PIN} does not match the SHA-256 of the engine file shipped inside the gem, " \
632
+ "so that engine was not used.", RULES_NOTE))
633
+ end
634
+ unless released || pinned
635
+ return unavailable("pin_missing",
636
+ with_note("The engine file shipped inside the gem is not a build this SDK release names " \
637
+ "and #{ENV_PIN} is not set, so that engine was not used.", RULES_NOTE))
638
+ end
639
+ load_checked(bytes, digest, File.basename(path), "bundled")
640
+ end
641
+
642
+ def read_engine_bytes(path)
643
+ File.binread(path)
644
+ rescue SystemCallError, IOError
645
+ nil
646
+ end
647
+
648
+ # Load a private copy of exactly +bytes+ and accept it only when it
649
+ # reports the rule set this SDK was built for.
650
+ def load_checked(bytes, digest, basename, source)
651
+ unless defined?(::Fiddle)
652
+ return unavailable("engine_unloadable",
653
+ with_note("Ruby's fiddle library is not available, so the engine file could not be loaded " \
654
+ "(on Ruby 3.5 or newer add gem \"fiddle\" to the Gemfile).", RULES_NOTE))
655
+ end
656
+ engine = begin
657
+ load_private_copy(bytes, digest, basename)
658
+ rescue LoadError, StandardError
659
+ return unavailable("engine_unloadable",
660
+ with_note("The CloakAPI engine file was found but could not be loaded on this system.",
661
+ libc_note, RULES_NOTE))
662
+ end
663
+ version = engine.ruleset_version.to_s
664
+ ruleset_hash = engine.ruleset_hash.to_s
665
+ unless version == RULESET_VERSION && ruleset_hash == CloakAPI::Pii::Ruleset.sha256
666
+ reported = version.match?(/\A[0-9A-Za-z.+-]{1,32}\z/) ? version : "an unknown version"
667
+ said = version == RULESET_VERSION ? "rule set #{reported} with other contents" : "rule set #{reported}"
668
+ return unavailable("ruleset_mismatch",
669
+ with_note("The engine reports #{said}; this SDK expects rule set #{RULESET_VERSION}, " \
670
+ "so that engine was not used.", RULES_NOTE))
671
+ end
672
+ [Status.new(used: true, reason: "", message: "The CloakAPI on-device engine is loaded.",
673
+ sha256: digest, source: source), engine]
674
+ end
675
+
676
+ # Write the already-hashed bytes to a private copy and +dlopen+ the copy.
677
+ # The pin check therefore covers the code that is actually mapped: a file
678
+ # swapped at the original path before or after the load changes nothing,
679
+ # because the loader never sees that path. The directory is created 0700
680
+ # and the copy exclusively, so it is not shared with other users. The copy
681
+ # is removed once the library is mapped; a platform that cannot delete a
682
+ # mapped file keeps it until process exit.
683
+ def load_private_copy(bytes, digest, basename)
684
+ dir = Dir.mktmpdir("cloakapi-engine-")
685
+ begin
686
+ private_path = File.join(dir, basename)
687
+ File.open(private_path, File::WRONLY | File::CREAT | File::EXCL, 0o700) do |f|
688
+ f.binmode
689
+ f.write(bytes)
690
+ end
691
+ LocalEngine.new(Fiddle.dlopen(private_path), private_path, digest)
692
+ ensure
693
+ begin
694
+ FileUtils.remove_entry(dir)
695
+ rescue SystemCallError
696
+ nil
697
+ end
698
+ end
699
+ end
700
+ end
701
+ end
702
+ end