cloakapi 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +160 -0
- data/LICENSE +202 -0
- data/NOTICE +18 -0
- data/README.md +407 -0
- data/THIRD-PARTY-NOTICES.md +22 -0
- data/TRADEMARKS.md +19 -0
- data/examples/fake_gateway.rb +78 -0
- data/examples/hello.rb +64 -0
- data/lib/cloakapi/client.rb +1272 -0
- data/lib/cloakapi/degraded.rb +77 -0
- data/lib/cloakapi/disclosure.rb +252 -0
- data/lib/cloakapi/engine.rb +702 -0
- data/lib/cloakapi/errors.rb +65 -0
- data/lib/cloakapi/pii/bundled_rules.rb +670 -0
- data/lib/cloakapi/pii/card_spans.rb +300 -0
- data/lib/cloakapi/pii/detector.rb +1925 -0
- data/lib/cloakapi/pii/patterns.json +2238 -0
- data/lib/cloakapi/pii/pii_match.rb +7 -0
- data/lib/cloakapi/pii/pii_type.rb +7 -0
- data/lib/cloakapi/pii/ruleset.rb +65 -0
- data/lib/cloakapi/pii/tokenizer.rb +411 -0
- data/lib/cloakapi/posture.rb +82 -0
- data/lib/cloakapi/receipts.rb +406 -0
- data/lib/cloakapi/resources/chat.rb +82 -0
- data/lib/cloakapi/resources/messages.rb +90 -0
- data/lib/cloakapi/resources/models.rb +25 -0
- data/lib/cloakapi/resources/receipts.rb +162 -0
- data/lib/cloakapi/resources/reconstruct.rb +120 -0
- data/lib/cloakapi/resources/request_strings.rb +485 -0
- data/lib/cloakapi/resources/reviewable.rb +163 -0
- data/lib/cloakapi/resources/tokens.rb +28 -0
- data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
- data/lib/cloakapi/routing.rb +500 -0
- data/lib/cloakapi/streaming.rb +196 -0
- data/lib/cloakapi/surrogate_mode.rb +233 -0
- data/lib/cloakapi/version.rb +15 -0
- data/lib/cloakapi/warnings.rb +137 -0
- data/lib/cloakapi.rb +37 -0
- data/licenses/Apache-2.0.txt +202 -0
- data/licenses/CC-BY-4.0.txt +104 -0
- data/licenses/MIT-Faker.txt +21 -0
- data/licenses/MIT.txt +21 -0
- metadata +149 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module CloakAPI
|
|
7
|
+
module Pii
|
|
8
|
+
# The CANONICAL shared ruleset (fasit), vendored into the gem.
|
|
9
|
+
#
|
|
10
|
+
# WHY THIS EXISTS (P0, 2026-07-28). Two values used to be HAND-TYPED into
|
|
11
|
+
# this SDK and both had silently drifted from
|
|
12
|
+
# +shared/pii-patterns/patterns.json+:
|
|
13
|
+
#
|
|
14
|
+
# * +Engine::RULESET_VERSION+ ("1.4.29" vs a canonical 1.6.0) — emitted as
|
|
15
|
+
# the +X-Cloak-Engine-Version+ header the strict gateway floor-checks
|
|
16
|
+
# (ADR-0014 §3), so the SDK was DECLARING a ruleset it was not running.
|
|
17
|
+
# * the pure-regex fallback's pattern table, which carried no ADDRESS type
|
|
18
|
+
# at all, so every street address egressed verbatim whenever the native
|
|
19
|
+
# engine was absent.
|
|
20
|
+
#
|
|
21
|
+
# Both are now DERIVED from this vendored copy at load time. The copy is
|
|
22
|
+
# byte-locked to the canonical file by +spec/ruleset_conformance_spec.rb+,
|
|
23
|
+
# which fails the suite the moment the two diverge — so "derived" cannot
|
|
24
|
+
# decay back into "hand-typed".
|
|
25
|
+
#
|
|
26
|
+
# The file is vendored (rather than read from the monorepo) because the gem
|
|
27
|
+
# must work standalone once installed; +shared/+ is not shipped inside it.
|
|
28
|
+
module Ruleset
|
|
29
|
+
PATH = File.expand_path("patterns.json", __dir__)
|
|
30
|
+
|
|
31
|
+
class << self
|
|
32
|
+
# The parsed ruleset document.
|
|
33
|
+
def data
|
|
34
|
+
@data ||= JSON.parse(File.read(PATH))
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Canonical ruleset semver — the X-Cloak-Engine-Version fallback.
|
|
38
|
+
def version
|
|
39
|
+
@version ||= data.fetch("version")
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# SHA-256 of the vendored ruleset bytes (drift guard / attestation).
|
|
43
|
+
def sha256
|
|
44
|
+
@sha256 ||= Digest::SHA256.hexdigest(File.binread(PATH))
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# The raw canonical regex SOURCE for a type, e.g. +pattern("ADDRESS")+.
|
|
48
|
+
# Returns nil when the type is absent.
|
|
49
|
+
def pattern(type)
|
|
50
|
+
entry = data.fetch("types", {})[type.to_s]
|
|
51
|
+
entry && entry["pattern"]
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# {pattern} compiled with Ruby's Onigmo engine. The canonical patterns
|
|
55
|
+
# are authored in a PORTABLE subset (no look-around, back-references,
|
|
56
|
+
# \p{...} or inline flags) precisely so Rust/RE2/PCRE/Java/.NET/Ruby all
|
|
57
|
+
# compile the SAME source string.
|
|
58
|
+
def regexp(type)
|
|
59
|
+
src = pattern(type)
|
|
60
|
+
src && Regexp.new(src)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
require_relative "../engine"
|
|
6
|
+
require_relative "../degraded"
|
|
7
|
+
require_relative "../warnings"
|
|
8
|
+
|
|
9
|
+
module CloakAPI
|
|
10
|
+
module Pii
|
|
11
|
+
# The SDK's ONE outbound text-tokenise entry point (masterplan §11a/§12a,
|
|
12
|
+
# "one engine everywhere"). ENGINE-FIRST + fail-closed, mirroring the Python
|
|
13
|
+
# reference +tokenize_text(prefer_engine=True)+:
|
|
14
|
+
#
|
|
15
|
+
# * PRIMARY path — the shared Rust engine via FFI (+CloakAPI::Engine+):
|
|
16
|
+
# fasit ruleset + checksum validators + native NER, so a prompt is
|
|
17
|
+
# protected IDENTICALLY to the drop-in proxy, desktop app and every
|
|
18
|
+
# other surface. An +EngineError+ (engine present but refused, e.g. an
|
|
19
|
+
# encrypted/undecodable file) is deliberately PROPAGATED — the caller must
|
|
20
|
+
# NOT then send the raw text. Secrets are NOT refused: a credential is
|
|
21
|
+
# tokenised+flagged to a +<<CLOAK:CRED_KEY:…>>+ token, not blocked.
|
|
22
|
+
# * FALLBACK path — the pure-regex {Detector}/{Tokenizer} — is taken ONLY
|
|
23
|
+
# when the shared library is absent ({Engine.available?} is false). NER
|
|
24
|
+
# +names+ require the engine and are honestly out of reach here.
|
|
25
|
+
#
|
|
26
|
+
# Returns +[tokenised_text, token_map, categories, engine]+ where +token_map+
|
|
27
|
+
# is +full-token => original+ (kept LOCAL to detokenise the reply),
|
|
28
|
+
# +categories+ are the cloud-token labels that fired, and +engine+ is
|
|
29
|
+
# "ffi" or "regex".
|
|
30
|
+
#
|
|
31
|
+
# @param text [String]
|
|
32
|
+
# @param existing_map [Hash, nil] token_map to extend (kept across a session)
|
|
33
|
+
# @param names [Array<String>, nil] externally-detected PERSON spans (engine only)
|
|
34
|
+
# @param prefer_engine [Boolean]
|
|
35
|
+
# @return [Array(String, Hash, Array<String>, String)]
|
|
36
|
+
# The SDK-level outcome of one outbound text tokenisation.
|
|
37
|
+
#
|
|
38
|
+
# +surrogate_mode+ is a REQUIRED member (API-shape lock #1) — +Data.define+
|
|
39
|
+
# makes it a required keyword, so a hand-written fake result raises
|
|
40
|
+
# +ArgumentError: missing keyword: :surrogate_mode+ and cannot quietly omit
|
|
41
|
+
# what the pass actually did.
|
|
42
|
+
#
|
|
43
|
+
# text — tokenised text; the ONLY field meant to leave the machine
|
|
44
|
+
# token_map — replacement string => original (works in BOTH modes)
|
|
45
|
+
# categories — cloud-token labels that fired (MAX-style output only)
|
|
46
|
+
# engine — "ffi" (shared Rust engine) or "regex" (pure-Ruby fallback)
|
|
47
|
+
# surrogate_mode — {CloakAPI::SurrogateMode::Report}: what actually happened
|
|
48
|
+
# bindings — conversation ledger, advanced. RE-IDENTIFICATION
|
|
49
|
+
# MATERIAL — never egress it.
|
|
50
|
+
# coverage — a {CloakAPI::Coverage} value: "full", "rules_and_names"
|
|
51
|
+
# or "rules_only"
|
|
52
|
+
# warnings — Array of {CloakAPI::SendWarning} (code, reason, message)
|
|
53
|
+
# warning — the same notices as one String (kept from before 1.4.0)
|
|
54
|
+
TextTokenisation = Data.define(:text, :token_map, :categories, :engine, :surrogate_mode,
|
|
55
|
+
:bindings, :degraded, :coverage, :warning, :warnings) do
|
|
56
|
+
def initialize(text:, token_map:, categories:, engine:, surrogate_mode:, bindings:,
|
|
57
|
+
degraded:, coverage:, warning:, warnings: [])
|
|
58
|
+
super(text: text, token_map: token_map, categories: categories, engine: engine,
|
|
59
|
+
surrogate_mode: surrogate_mode, bindings: bindings, degraded: degraded,
|
|
60
|
+
coverage: coverage, warning: warning, warnings: warnings)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Tokenise a prompt and report WHAT WAS ACTUALLY DONE.
|
|
65
|
+
#
|
|
66
|
+
# This is the full-fidelity entry point: unlike the legacy 4-array
|
|
67
|
+
# {tokenize_text}, it returns the REQUIRED {CloakAPI::SurrogateMode::Report}
|
|
68
|
+
# and the advanced bindings ledger, which is what the client threads into the
|
|
69
|
+
# next turn and into the disclosure's continuity scope.
|
|
70
|
+
#
|
|
71
|
+
# +mode+ is an EXPLICIT call-site argument: a bad literal RAISES
|
|
72
|
+
# {CloakAPI::SurrogateMode::Error}. It selects STYLE ONLY and can never gate
|
|
73
|
+
# whether tokenisation happens.
|
|
74
|
+
def self.tokenize_text_ex(text, existing_map: nil, names: nil, prefer_engine: true,
|
|
75
|
+
seed_hex: nil, mode: nil, bindings: nil, accepted: nil)
|
|
76
|
+
resolved_mode = CloakAPI::SurrogateMode.resolve(mode)
|
|
77
|
+
token_map = existing_map ? existing_map.dup : {}
|
|
78
|
+
text = utf8_text(text)
|
|
79
|
+
|
|
80
|
+
# Engine.get never raises: an engine that is missing, not pinned, pinned
|
|
81
|
+
# wrongly, not loadable or built for another rule set is treated as not
|
|
82
|
+
# there (nil), and Engine.status carries the reason for the notice.
|
|
83
|
+
engine = prefer_engine ? CloakAPI::Engine.get : nil
|
|
84
|
+
lane_warning = prefer_engine ? CloakAPI::Engine.status.warning : nil
|
|
85
|
+
|
|
86
|
+
if engine
|
|
87
|
+
result = begin
|
|
88
|
+
engine.tokenise_text(text, names: names, seed_hex: seed_hex,
|
|
89
|
+
mode: resolved_mode, bindings: bindings, accepted: accepted)
|
|
90
|
+
rescue CloakAPI::Engine::EngineError
|
|
91
|
+
# The loaded engine could not process this text. A send is never
|
|
92
|
+
# stopped for that: this text is handled by the built-in rules.
|
|
93
|
+
lane_warning = CloakAPI::Warnings.reduced(
|
|
94
|
+
"engine_unloadable",
|
|
95
|
+
"The CloakAPI on-device engine could not process this text. #{CloakAPI::Engine::RULES_NOTE}"
|
|
96
|
+
)
|
|
97
|
+
nil
|
|
98
|
+
end
|
|
99
|
+
if result
|
|
100
|
+
token_map.merge!(result.token_map)
|
|
101
|
+
reduced = result.meta.is_a?(Hash) && result.meta["ner_enabled"] == false
|
|
102
|
+
warning = reduced ? CloakAPI::Degraded.message("tokenise text", CloakAPI::Degraded::KIND_NO_NER) : nil
|
|
103
|
+
warnings = reduced ? [CloakAPI::Warnings.reduced("model_missing", CloakAPI::Warnings::MODEL_MISSING_MESSAGE)] : []
|
|
104
|
+
warnings.each { |notice| CloakAPI::Warnings.emit(notice) }
|
|
105
|
+
return TextTokenisation.new(
|
|
106
|
+
text: result.text, token_map: token_map, categories: result.categories,
|
|
107
|
+
engine: "ffi", surrogate_mode: result.surrogate_mode, bindings: result.bindings,
|
|
108
|
+
degraded: reduced,
|
|
109
|
+
coverage: reduced ? CloakAPI::Coverage::RULES_AND_NAMES : CloakAPI::Coverage::FULL,
|
|
110
|
+
warning: warning, warnings: warnings
|
|
111
|
+
)
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# PURE-REGEX FALLBACK. It emits opaque <<CLOAK:TYPE:hash>> tokens and has no
|
|
116
|
+
# realistic-surrogate generator, so whatever mode was ASKED for, the style
|
|
117
|
+
# it actually produced is MAX. Reporting the ASKED-for mode here would be
|
|
118
|
+
# exactly the class of lie this field exists to prevent, so it reports what
|
|
119
|
+
# it did.
|
|
120
|
+
#
|
|
121
|
+
# ⛔ CORRECTED 2026-08-24 (XF). This used to read: "Protection is unaffected:
|
|
122
|
+
# the same spans are tokenised either way — the fallback differs in STYLE,
|
|
123
|
+
# never in coverage." THAT SENTENCE HAS BEEN MEASURED FALSE. The lanes differ
|
|
124
|
+
# in COVERAGE, not just style: the pure regex detector has no PERSON
|
|
125
|
+
# detection at all. Measured on this box, n=20, with
|
|
126
|
+
# Engine.available? == false. Measured output, verbatim:
|
|
127
|
+
# «Kontakt Andreas Kopervik og Marikeb Smith paa <<CLOAK:EMAIL:bc9dc5>> …»
|
|
128
|
+
# — the email was tokenised (the positive control showing the channel
|
|
129
|
+
# worked), BOTH person names untouched, and no error raised.
|
|
130
|
+
#
|
|
131
|
+
# The comment was a plausible explanation nobody had measured, and it made
|
|
132
|
+
# this very branch UNINTERESTING to look at. Hence the gate below.
|
|
133
|
+
#
|
|
134
|
+
# Continuity: the fallback's tokens are a pure function of the value
|
|
135
|
+
# (sha256) and any supplied +existing_map+ is carried forward, so when the
|
|
136
|
+
# caller is running a conversation the stand-ins genuinely are stable across
|
|
137
|
+
# turns. With no ledger and no prior map, the honest claim is the narrow one.
|
|
138
|
+
# ⛔ THE LANE B GATE. Only the IMPLICIT fallthrough is stopped: a caller that
|
|
139
|
+
# wrote +prefer_engine: false+ itself has already chosen the reduced lane
|
|
140
|
+
# explicitly, and that choice is the opt-in.
|
|
141
|
+
CloakAPI::Degraded.guard!("tokenise text", accepted, CloakAPI::Degraded::KIND_NO_ENGINE)
|
|
142
|
+
|
|
143
|
+
detector = Detector.new
|
|
144
|
+
tokenizer = Tokenizer.new(token_map)
|
|
145
|
+
input_text = text
|
|
146
|
+
invalid_encoding = input_text.is_a?(String) && !input_text.valid_encoding?
|
|
147
|
+
if invalid_encoding
|
|
148
|
+
# Keep token offsets and replacement slicing on one valid UTF-8 copy.
|
|
149
|
+
# Invalid source bytes are represented by U+FFFD in the fallback output;
|
|
150
|
+
# valid PII byte sequences elsewhere in the input remain scannable.
|
|
151
|
+
input_text = input_text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
|
|
152
|
+
end
|
|
153
|
+
short_compat = input_text.is_a?(String) && !invalid_encoding &&
|
|
154
|
+
input_text.bytesize <= Detector::SHORT_COMPAT_BYTES &&
|
|
155
|
+
input_text.valid_encoding? &&
|
|
156
|
+
(input_text.encoding == Encoding::UTF_8 || input_text.ascii_only?)
|
|
157
|
+
detection = if short_compat
|
|
158
|
+
CloakAPI::Pii::Detector::Detection.new(
|
|
159
|
+
matches: detector.detect(input_text), degraded: false,
|
|
160
|
+
coverage: "rules_only", warning: nil
|
|
161
|
+
)
|
|
162
|
+
else
|
|
163
|
+
detector.detect_with_status(input_text)
|
|
164
|
+
end
|
|
165
|
+
matches = detection.matches
|
|
166
|
+
if invalid_encoding
|
|
167
|
+
detection = CloakAPI::Pii::Detector::Detection.new(
|
|
168
|
+
matches: detection.matches, degraded: true, coverage: detection.coverage,
|
|
169
|
+
warning: "Detection was partial because invalid input bytes were replaced while scanning."
|
|
170
|
+
)
|
|
171
|
+
end
|
|
172
|
+
reduced_warning = CloakAPI::Degraded.message("tokenise text", CloakAPI::Degraded::KIND_NO_ENGINE)
|
|
173
|
+
warning = [detection.warning, reduced_warning].compact.reject(&:empty?).join("; ")
|
|
174
|
+
detection = CloakAPI::Pii::Detector::Detection.new(
|
|
175
|
+
matches: detection.matches, degraded: true, coverage: "rules_only", warning: warning
|
|
176
|
+
)
|
|
177
|
+
out = tokenizer.tokenize(input_text, matches)
|
|
178
|
+
if tokenizer.tokenize_budget_hit?
|
|
179
|
+
extra = "tokenize time budget exhausted"
|
|
180
|
+
warning = detection.warning
|
|
181
|
+
warning = warning.nil? || warning.empty? ? extra : "#{warning}; #{extra}"
|
|
182
|
+
detection = CloakAPI::Pii::Detector::Detection.new(
|
|
183
|
+
matches: detection.matches, degraded: true, coverage: detection.coverage, warning: warning
|
|
184
|
+
)
|
|
185
|
+
end
|
|
186
|
+
lane_warning ||= CloakAPI::Warnings.reduced(
|
|
187
|
+
"engine_missing", "The on-device engine was not asked for. #{CloakAPI::Engine::RULES_NOTE}"
|
|
188
|
+
)
|
|
189
|
+
warnings = [lane_warning]
|
|
190
|
+
partial = invalid_encoding || tokenizer.tokenize_budget_hit? ||
|
|
191
|
+
(detection.warning.to_s != reduced_warning && detection.warning.to_s.include?("partial"))
|
|
192
|
+
warnings << CloakAPI::Warnings.unchecked("value_not_replaced", CloakAPI::Warnings::PARTIAL_SCAN_MESSAGE) if partial
|
|
193
|
+
CloakAPI::Warnings.emit(lane_warning)
|
|
194
|
+
categories = out.scan(/<<CLOAK:([A-Z_]+):[0-9a-f]{6}>>/).flatten.uniq.sort
|
|
195
|
+
conversational = !bindings.nil? || !(existing_map.nil? || existing_map.empty?)
|
|
196
|
+
TextTokenisation.new(
|
|
197
|
+
text: out, token_map: tokenizer.token_map, categories: categories, engine: "regex",
|
|
198
|
+
degraded: detection.degraded, coverage: detection.coverage, warning: detection.warning,
|
|
199
|
+
warnings: warnings,
|
|
200
|
+
surrogate_mode: CloakAPI::SurrogateMode::Report.new(
|
|
201
|
+
mode: "max",
|
|
202
|
+
continuity: conversational ? "conversation" : "single-message"
|
|
203
|
+
),
|
|
204
|
+
bindings: bindings.nil? ? {} : bindings.dup
|
|
205
|
+
)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Both lanes work on UTF-8. A string tagged as binary is read as the UTF-8
|
|
209
|
+
# it almost always is (broken bytes are then handled, with a notice, by the
|
|
210
|
+
# rules lane); a string in another encoding is transcoded. Offsets, stand-ins
|
|
211
|
+
# and the restore map all refer to this one UTF-8 text.
|
|
212
|
+
def self.utf8_text(text)
|
|
213
|
+
return text unless text.is_a?(String)
|
|
214
|
+
return text if text.encoding == Encoding::UTF_8
|
|
215
|
+
return text.dup.force_encoding(Encoding::UTF_8) if text.encoding == Encoding::BINARY
|
|
216
|
+
|
|
217
|
+
text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
|
|
218
|
+
rescue EncodingError
|
|
219
|
+
text
|
|
220
|
+
end
|
|
221
|
+
private_class_method :utf8_text
|
|
222
|
+
|
|
223
|
+
# LEGACY 4-ARRAY SHAPE, kept for source compatibility. It cannot carry the
|
|
224
|
+
# REQUIRED surrogate-mode report, so anything that needs to know what the pass
|
|
225
|
+
# actually did — the relay, the disclosure's continuity scope, the next turn's
|
|
226
|
+
# ledger — must call {tokenize_text_ex} instead. The client does.
|
|
227
|
+
def self.tokenize_text(text, existing_map: nil, names: nil, prefer_engine: true,
|
|
228
|
+
seed_hex: nil, mode: nil, bindings: nil, accepted: nil)
|
|
229
|
+
res = tokenize_text_ex(text, existing_map: existing_map, names: names,
|
|
230
|
+
prefer_engine: prefer_engine, seed_hex: seed_hex,
|
|
231
|
+
mode: mode, bindings: bindings, accepted: accepted)
|
|
232
|
+
[res.text, res.token_map, res.categories, res.engine]
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Replace every token in +text+ with its original value, using a plain
|
|
236
|
+
# +full-token => original+ map. Companion to {tokenize_text} for restoring a
|
|
237
|
+
# model reply on-device.
|
|
238
|
+
# Replace every replacement string in +text+ with its original value.
|
|
239
|
+
#
|
|
240
|
+
# +token_map+ is keyed by the FULL replacement string — an opaque
|
|
241
|
+
# <<CLOAK:TYPE:hash>> token in MAX, a realistic stand-in ("Sandra Hall") in
|
|
242
|
+
# BALANCED — because a hash-keyed table cannot express a BALANCED replacement
|
|
243
|
+
# and so cannot restore a BALANCED reply.
|
|
244
|
+
#
|
|
245
|
+
# LONGEST KEY FIRST, so a replacement that is a substring of another (e.g. a
|
|
246
|
+
# surname inside a full name) cannot corrupt the longer one — a
|
|
247
|
+
# realistic-surrogate hazard MAX never had.
|
|
248
|
+
def self.detokenize(text, token_map)
|
|
249
|
+
# Find every hit on the original text first. Replacing longest-first with
|
|
250
|
+
# repeated gsub also scans text inserted by an earlier replacement, so an
|
|
251
|
+
# original containing another stand-in can be corrupted on the next pass.
|
|
252
|
+
hits = []
|
|
253
|
+
token_map.each do |raw_key, original|
|
|
254
|
+
key = raw_key.to_s
|
|
255
|
+
next if key.empty? || !original.is_a?(String)
|
|
256
|
+
|
|
257
|
+
offset = 0
|
|
258
|
+
while (start = text.index(key, offset))
|
|
259
|
+
finish = start + key.length
|
|
260
|
+
hits << [start, finish, key, original]
|
|
261
|
+
offset = start + 1
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
return text if hits.empty?
|
|
265
|
+
|
|
266
|
+
hits.sort_by! { |start, finish, key, _original| [-(finish - start), start, key] }
|
|
267
|
+
chosen = []
|
|
268
|
+
hits.each do |hit|
|
|
269
|
+
next if chosen.any? { |taken| hit[0] < taken[1] && taken[0] < hit[1] }
|
|
270
|
+
|
|
271
|
+
chosen << hit
|
|
272
|
+
end
|
|
273
|
+
chosen.sort_by!(&:first)
|
|
274
|
+
|
|
275
|
+
out = +""
|
|
276
|
+
offset = 0
|
|
277
|
+
chosen.each do |start, finish, _key, original|
|
|
278
|
+
out << text[offset...start] << original
|
|
279
|
+
offset = finish
|
|
280
|
+
end
|
|
281
|
+
out << text[offset..]
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
class Tokenizer
|
|
285
|
+
def initialize(existing_map = nil)
|
|
286
|
+
@token_map = existing_map ? existing_map.dup : {}
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Replace PII matches with <<CLOAK:TYPE:HASH>> tokens.
|
|
290
|
+
#
|
|
291
|
+
# The spans are resolved BEFORE anything is written (see {resolve_spans}),
|
|
292
|
+
# and the text is then assembled in one forward pass over offsets of the
|
|
293
|
+
# ORIGINAL text. Every character a span covers is replaced, no other
|
|
294
|
+
# character is touched, and a stand-in is always written whole.
|
|
295
|
+
def tokenize(text, matches)
|
|
296
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
297
|
+
tokenize_forward(text, resolve_spans(text, matches), started)
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def tokenize_budget_hit?
|
|
301
|
+
@tokenize_budget_hit == true || CloakAPI::Pii::Detector.wall_budget_hit?
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def tokenize_forward(text, selected, started)
|
|
305
|
+
parts = []
|
|
306
|
+
cursor = 0
|
|
307
|
+
token_by_value = {}
|
|
308
|
+
@tokenize_budget_hit = false
|
|
309
|
+
selected.each_with_index do |m, index|
|
|
310
|
+
if (index & 255).zero? && Process.clock_gettime(Process::CLOCK_MONOTONIC) - started >= 2.0
|
|
311
|
+
@tokenize_budget_hit = true
|
|
312
|
+
end
|
|
313
|
+
next if m.start < cursor
|
|
314
|
+
|
|
315
|
+
token = token_by_value[m.value] ||= make_token(m.type, m.value)
|
|
316
|
+
@token_map[token] = m.value
|
|
317
|
+
parts << text[cursor...m.start] if m.start > cursor
|
|
318
|
+
parts << token
|
|
319
|
+
cursor = m.finish
|
|
320
|
+
end
|
|
321
|
+
parts << text[cursor..]
|
|
322
|
+
parts.join
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# Replace all tokens in text with original values.
|
|
326
|
+
# Restore the originals. LONGEST KEY FIRST (delegates to the module-level
|
|
327
|
+
# {Pii.detokenize}) — this instance can be seeded with an +existing_map+
|
|
328
|
+
# from the engine, which in BALANCED holds realistic stand-ins where one
|
|
329
|
+
# key can be a substring of another ("Hall" inside "Sandra Hall"); an
|
|
330
|
+
# insertion-ordered pass would then corrupt the longer one.
|
|
331
|
+
def detokenize(text)
|
|
332
|
+
CloakAPI::Pii.detokenize(text, @token_map)
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
def clear_map!
|
|
336
|
+
@token_map.clear
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
def token_map
|
|
340
|
+
@token_map.dup
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
private
|
|
344
|
+
|
|
345
|
+
# Turn reported matches into disjoint spans, ordered by start.
|
|
346
|
+
#
|
|
347
|
+
# * A span that is not a real range of this text is dropped: offsets
|
|
348
|
+
# that are not integers, a negative start, a start at or past the
|
|
349
|
+
# end, an empty or reversed range. An end past the text is cut to
|
|
350
|
+
# the text.
|
|
351
|
+
# * Spans that share at least one character become ONE span over their
|
|
352
|
+
# union, labelled with the type of the longest of them (the earliest
|
|
353
|
+
# start wins a tie). Spans that only touch stay separate. Identical
|
|
354
|
+
# spans collapse.
|
|
355
|
+
#
|
|
356
|
+
# Offsets are character offsets (String#[] semantics), so a span never
|
|
357
|
+
# ends inside a multi-byte character. Two rules that hit overlapping
|
|
358
|
+
# characters used to be settled by keeping the longest span and dropping
|
|
359
|
+
# the others, which left the uncovered remainder of a dropped match in
|
|
360
|
+
# the outgoing text; with an empty span at the same start the old
|
|
361
|
+
# back-to-front splice could also cut into a stand-in already written.
|
|
362
|
+
def resolve_spans(text, matches)
|
|
363
|
+
length = text.length
|
|
364
|
+
valid = matches.filter_map do |match|
|
|
365
|
+
start = match.start
|
|
366
|
+
finish = match.finish
|
|
367
|
+
next unless start.is_a?(Integer) && finish.is_a?(Integer)
|
|
368
|
+
next if start.negative? || start >= length || finish <= start
|
|
369
|
+
|
|
370
|
+
finish > length ? PiiMatch.new(type: match.type, start: start, finish: length, value: text[start...length]) : match
|
|
371
|
+
end
|
|
372
|
+
return valid if valid.length < 2
|
|
373
|
+
|
|
374
|
+
ordered = valid.sort_by { |match| [match.start, -(match.finish - match.start), match.type.to_s, match.value.to_s] }
|
|
375
|
+
resolved = []
|
|
376
|
+
start = finish = nil
|
|
377
|
+
best = nil
|
|
378
|
+
flush = lambda do
|
|
379
|
+
next if best.nil?
|
|
380
|
+
|
|
381
|
+
# A union that IS one of the reported spans keeps that span (and so
|
|
382
|
+
# its stand-in); a real merge is a new span over the union.
|
|
383
|
+
resolved << if best.start == start && best.finish == finish
|
|
384
|
+
best
|
|
385
|
+
else
|
|
386
|
+
PiiMatch.new(type: best.type, start: start, finish: finish, value: text[start...finish])
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
ordered.each do |match|
|
|
390
|
+
if best && match.start < finish
|
|
391
|
+
# Shares a character with the span being built.
|
|
392
|
+
finish = match.finish if match.finish > finish
|
|
393
|
+
best = match if (match.finish - match.start) > (best.finish - best.start)
|
|
394
|
+
else
|
|
395
|
+
flush.call
|
|
396
|
+
start = match.start
|
|
397
|
+
finish = match.finish
|
|
398
|
+
best = match
|
|
399
|
+
end
|
|
400
|
+
end
|
|
401
|
+
flush.call
|
|
402
|
+
resolved
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def make_token(type, value)
|
|
406
|
+
hash = Digest::SHA256.digest(value).unpack1("H*")[0, 6]
|
|
407
|
+
"<<CLOAK:#{type.to_s.upcase}:#{hash}>>"
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
end
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CloakAPI
|
|
4
|
+
# Named privacy tiers + PRIVACY-POSTURE introspection (C20).
|
|
5
|
+
#
|
|
6
|
+
# A small, pure descriptor of what a CloakAPI integration actually protects, so
|
|
7
|
+
# a developer can render an honest status badge ("what's protected") without
|
|
8
|
+
# hand-assembling it from scattered engine-identity / detector flags. Mirrors
|
|
9
|
+
# the TS +privacyPosture+ / +PRIVACY_TIERS+ / +VALID_TIERS+.
|
|
10
|
+
module Posture
|
|
11
|
+
# The Level-1 tiers CloakAPI exposes — there is NO plaintext +off+ tier
|
|
12
|
+
# (Level-1-only strict flip). +off+/+plaintext+ are rejected at construction.
|
|
13
|
+
#
|
|
14
|
+
# TASK #24 / DECISION 11: +continuity+ IS RETIRED AS A CHOICE. There are
|
|
15
|
+
# EXACTLY TWO tiers, forever: +balanced+ (default) and +max+. Until
|
|
16
|
+
# 2026-08-10 this list advertised +continuity+ as a third selectable
|
|
17
|
+
# posture, which the two-mode contract forbids. The continuity BEHAVIOUR
|
|
18
|
+
# (the same real value keeps the same stand-in for the whole conversation)
|
|
19
|
+
# is ALWAYS ON in BOTH tiers and is never named or offered. Matches
|
|
20
|
+
# sdk/php/src/Tier.php and sdk/typescript/src/client.ts, retired 2026-07-29.
|
|
21
|
+
VALID_TIERS = %w[balanced max].freeze
|
|
22
|
+
PRIVACY_TIERS = VALID_TIERS
|
|
23
|
+
|
|
24
|
+
# Retired tier literals still accepted AS DATA (a saved config, an old wire
|
|
25
|
+
# value), migrated onto a PROTECTED tier — never plaintext, so a config
|
|
26
|
+
# written last month cannot break a client or, far worse, weaken it.
|
|
27
|
+
LEGACY_TIER_ALIASES = { "continuity" => "balanced" }.freeze
|
|
28
|
+
|
|
29
|
+
# Map an accepted tier literal onto one of {VALID_TIERS}. Retired literals
|
|
30
|
+
# migrate; anything else is returned lowercased for the caller to reject.
|
|
31
|
+
def self.normalise_tier(tier)
|
|
32
|
+
t = tier.to_s.strip.downcase
|
|
33
|
+
LEGACY_TIER_ALIASES.fetch(t, t)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Build the privacy posture from an engine identity (or anything responding to
|
|
37
|
+
# +#identity+ returning +{version:, hash:}+). The engine binary is never
|
|
38
|
+
# loaded here — this runs in a plain unit test.
|
|
39
|
+
#
|
|
40
|
+
# @param source [Hash, #identity] +{version:, hash:}+ or an identity provider
|
|
41
|
+
# @param enhanced_gazetteer [Boolean] tier-1 expanded gazetteer active?
|
|
42
|
+
# @param model_tier_loaded [Boolean] tier-2 transformer NER model active?
|
|
43
|
+
# @param marker_format [String] 'clk' | 'cloak-angle' (drives auto-disclosure)
|
|
44
|
+
# @param tiers [Array<String>] override the tier list (default {PRIVACY_TIERS})
|
|
45
|
+
# @return [Hash] content-free, honest posture descriptor
|
|
46
|
+
def self.privacy_posture(source, enhanced_gazetteer: false, model_tier_loaded: false,
|
|
47
|
+
marker_format: "clk", tiers: nil)
|
|
48
|
+
id =
|
|
49
|
+
if source.respond_to?(:identity)
|
|
50
|
+
source.identity
|
|
51
|
+
else
|
|
52
|
+
source
|
|
53
|
+
end
|
|
54
|
+
version = fetch(id, :version) || ""
|
|
55
|
+
hash = fetch(id, :hash) || ""
|
|
56
|
+
|
|
57
|
+
detectors = %w[structured-ffi name-dictionary]
|
|
58
|
+
detectors << "enhanced-gazetteer" if enhanced_gazetteer
|
|
59
|
+
detectors << "enhanced-ner-model" if model_tier_loaded
|
|
60
|
+
|
|
61
|
+
{
|
|
62
|
+
"engineVersion" => version,
|
|
63
|
+
"engineHash" => hash,
|
|
64
|
+
"genuine" => hash.is_a?(String) && !hash.empty?,
|
|
65
|
+
"failClosed" => true,
|
|
66
|
+
"tiers" => (tiers || PRIVACY_TIERS).dup,
|
|
67
|
+
"detectors" => detectors,
|
|
68
|
+
"modelTierLoaded" => !!model_tier_loaded,
|
|
69
|
+
"markerFormat" => marker_format,
|
|
70
|
+
}
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def self.fetch(obj, key)
|
|
74
|
+
if obj.is_a?(Hash)
|
|
75
|
+
obj[key] || obj[key.to_s]
|
|
76
|
+
elsif obj.respond_to?(key)
|
|
77
|
+
obj.public_send(key)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
private_class_method :fetch
|
|
81
|
+
end
|
|
82
|
+
end
|