cloakapi 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +160 -0
- data/LICENSE +202 -0
- data/NOTICE +18 -0
- data/README.md +407 -0
- data/THIRD-PARTY-NOTICES.md +22 -0
- data/TRADEMARKS.md +19 -0
- data/examples/fake_gateway.rb +78 -0
- data/examples/hello.rb +64 -0
- data/lib/cloakapi/client.rb +1272 -0
- data/lib/cloakapi/degraded.rb +77 -0
- data/lib/cloakapi/disclosure.rb +252 -0
- data/lib/cloakapi/engine.rb +702 -0
- data/lib/cloakapi/errors.rb +65 -0
- data/lib/cloakapi/pii/bundled_rules.rb +670 -0
- data/lib/cloakapi/pii/card_spans.rb +300 -0
- data/lib/cloakapi/pii/detector.rb +1925 -0
- data/lib/cloakapi/pii/patterns.json +2238 -0
- data/lib/cloakapi/pii/pii_match.rb +7 -0
- data/lib/cloakapi/pii/pii_type.rb +7 -0
- data/lib/cloakapi/pii/ruleset.rb +65 -0
- data/lib/cloakapi/pii/tokenizer.rb +411 -0
- data/lib/cloakapi/posture.rb +82 -0
- data/lib/cloakapi/receipts.rb +406 -0
- data/lib/cloakapi/resources/chat.rb +82 -0
- data/lib/cloakapi/resources/messages.rb +90 -0
- data/lib/cloakapi/resources/models.rb +25 -0
- data/lib/cloakapi/resources/receipts.rb +162 -0
- data/lib/cloakapi/resources/reconstruct.rb +120 -0
- data/lib/cloakapi/resources/request_strings.rb +485 -0
- data/lib/cloakapi/resources/reviewable.rb +163 -0
- data/lib/cloakapi/resources/tokens.rb +28 -0
- data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
- data/lib/cloakapi/routing.rb +500 -0
- data/lib/cloakapi/streaming.rb +196 -0
- data/lib/cloakapi/surrogate_mode.rb +233 -0
- data/lib/cloakapi/version.rb +15 -0
- data/lib/cloakapi/warnings.rb +137 -0
- data/lib/cloakapi.rb +37 -0
- data/licenses/Apache-2.0.txt +202 -0
- data/licenses/CC-BY-4.0.txt +104 -0
- data/licenses/MIT-Faker.txt +21 -0
- data/licenses/MIT.txt +21 -0
- metadata +149 -0
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CloakAPI
|
|
4
|
+
module Pii
|
|
5
|
+
# Card numbers as the reference rules lane finds them: a port of its card
|
|
6
|
+
# scanner (packages/pii-patterns-ts/src/card-spans.ts, findCardSpans), on
|
|
7
|
+
# character offsets.
|
|
8
|
+
#
|
|
9
|
+
# The rule type CREDIT_CARD is applied by this scanner and not by its
|
|
10
|
+
# pattern, in the reference and here. It walks chains of digit groups joined
|
|
11
|
+
# by single separators, takes the windows that have a card network's length
|
|
12
|
+
# and prefix and pass the Luhn check, and also takes a card-shaped head
|
|
13
|
+
# inside a longer digit run.
|
|
14
|
+
#
|
|
15
|
+
# The detector (detector.rb) has its own, stricter card scanner, which the
|
|
16
|
+
# lane also runs; this one exists so that the bundled rule types give what
|
|
17
|
+
# the reference gives (sdk/conformance/rules-lane-vectors.json).
|
|
18
|
+
module CardSpans
|
|
19
|
+
HEAD = /(?:4[0-9]{12}(?:[0-9]{3})?|(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|6(?:011|5[0-9]{2})[0-9]{12}|(?:2131|1800|35[0-9]{3})[0-9]{11})/
|
|
20
|
+
SEPARATORS = [" ", "\t", "\n", "\r", "-", ".", " ", "·", "‐", "‑", "‒",
|
|
21
|
+
"–", "—", "―", "−", "‧"].to_h { |c| [c, true] }.freeze
|
|
22
|
+
WORD = /\A[\p{L}\p{M}\p{N}_]\z/
|
|
23
|
+
|
|
24
|
+
module_function
|
|
25
|
+
|
|
26
|
+
# @return [Array<Array(Integer, Integer)>] character spans, sorted, disjoint
|
|
27
|
+
def find(text)
|
|
28
|
+
chars = text.chars
|
|
29
|
+
n = chars.length
|
|
30
|
+
spans = []
|
|
31
|
+
saw_double = false
|
|
32
|
+
i = 0
|
|
33
|
+
while i < n
|
|
34
|
+
unless digit?(chars, i)
|
|
35
|
+
i += 1
|
|
36
|
+
next
|
|
37
|
+
end
|
|
38
|
+
j = run_end(chars, i)
|
|
39
|
+
boundary = separator_length(chars, j)
|
|
40
|
+
unless boundary.positive? && j + boundary < n && digit?(chars, j + boundary)
|
|
41
|
+
saw_double = true if opens_double_separator?(chars, j)
|
|
42
|
+
len = j - i
|
|
43
|
+
digits = chars[i...j].join
|
|
44
|
+
if len.between?(13, 19) && network?(digits) && luhn?(digits)
|
|
45
|
+
wrapped = word?(i.positive? ? chars[i - 1] : nil) && !digit?(chars, i - 1) && word?(chars[j])
|
|
46
|
+
spans << [i, j] unless wrapped
|
|
47
|
+
elsif len > 19
|
|
48
|
+
emit_chain(chars, [[i, j]], [], spans)
|
|
49
|
+
elsif len > 13
|
|
50
|
+
emit_head_inside_long_run(chars, i, j, spans)
|
|
51
|
+
end
|
|
52
|
+
i = j
|
|
53
|
+
next
|
|
54
|
+
end
|
|
55
|
+
groups = []
|
|
56
|
+
separators = []
|
|
57
|
+
j = i
|
|
58
|
+
while j < n && digit?(chars, j)
|
|
59
|
+
start = j
|
|
60
|
+
j = run_end(chars, j)
|
|
61
|
+
groups << [start, j]
|
|
62
|
+
sep = separator_length(chars, j)
|
|
63
|
+
break unless sep.positive? && j + sep < n && digit?(chars, j + sep)
|
|
64
|
+
|
|
65
|
+
separators << chars[j, sep].join
|
|
66
|
+
j += sep
|
|
67
|
+
end
|
|
68
|
+
saw_double = true if opens_double_separator?(chars, j)
|
|
69
|
+
emit_chain(chars, groups, separators, spans)
|
|
70
|
+
i = [j, i + 1].max
|
|
71
|
+
end
|
|
72
|
+
emit_double_separator(chars, spans) if saw_double
|
|
73
|
+
spans.sort_by! { |span| [span[0], -(span[1] - span[0])] }
|
|
74
|
+
kept = []
|
|
75
|
+
last_end = -1
|
|
76
|
+
spans.each do |span|
|
|
77
|
+
next if span[0] < last_end
|
|
78
|
+
|
|
79
|
+
kept << span
|
|
80
|
+
last_end = span[1]
|
|
81
|
+
end
|
|
82
|
+
kept
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def digit?(chars, index)
|
|
86
|
+
return false if index.negative?
|
|
87
|
+
|
|
88
|
+
c = chars[index]
|
|
89
|
+
!c.nil? && c >= "0" && c <= "9" && c.length == 1 && c.ascii_only?
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def run_end(chars, index)
|
|
93
|
+
index += 1 while digit?(chars, index)
|
|
94
|
+
index
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Length in characters of the separator at +index+: 0 when there is none;
|
|
98
|
+
# "\r\n" counts as one separator of two characters.
|
|
99
|
+
def separator_length(chars, index)
|
|
100
|
+
c = chars[index]
|
|
101
|
+
return 0 if c.nil?
|
|
102
|
+
return 2 if c == "\r" && chars[index + 1] == "\n"
|
|
103
|
+
|
|
104
|
+
SEPARATORS.key?(c) ? 1 : 0
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# A separator of one character (anything but "\r\n"), or 0.
|
|
108
|
+
def single_separator_length(chars, index)
|
|
109
|
+
length = separator_length(chars, index)
|
|
110
|
+
length == 2 ? 0 : length
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def opens_double_separator?(chars, index)
|
|
114
|
+
sep = single_separator_length(chars, index)
|
|
115
|
+
sep.positive? && chars[index + sep] == " " && digit?(chars, index + sep + 1)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def word?(char)
|
|
119
|
+
return false if char.nil? || char.empty?
|
|
120
|
+
|
|
121
|
+
char.match?(WORD)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def luhn?(digits)
|
|
125
|
+
sum = 0
|
|
126
|
+
double = false
|
|
127
|
+
digits.reverse.each_char do |char|
|
|
128
|
+
value = char.ord - 48
|
|
129
|
+
if double
|
|
130
|
+
value *= 2
|
|
131
|
+
value -= 9 if value > 9
|
|
132
|
+
end
|
|
133
|
+
sum += value
|
|
134
|
+
double = !double
|
|
135
|
+
end
|
|
136
|
+
(sum % 10).zero?
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Does the digit string have a card network's prefix and length?
|
|
140
|
+
def network?(digits)
|
|
141
|
+
length = digits.length
|
|
142
|
+
num = ->(width) { length >= width ? digits[0, width].to_i : -1 }
|
|
143
|
+
d0 = digits[0].to_i
|
|
144
|
+
first2 = num.call(2)
|
|
145
|
+
first3 = num.call(3)
|
|
146
|
+
first4 = num.call(4)
|
|
147
|
+
first6 = num.call(6)
|
|
148
|
+
return true if d0 == 4 && [13, 16, 19].include?(length)
|
|
149
|
+
return true if length == 16 && (first2.between?(51, 55) || first4.between?(2221, 2720))
|
|
150
|
+
return true if length == 15 && [34, 37].include?(first2)
|
|
151
|
+
return true if length == 14 && (first3.between?(300, 305) || [36, 38, 39].include?(first2))
|
|
152
|
+
return true if [16, 19].include?(length) && (first4 == 6011 || first2 == 65)
|
|
153
|
+
return true if length == 16 && (first3.between?(644, 649) || first6.between?(622_126, 622_925))
|
|
154
|
+
return true if length.between?(16, 19) && first4.between?(3528, 3589)
|
|
155
|
+
|
|
156
|
+
length.between?(16, 19) && first2 == 62
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def emit_head_inside_long_run(chars, start, finish, spans)
|
|
160
|
+
slice = chars[start...finish].join
|
|
161
|
+
position = 0
|
|
162
|
+
while (found = HEAD.match(slice, position))
|
|
163
|
+
span_start = start + found.begin(0)
|
|
164
|
+
match = found[0]
|
|
165
|
+
position = found.end(0) > found.begin(0) ? found.end(0) : found.begin(0) + 1
|
|
166
|
+
len = match.length
|
|
167
|
+
span_end = span_start + len
|
|
168
|
+
next if len < 13 || len > 19
|
|
169
|
+
next unless network?(match) && luhn?(match)
|
|
170
|
+
|
|
171
|
+
wrapped = word?(span_start.positive? ? chars[span_start - 1] : nil) && !digit?(chars, span_start - 1) &&
|
|
172
|
+
word?(chars[span_end]) && !digit?(chars, span_end)
|
|
173
|
+
spans << [span_start, span_end] unless wrapped
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def canonical_group_window?(widths, from, to)
|
|
178
|
+
w = ->(offset) { widths[from + offset] }
|
|
179
|
+
case to - from
|
|
180
|
+
when 1 then w.call(0).between?(13, 19)
|
|
181
|
+
when 2 then w.call(0) == 6 && w.call(1) == 10
|
|
182
|
+
when 3 then w.call(0) == 4 && w.call(1) == 6 && w.call(2) == 5
|
|
183
|
+
when 4 then w.call(0) == 4 && w.call(1) == 4 && w.call(2) == 4 && [4, 3].include?(w.call(3))
|
|
184
|
+
when 5 then w.call(0) == 4 && w.call(1) == 4 && w.call(2) == 4 && w.call(3) == 4 && w.call(4) == 3
|
|
185
|
+
else false
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def emit_chain(chars, groups, separators, spans)
|
|
190
|
+
group_count = groups.length
|
|
191
|
+
return if group_count.zero?
|
|
192
|
+
if group_count == 5 && separators == ["-", "-", "-", "-"] &&
|
|
193
|
+
groups.map { |start, finish| finish - start } == [8, 4, 4, 4, 12]
|
|
194
|
+
return # the digit-only form of a UUID
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
first_start, first_end = groups[0]
|
|
198
|
+
lead = first_start.positive? ? chars[first_start - 1] : nil
|
|
199
|
+
group_index = 0
|
|
200
|
+
if word?(lead) && !digit?(chars, first_start - 1)
|
|
201
|
+
first_digits = chars[first_start...first_end].join
|
|
202
|
+
first_is_pan = first_digits.length.between?(13, 19) && network?(first_digits) && luhn?(first_digits)
|
|
203
|
+
if !first_is_pan
|
|
204
|
+
group_index = 1 unless group_count == 1
|
|
205
|
+
elsif group_count == 1 && word?(chars[first_end])
|
|
206
|
+
return
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
widths = []
|
|
210
|
+
digits = +""
|
|
211
|
+
groups.each do |start, finish|
|
|
212
|
+
width = finish - start
|
|
213
|
+
group_digits = chars[start...finish].join
|
|
214
|
+
widths << width
|
|
215
|
+
digits << group_digits
|
|
216
|
+
if width > 19
|
|
217
|
+
emit_head_inside_long_run(chars, start, finish, spans)
|
|
218
|
+
elsif width > 13 && !(network?(group_digits) && luhn?(group_digits))
|
|
219
|
+
emit_head_inside_long_run(chars, start, finish, spans)
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
prefix = [0]
|
|
223
|
+
widths.each { |width| prefix << prefix.last + width }
|
|
224
|
+
while group_index < group_count
|
|
225
|
+
taken = 0
|
|
226
|
+
from = prefix[group_index]
|
|
227
|
+
end_group = group_index + 1
|
|
228
|
+
end_group += 1 while end_group <= group_count && prefix[end_group] - from < 13
|
|
229
|
+
candidates = []
|
|
230
|
+
while end_group <= group_count && prefix[end_group] - from <= 19
|
|
231
|
+
candidates << end_group
|
|
232
|
+
end_group += 1
|
|
233
|
+
end
|
|
234
|
+
candidates.reverse_each do |stop|
|
|
235
|
+
len = prefix[stop] - from
|
|
236
|
+
eligible = (group_index.zero? && stop == group_count) || canonical_group_window?(widths, group_index, stop)
|
|
237
|
+
next unless eligible
|
|
238
|
+
|
|
239
|
+
window = digits[from, len]
|
|
240
|
+
if network?(window) && luhn?(window)
|
|
241
|
+
taken = stop - group_index
|
|
242
|
+
break
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
if taken.zero?
|
|
246
|
+
group_index += 1
|
|
247
|
+
next
|
|
248
|
+
end
|
|
249
|
+
spans << [groups[group_index][0], groups[group_index + taken - 1][1]]
|
|
250
|
+
group_index += taken
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def emit_double_separator(chars, spans)
|
|
255
|
+
n = chars.length
|
|
256
|
+
i = 0
|
|
257
|
+
while i < n
|
|
258
|
+
unless digit?(chars, i)
|
|
259
|
+
i += 1
|
|
260
|
+
next
|
|
261
|
+
end
|
|
262
|
+
start = i
|
|
263
|
+
digit_count = 0
|
|
264
|
+
doubles = 0
|
|
265
|
+
finish = i
|
|
266
|
+
j = i
|
|
267
|
+
too_long = false
|
|
268
|
+
digits = +""
|
|
269
|
+
while j < n
|
|
270
|
+
if digit?(chars, j)
|
|
271
|
+
digit_count += 1
|
|
272
|
+
digits << chars[j] if digit_count <= 19
|
|
273
|
+
finish = j + 1
|
|
274
|
+
j += 1
|
|
275
|
+
too_long = true if digit_count > 19
|
|
276
|
+
next
|
|
277
|
+
end
|
|
278
|
+
sep = single_separator_length(chars, j)
|
|
279
|
+
if sep.positive? && digit?(chars, j + sep)
|
|
280
|
+
j += sep
|
|
281
|
+
next
|
|
282
|
+
end
|
|
283
|
+
if sep.positive? && chars[j + sep] == " " && digit?(chars, j + sep + 1)
|
|
284
|
+
doubles += 1
|
|
285
|
+
j += sep + 1
|
|
286
|
+
next
|
|
287
|
+
end
|
|
288
|
+
break
|
|
289
|
+
end
|
|
290
|
+
lead = start.positive? ? chars[start - 1] : nil
|
|
291
|
+
lead_is_word = !lead.nil? && !digit?(chars, start - 1) && word?(lead)
|
|
292
|
+
if !too_long && doubles == 1 && digit_count >= 13 && !lead_is_word && network?(digits) && luhn?(digits)
|
|
293
|
+
spans << [start, finish]
|
|
294
|
+
end
|
|
295
|
+
i = [j, i + 1].max
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
end
|
|
300
|
+
end
|