cloakapi 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +160 -0
- data/LICENSE +202 -0
- data/NOTICE +18 -0
- data/README.md +407 -0
- data/THIRD-PARTY-NOTICES.md +22 -0
- data/TRADEMARKS.md +19 -0
- data/examples/fake_gateway.rb +78 -0
- data/examples/hello.rb +64 -0
- data/lib/cloakapi/client.rb +1272 -0
- data/lib/cloakapi/degraded.rb +77 -0
- data/lib/cloakapi/disclosure.rb +252 -0
- data/lib/cloakapi/engine.rb +702 -0
- data/lib/cloakapi/errors.rb +65 -0
- data/lib/cloakapi/pii/bundled_rules.rb +670 -0
- data/lib/cloakapi/pii/card_spans.rb +300 -0
- data/lib/cloakapi/pii/detector.rb +1925 -0
- data/lib/cloakapi/pii/patterns.json +2238 -0
- data/lib/cloakapi/pii/pii_match.rb +7 -0
- data/lib/cloakapi/pii/pii_type.rb +7 -0
- data/lib/cloakapi/pii/ruleset.rb +65 -0
- data/lib/cloakapi/pii/tokenizer.rb +411 -0
- data/lib/cloakapi/posture.rb +82 -0
- data/lib/cloakapi/receipts.rb +406 -0
- data/lib/cloakapi/resources/chat.rb +82 -0
- data/lib/cloakapi/resources/messages.rb +90 -0
- data/lib/cloakapi/resources/models.rb +25 -0
- data/lib/cloakapi/resources/receipts.rb +162 -0
- data/lib/cloakapi/resources/reconstruct.rb +120 -0
- data/lib/cloakapi/resources/request_strings.rb +485 -0
- data/lib/cloakapi/resources/reviewable.rb +163 -0
- data/lib/cloakapi/resources/tokens.rb +28 -0
- data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
- data/lib/cloakapi/routing.rb +500 -0
- data/lib/cloakapi/streaming.rb +196 -0
- data/lib/cloakapi/surrogate_mode.rb +233 -0
- data/lib/cloakapi/version.rb +15 -0
- data/lib/cloakapi/warnings.rb +137 -0
- data/lib/cloakapi.rb +37 -0
- data/licenses/Apache-2.0.txt +202 -0
- data/licenses/CC-BY-4.0.txt +104 -0
- data/licenses/MIT-Faker.txt +21 -0
- data/licenses/MIT.txt +21 -0
- metadata +149 -0
|
@@ -0,0 +1,1925 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "ruleset"
|
|
4
|
+
require "date"
|
|
5
|
+
|
|
6
|
+
module CloakAPI
|
|
7
|
+
module Pii
|
|
8
|
+
class Detector
|
|
9
|
+
WINDOW_BYTES = 65_536
|
|
10
|
+
MAX_WINDOWS_PER_PASS = 32
|
|
11
|
+
REGEX_STEP_BUDGET = 4_096
|
|
12
|
+
EMERGENCY_WALL_SECONDS = 3.75
|
|
13
|
+
SHORT_INPUT_HANG_SECONDS = 2.0
|
|
14
|
+
REGEX_TIMEOUT_SECONDS = 0.25
|
|
15
|
+
MAX_SPAN_BYTES = 65_536
|
|
16
|
+
ADDRESS_CANONICAL_SCAN_BYTES = 4_096
|
|
17
|
+
ADDRESS_WINDOW_SCAN_BYTES = 1_048_576
|
|
18
|
+
MAX_ADDRESS_RESERVE_BYTES = 8_388_608
|
|
19
|
+
LINEAR_RESERVE_SECONDS = 3.0
|
|
20
|
+
LINEAR_RESERVE_CHUNK_BYTES = 262_144
|
|
21
|
+
LINEAR_RESERVE_OVERLAP_BYTES = 65_536
|
|
22
|
+
SHORT_COMPAT_BYTES = 65_536
|
|
23
|
+
ADDRESS_ANCHOR_BUDGET = 256
|
|
24
|
+
ADDRESS_STREET_SUFFIXES = %w[
|
|
25
|
+
straße strasse straat street gata gatan gaten gate gade veien vegen vägen vei veg vej weg
|
|
26
|
+
avenue ave avenida rue via road rd lane ln gasse allee platz plein torg torget plass square
|
|
27
|
+
boulevard blvd crescent gardens terrace parade court ct drive dr place pl wharf mews quay way
|
|
28
|
+
st sq carrer calle rua strada ulica intie tie torvet gränd gracht väg allée
|
|
29
|
+
].freeze
|
|
30
|
+
ADDRESS_STREET_SUFFIXES_BY_END = ADDRESS_STREET_SUFFIXES.group_by { |suffix| suffix[-2, 2] }
|
|
31
|
+
.transform_values(&:freeze).freeze
|
|
32
|
+
MRZ_MAX_BYTES = 512
|
|
33
|
+
NATIONAL_ID_CUES = [
|
|
34
|
+
"national id", "national identification", "national identification number", "national number",
|
|
35
|
+
"identity number", "id number", "id no", "id no.", "nid", "personal id", "personal number",
|
|
36
|
+
"id-nummer", "id-nr", "nasjonal id", "ausweisnummer", "personalausweis", "nationale id",
|
|
37
|
+
"numéro national", "numéro d'identité", "carte d'identité", "pièce d'identité",
|
|
38
|
+
"número de identidad", "identificación nacional", "cédula", "numero di identità",
|
|
39
|
+
"documento d'identità", "carta d'identità", "identiteitsnummer", "nationaal id",
|
|
40
|
+
"número de identidade", "cartão de identidade", "identificação nacional", "social security",
|
|
41
|
+
"nino", "nric", "personnummer", "fødselsnummer",
|
|
42
|
+
"fodselsnummer", "kennitala", "cpr", "tax id", "driver licence", "driver license",
|
|
43
|
+
"resident card", "citizen card"
|
|
44
|
+
].uniq.freeze
|
|
45
|
+
API_KEY_CUES = %w[
|
|
46
|
+
api key secret token bearer sk-live- sk-test- sk-prod- sk-proj- pk-live- rk-live- ak-live-
|
|
47
|
+
].freeze
|
|
48
|
+
API_KEY_CUE_RE = Regexp.new(API_KEY_CUES.map { |cue| Regexp.escape(cue) }.join("|"), Regexp::IGNORECASE)
|
|
49
|
+
NATIONAL_ID_CUE_RE = Regexp.new(NATIONAL_ID_CUES.map { |cue| Regexp.escape(cue) }.join("|"), Regexp::IGNORECASE)
|
|
50
|
+
NATIONAL_ID_CHECKSUM_CUE_RE = /(?:bsn|sin|social[ ]+insurance[ ]+number|aadhaar|aadhar|uid|my[ ]+number)/i
|
|
51
|
+
NATIONAL_ID_SHAPE_RE = /(?:\b\d(?:[ .\/-]?\d){7,16}[Xx]?\b|\b[A-Z]{4}\d{6}[HM][A-Z]{5}[A-Z0-9]\d\b)/
|
|
52
|
+
NUMERIC_FALLBACK_RE = /\d(?:[ \t().+\-]?\d){6,18}/
|
|
53
|
+
IBAN_SHAPE_RE = /\b[A-Za-z]{2}\d{2}/
|
|
54
|
+
ALL_DIGITS_RE = /\A[0-9]{65,}\z/
|
|
55
|
+
DOB_CUES = %w[dob date of birth birth date born geboren nacido nacida nascita née né natød natø].freeze
|
|
56
|
+
DOB_CUE_RE = Regexp.new(DOB_CUES.map { |cue| Regexp.escape(cue) }.join("|"), Regexp::IGNORECASE)
|
|
57
|
+
OPTIONAL_CUE_RE = Regexp.new((API_KEY_CUES + NATIONAL_ID_CUES + DOB_CUES).uniq
|
|
58
|
+
.map { |cue| Regexp.escape(cue) }.join("|"), Regexp::IGNORECASE)
|
|
59
|
+
LINEAR_WARNING = "Detection was partial because some pattern checks were skipped. The fallback scans " \
|
|
60
|
+
"e-mail, phone, card, IBAN, SSN, labelled ID, marked DOB, credentials and MRZ; " \
|
|
61
|
+
"address and other regex-only matches may be missed."
|
|
62
|
+
|
|
63
|
+
ADDRESS_RE = Ruleset.regexp("ADDRESS")
|
|
64
|
+
PATTERNS = {
|
|
65
|
+
email: /\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b/,
|
|
66
|
+
phone: /\b(\+?1[\s\-.]?)?\(?\d{3}\)?[\s\-.]?\d{3}[\s\-.]?\d{4}\b/,
|
|
67
|
+
ssn: /\b[0-8]\d{2}[-\s]?\d{2}[-\s]?\d{4}\b/,
|
|
68
|
+
iban: /\b[A-Z]{2}[0-9]{2}(?: ?[A-Z0-9]{1,4}){2,7}\b/,
|
|
69
|
+
ip: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/,
|
|
70
|
+
ccnum: /\b(?:4[0-9]{3} ?[0-9]{4} ?[0-9]{4} ?[0-9]{4}|4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{2} ?[0-9]{4} ?[0-9]{4} ?[0-9]{4}|5[1-5][0-9]{14}|3[47][0-9]{2} ?[0-9]{6} ?[0-9]{5}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11})\b/,
|
|
71
|
+
api_key: Ruleset.regexp("API_KEY"),
|
|
72
|
+
auth_jwt: Ruleset.regexp("AUTH_JWT"),
|
|
73
|
+
nid_generic_labelled: Ruleset.regexp("NID_GENERIC_LABELLED"),
|
|
74
|
+
date_of_birth: Ruleset.regexp("DATE_OF_BIRTH"),
|
|
75
|
+
address: ADDRESS_RE,
|
|
76
|
+
}.freeze
|
|
77
|
+
LEGACY_SHORT_PATTERNS = PATTERNS.slice(:email, :phone, :ssn, :iban, :ip, :ccnum, :address).freeze
|
|
78
|
+
BOUNDED_PATTERNS = PATTERNS.transform_values do |regexp|
|
|
79
|
+
Regexp.new(regexp.source, regexp.options, timeout: REGEX_TIMEOUT_SECONDS)
|
|
80
|
+
end.freeze
|
|
81
|
+
|
|
82
|
+
Detection = Data.define(:matches, :degraded, :coverage, :warning)
|
|
83
|
+
ReserveDeadlineReached = Class.new(StandardError)
|
|
84
|
+
|
|
85
|
+
# The tokenize 2s wall clock also covers detect. A mebibyte of
|
|
86
|
+
# email+card finished detect in about 3s with tokenize_budget_hit? still
|
|
87
|
+
# false, because the emergency cap is 3.75s and nothing recorded 2s.
|
|
88
|
+
def self.reset_wall_budget_hit!
|
|
89
|
+
@wall_budget_hit = false
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def self.note_wall_budget_hit!
|
|
93
|
+
@wall_budget_hit = true
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def self.wall_budget_hit?
|
|
97
|
+
@wall_budget_hit == true
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Compatibility API: callers which only need spans still receive an Array.
|
|
101
|
+
def detect(text)
|
|
102
|
+
self.class.reset_wall_budget_hit!
|
|
103
|
+
if text.is_a?(String) && text.bytesize <= SHORT_COMPAT_BYTES && text.valid_encoding? &&
|
|
104
|
+
(text.encoding == Encoding::UTF_8 || text.ascii_only?)
|
|
105
|
+
legacy_short_matches(text)
|
|
106
|
+
else
|
|
107
|
+
detect_with_status(text).matches
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def legacy_short_matches(text)
|
|
112
|
+
matches = []
|
|
113
|
+
LEGACY_SHORT_PATTERNS.each do |type, regexp|
|
|
114
|
+
next if type == :ccnum
|
|
115
|
+
|
|
116
|
+
text.scan(regexp) do
|
|
117
|
+
match = Regexp.last_match
|
|
118
|
+
matches << PiiMatch.new(type: type, start: match.begin(0), finish: match.end(0), value: match[0])
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
# Texts at or under 64 KiB never reach the regex credential pass.
|
|
122
|
+
# The byte scanner is the same reserve used above that line, and it
|
|
123
|
+
# runs only when a credential cue is present so the 50 KiB gate does
|
|
124
|
+
# not pay for it on ordinary prose.
|
|
125
|
+
if short_credential_cue?(text)
|
|
126
|
+
bytes = text.b
|
|
127
|
+
scan_credentials(bytes, bytes.downcase, matches, ascii_offsets: text.ascii_only?)
|
|
128
|
+
end
|
|
129
|
+
merge_card_matches!(text, matches)
|
|
130
|
+
matches.uniq { |m| [m.type, m.start, m.finish] }.sort_by(&:start)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Scans bounded byte windows. A match belongs to the window whose core
|
|
134
|
+
# contains its starting character offset; overlap is context, not ownership.
|
|
135
|
+
# Ruby regex timeouts are per Regexp instance, so no process-global timeout
|
|
136
|
+
# is mutated while concurrent SDK calls are running.
|
|
137
|
+
def detect_with_status(text, step_budget: REGEX_STEP_BUDGET)
|
|
138
|
+
self.class.reset_wall_budget_hit!
|
|
139
|
+
unless text.is_a?(String)
|
|
140
|
+
return Detection.new(
|
|
141
|
+
matches: [], degraded: true, coverage: "rules_only",
|
|
142
|
+
warning: "Detection was partial because input was not a String."
|
|
143
|
+
)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
147
|
+
step_budget = [[Integer(step_budget), 0].max, REGEX_STEP_BUDGET].min
|
|
148
|
+
matches = []
|
|
149
|
+
needs_reserve = false
|
|
150
|
+
address_reserve_needed = false
|
|
151
|
+
scanned_end_byte = 0
|
|
152
|
+
reserve_start_byte = nil
|
|
153
|
+
reserve_unscanned_bytes = nil
|
|
154
|
+
ascii_input = text.ascii_only?
|
|
155
|
+
scannable_encoding = text.valid_encoding? && (text.encoding == Encoding::UTF_8 || ascii_input)
|
|
156
|
+
degraded = !scannable_encoding
|
|
157
|
+
if scannable_encoding && text.bytesize > SHORT_COMPAT_BYTES &&
|
|
158
|
+
text.bytesize <= ADDRESS_WINDOW_SCAN_BYTES
|
|
159
|
+
return scan_single_large_window(text, started)
|
|
160
|
+
end
|
|
161
|
+
if scannable_encoding
|
|
162
|
+
base_chars = 0
|
|
163
|
+
windows_processed = 0
|
|
164
|
+
each_window(text) do |window, base_byte, core_end_byte|
|
|
165
|
+
windows_processed += 1
|
|
166
|
+
scanned_end_byte = core_end_byte
|
|
167
|
+
BOUNDED_PATTERNS.each do |type, regexp|
|
|
168
|
+
if type == :address &&
|
|
169
|
+
((text.bytesize > ADDRESS_CANONICAL_SCAN_BYTES &&
|
|
170
|
+
text.bytesize <= SHORT_COMPAT_BYTES) || text.bytesize > ADDRESS_WINDOW_SCAN_BYTES)
|
|
171
|
+
degraded = true
|
|
172
|
+
needs_reserve = true
|
|
173
|
+
address_reserve_needed = text.bytesize <= MAX_ADDRESS_RESERVE_BYTES
|
|
174
|
+
next
|
|
175
|
+
end
|
|
176
|
+
possible = pattern_possible?(type, window)
|
|
177
|
+
unless possible || (text.bytesize <= ADDRESS_CANONICAL_SCAN_BYTES &&
|
|
178
|
+
%i[nid_generic_labelled date_of_birth].include?(type))
|
|
179
|
+
# A cue is only a performance hint. Unless it is proven complete
|
|
180
|
+
# for the canonical expression, skipping its scan must be visible.
|
|
181
|
+
degraded = true if text.bytesize > ADDRESS_CANONICAL_SCAN_BYTES
|
|
182
|
+
next
|
|
183
|
+
end
|
|
184
|
+
if emergency_expired?(text, started)
|
|
185
|
+
degraded = true
|
|
186
|
+
needs_reserve = true
|
|
187
|
+
reserve_start_byte ||= base_byte
|
|
188
|
+
break
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
begin
|
|
192
|
+
cursor = 0
|
|
193
|
+
steps = 0
|
|
194
|
+
while (match = regexp.match(window, cursor))
|
|
195
|
+
if emergency_expired?(text, started) || steps >= step_budget
|
|
196
|
+
degraded = true
|
|
197
|
+
needs_reserve = true
|
|
198
|
+
reserve_start_byte ||= base_byte
|
|
199
|
+
break
|
|
200
|
+
end
|
|
201
|
+
steps += 1
|
|
202
|
+
start_byte = ascii_input ? match.begin(0) : match.pre_match.bytesize
|
|
203
|
+
finish_byte = start_byte + match[0].bytesize
|
|
204
|
+
global_byte = base_byte + start_byte
|
|
205
|
+
global_finish_byte = base_byte + finish_byte
|
|
206
|
+
clipped_left = start_byte.zero? && base_byte.positive? &&
|
|
207
|
+
match_continues?(type, text.getbyte(base_byte - 1))
|
|
208
|
+
clipped_right = finish_byte == window.bytesize && global_finish_byte < text.bytesize &&
|
|
209
|
+
match_continues?(type, text.getbyte(global_finish_byte))
|
|
210
|
+
if clipped_left || clipped_right
|
|
211
|
+
degraded = true
|
|
212
|
+
needs_reserve = true
|
|
213
|
+
reserve_start_byte ||= base_byte
|
|
214
|
+
cursor = match.end(0)
|
|
215
|
+
next
|
|
216
|
+
end
|
|
217
|
+
if owns_match_start?(global_byte, core_end_byte)
|
|
218
|
+
if span_too_long?(finish_byte - start_byte)
|
|
219
|
+
degraded = true
|
|
220
|
+
needs_reserve = true
|
|
221
|
+
reserve_start_byte ||= base_byte
|
|
222
|
+
end
|
|
223
|
+
start_char = base_chars + match.begin(0)
|
|
224
|
+
matches << PiiMatch.new(type: type, start: start_char,
|
|
225
|
+
finish: start_char + match[0].length, value: match[0])
|
|
226
|
+
end
|
|
227
|
+
cursor = match.end(0)
|
|
228
|
+
cursor += 1 if match.begin(0) == match.end(0)
|
|
229
|
+
end
|
|
230
|
+
if type == :nid_generic_labelled
|
|
231
|
+
# The canonical ruleset deliberately has a narrower cue set
|
|
232
|
+
# than this SDK's labelled-ID reserve (for example, it does
|
|
233
|
+
# not include the Norwegian "fødselsnummer" cue). When a
|
|
234
|
+
# cue is present but the canonical regex has no match, use
|
|
235
|
+
# the same bounded, label-anchored fallback used by the
|
|
236
|
+
# linear reserve on this window instead of silently skipping
|
|
237
|
+
# the cue in the prefix before a large-input reserve starts.
|
|
238
|
+
local_matches = []
|
|
239
|
+
scan_labeled_ids(window.b, window.b.downcase, local_matches,
|
|
240
|
+
ascii_offsets: window.ascii_only?)
|
|
241
|
+
local_matches.each do |local_match|
|
|
242
|
+
local_start_byte = window.byteslice(0...local_match.start).bytesize
|
|
243
|
+
global_start_byte = base_byte + local_start_byte
|
|
244
|
+
next unless owns_match_start?(global_start_byte, core_end_byte)
|
|
245
|
+
|
|
246
|
+
start_char = base_chars + local_match.start
|
|
247
|
+
matches << PiiMatch.new(type: local_match.type, start: start_char,
|
|
248
|
+
finish: start_char + local_match.value.length,
|
|
249
|
+
value: local_match.value)
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
rescue Regexp::TimeoutError
|
|
253
|
+
degraded = true
|
|
254
|
+
needs_reserve = true
|
|
255
|
+
reserve_start_byte ||= base_byte
|
|
256
|
+
break
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
base_chars += text.byteslice(base_byte...core_end_byte).length
|
|
260
|
+
if text.bytesize > MAX_ADDRESS_RESERVE_BYTES &&
|
|
261
|
+
windows_processed >= MAX_WINDOWS_PER_PASS && core_end_byte < text.bytesize
|
|
262
|
+
degraded = true
|
|
263
|
+
needs_reserve = true
|
|
264
|
+
break
|
|
265
|
+
end
|
|
266
|
+
break if emergency_expired?(text, started)
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
if scannable_encoding && (needs_reserve || degraded) && text.bytesize > SHORT_COMPAT_BYTES &&
|
|
271
|
+
text.bytesize <= MAX_ADDRESS_RESERVE_BYTES
|
|
272
|
+
degraded = true
|
|
273
|
+
linear_matches(text, matches)
|
|
274
|
+
elsif needs_reserve && text.bytesize > SHORT_COMPAT_BYTES
|
|
275
|
+
degraded = true
|
|
276
|
+
reserve_unscanned_bytes = scan_linear_reserve_chunks(
|
|
277
|
+
text, reserve_start_byte || scanned_end_byte, matches, deadline: started + EMERGENCY_WALL_SECONDS
|
|
278
|
+
)
|
|
279
|
+
elsif needs_reserve || text.bytesize <= SHORT_COMPAT_BYTES
|
|
280
|
+
linear_degraded = linear_matches(text, matches)
|
|
281
|
+
degraded ||= linear_degraded
|
|
282
|
+
end
|
|
283
|
+
address_degraded = false
|
|
284
|
+
if scannable_encoding && (text.bytesize <= SHORT_COMPAT_BYTES || address_reserve_needed ||
|
|
285
|
+
(text.bytesize <= MAX_ADDRESS_RESERVE_BYTES && needs_reserve &&
|
|
286
|
+
!emergency_expired?(text, started)))
|
|
287
|
+
address_degraded = scan_address_reserve(text, matches)
|
|
288
|
+
end
|
|
289
|
+
degraded ||= address_degraded
|
|
290
|
+
# A second full-text card walk on multi-megabyte input blows the R12
|
|
291
|
+
# six-second deadline. Windows above 1 MiB keep the regex card class.
|
|
292
|
+
merge_card_matches!(text, matches) if text.bytesize <= ADDRESS_WINDOW_SCAN_BYTES
|
|
293
|
+
matches = matches.uniq { |m| [m.type, m.start, m.finish] }.sort_by(&:start)
|
|
294
|
+
warning = if !scannable_encoding
|
|
295
|
+
"Detection was partial because input encoding could not be scanned as UTF-8."
|
|
296
|
+
elsif degraded
|
|
297
|
+
warning = LINEAR_WARNING
|
|
298
|
+
if text.bytesize > MAX_ADDRESS_RESERVE_BYTES && needs_reserve
|
|
299
|
+
warning += " Address patterns were not scanned for input larger than 8 MiB."
|
|
300
|
+
end
|
|
301
|
+
if reserve_unscanned_bytes
|
|
302
|
+
first, last = reserve_unscanned_bytes
|
|
303
|
+
warning += " Linear reserve stopped; bytes #{first}..#{last} were not scanned. " \
|
|
304
|
+
"The end offset is exclusive (interval [#{first}, #{last})); " \
|
|
305
|
+
"equivalently, bytes #{first}..#{last - 1} were not scanned. " \
|
|
306
|
+
"That final byte offset is inclusive."
|
|
307
|
+
end
|
|
308
|
+
warning
|
|
309
|
+
end
|
|
310
|
+
Detection.new(matches: matches, degraded: degraded, coverage: "rules_only", warning: warning)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
private
|
|
314
|
+
|
|
315
|
+
def scan_single_large_window(text, started)
|
|
316
|
+
matches = []
|
|
317
|
+
degraded = false
|
|
318
|
+
needs_reserve = false
|
|
319
|
+
LEGACY_SHORT_PATTERNS.each_key do |type|
|
|
320
|
+
next if type == :ccnum
|
|
321
|
+
|
|
322
|
+
if wall_two_seconds?(text, started)
|
|
323
|
+
degraded = true
|
|
324
|
+
break
|
|
325
|
+
end
|
|
326
|
+
if emergency_expired?(text, started)
|
|
327
|
+
degraded = true
|
|
328
|
+
needs_reserve = true
|
|
329
|
+
break
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
333
|
+
regexp = Regexp.new(PATTERNS.fetch(type).source, PATTERNS.fetch(type).options,
|
|
334
|
+
timeout: [[2.0 - elapsed, EMERGENCY_WALL_SECONDS - elapsed].min, 0.001].max)
|
|
335
|
+
begin
|
|
336
|
+
text.to_enum(:scan, regexp).each do
|
|
337
|
+
match = Regexp.last_match
|
|
338
|
+
matches << PiiMatch.new(type: type, start: match.begin(0), finish: match.end(0), value: match[0])
|
|
339
|
+
if wall_two_seconds?(text, started)
|
|
340
|
+
degraded = true
|
|
341
|
+
break
|
|
342
|
+
end
|
|
343
|
+
if emergency_expired?(text, started)
|
|
344
|
+
degraded = true
|
|
345
|
+
needs_reserve = true
|
|
346
|
+
break
|
|
347
|
+
end
|
|
348
|
+
end
|
|
349
|
+
rescue Regexp::TimeoutError
|
|
350
|
+
degraded = true
|
|
351
|
+
needs_reserve = true
|
|
352
|
+
break
|
|
353
|
+
end
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
optional_cues_present = text.match?(OPTIONAL_CUE_RE)
|
|
357
|
+
%i[api_key auth_jwt nid_generic_labelled date_of_birth].each do |type|
|
|
358
|
+
if wall_two_seconds?(text, started)
|
|
359
|
+
degraded = true
|
|
360
|
+
break
|
|
361
|
+
end
|
|
362
|
+
if emergency_expired?(text, started)
|
|
363
|
+
degraded = true
|
|
364
|
+
needs_reserve = true
|
|
365
|
+
break
|
|
366
|
+
end
|
|
367
|
+
cue_type = %i[api_key nid_generic_labelled date_of_birth].include?(type)
|
|
368
|
+
unless (!cue_type || optional_cues_present) && pattern_possible?(type, text)
|
|
369
|
+
degraded = true
|
|
370
|
+
next
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
regexp = BOUNDED_PATTERNS.fetch(type)
|
|
374
|
+
begin
|
|
375
|
+
text.to_enum(:scan, regexp).each do
|
|
376
|
+
match = Regexp.last_match
|
|
377
|
+
matches << PiiMatch.new(type: type, start: match.begin(0), finish: match.end(0), value: match[0])
|
|
378
|
+
if wall_two_seconds?(text, started)
|
|
379
|
+
degraded = true
|
|
380
|
+
break
|
|
381
|
+
end
|
|
382
|
+
if emergency_expired?(text, started)
|
|
383
|
+
degraded = true
|
|
384
|
+
needs_reserve = true
|
|
385
|
+
break
|
|
386
|
+
end
|
|
387
|
+
end
|
|
388
|
+
rescue Regexp::TimeoutError
|
|
389
|
+
degraded = true
|
|
390
|
+
needs_reserve = true
|
|
391
|
+
break
|
|
392
|
+
end
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
if pattern_possible?(:nid_generic_labelled, text)
|
|
396
|
+
scan_labeled_ids(text.b, text.b.downcase, matches, ascii_offsets: text.ascii_only?)
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
if needs_reserve && text.bytesize <= MAX_ADDRESS_RESERVE_BYTES
|
|
400
|
+
linear_degraded = linear_matches(text, matches)
|
|
401
|
+
degraded ||= linear_degraded
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
merge_card_matches!(text, matches)
|
|
405
|
+
if wall_two_seconds?(text, started)
|
|
406
|
+
degraded = true
|
|
407
|
+
end
|
|
408
|
+
matches = matches.uniq { |match| [match.type, match.start, match.finish] }.sort_by(&:start)
|
|
409
|
+
warning = degraded ? LINEAR_WARNING : nil
|
|
410
|
+
Detection.new(matches: matches, degraded: degraded, coverage: "rules_only", warning: warning)
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
# The normal window pass owns this prefix. When its cap or deadline is
|
|
414
|
+
# reached on very large inputs, scan the remainder with the bounded,
|
|
415
|
+
# byte-oriented reserve in chunks. Overlap allows tokens crossing chunk
|
|
416
|
+
# boundaries; duplicate spans are removed by the caller.
|
|
417
|
+
def scan_linear_reserve_chunks(text, scanned_end_byte, out, deadline:)
|
|
418
|
+
total = text.bytesize
|
|
419
|
+
reserve_started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
420
|
+
deadline = [deadline, reserve_started + LINEAR_RESERVE_SECONDS].min
|
|
421
|
+
core_start = [scanned_end_byte - LINEAR_RESERVE_OVERLAP_BYTES, 0].max
|
|
422
|
+
core_start = utf8_boundary(text, core_start, backwards: true)
|
|
423
|
+
ascii_offsets = text.ascii_only?
|
|
424
|
+
base_chars = ascii_offsets ? core_start : text.byteslice(0...core_start).length
|
|
425
|
+
first_allowed = ascii_offsets ? [scanned_end_byte, core_start].min :
|
|
426
|
+
text.byteslice(0...[scanned_end_byte, core_start].min).length
|
|
427
|
+
while core_start < total
|
|
428
|
+
return [core_start, total] if monotonic_now >= deadline
|
|
429
|
+
|
|
430
|
+
core_end = utf8_boundary(text, [core_start + LINEAR_RESERVE_CHUNK_BYTES, total].min, backwards: true)
|
|
431
|
+
core_end = total if core_end <= core_start
|
|
432
|
+
chunk_end = utf8_boundary(text, [core_end + LINEAR_RESERVE_OVERLAP_BYTES, total].min, backwards: true)
|
|
433
|
+
chunk_end = total if chunk_end < core_end
|
|
434
|
+
chunk = text.byteslice(core_start...chunk_end)
|
|
435
|
+
chunk_matches = []
|
|
436
|
+
begin
|
|
437
|
+
linear_matches(chunk, chunk_matches, reserve_only: true, deadline: deadline)
|
|
438
|
+
rescue ReserveDeadlineReached
|
|
439
|
+
return [core_start, total]
|
|
440
|
+
end
|
|
441
|
+
core_chars = base_chars + (ascii_offsets ? core_end - core_start : text.byteslice(core_start...core_end).length)
|
|
442
|
+
chunk_matches.each do |match|
|
|
443
|
+
absolute_start = base_chars + match.start
|
|
444
|
+
next if absolute_start < first_allowed
|
|
445
|
+
next if absolute_start >= core_chars
|
|
446
|
+
|
|
447
|
+
out << PiiMatch.new(type: match.type, start: absolute_start,
|
|
448
|
+
finish: absolute_start + match.value.length, value: match.value)
|
|
449
|
+
end
|
|
450
|
+
base_chars = core_chars
|
|
451
|
+
core_start = core_end
|
|
452
|
+
end
|
|
453
|
+
nil
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def pattern_possible?(type, window)
|
|
457
|
+
case type
|
|
458
|
+
when :email then window.include?("@")
|
|
459
|
+
when :phone then window.match?(/\d{3}/)
|
|
460
|
+
when :ssn then window.match?(/\d{4}/)
|
|
461
|
+
when :iban then window.match?(/\b[A-Z]{2}\d{2}/)
|
|
462
|
+
when :ip then window.count(".") >= 3 || window.include?(":")
|
|
463
|
+
when :ccnum then window.match?(/\b\d(?:[ \-.]?\d){12,18}\b/)
|
|
464
|
+
when :api_key
|
|
465
|
+
window.match?(API_KEY_CUE_RE)
|
|
466
|
+
when :auth_jwt then window.include?("eyJ") && window.count(".") >= 2
|
|
467
|
+
when :nid_generic_labelled
|
|
468
|
+
window.match?(NATIONAL_ID_CUE_RE)
|
|
469
|
+
when :date_of_birth
|
|
470
|
+
window.match?(DOB_CUE_RE)
|
|
471
|
+
else true
|
|
472
|
+
end
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
# A linear Unicode-token reserve for numbered addresses. The canonical
|
|
476
|
+
# expression remains useful on short text; larger text uses this scanner
|
|
477
|
+
# and an explicit anchor budget instead of spending unbounded regex work.
|
|
478
|
+
def scan_address_reserve(text, out)
|
|
479
|
+
words = []
|
|
480
|
+
byte_offset = 0
|
|
481
|
+
char_offset = 0
|
|
482
|
+
total_bytes = text.bytesize
|
|
483
|
+
token_byte = nil
|
|
484
|
+
token_char = nil
|
|
485
|
+
while byte_offset < total_bytes
|
|
486
|
+
first = text.getbyte(byte_offset)
|
|
487
|
+
if first < 0x80
|
|
488
|
+
size = 1
|
|
489
|
+
word_char = first.between?(48, 57) || first.between?(65, 90) || first.between?(97, 122) ||
|
|
490
|
+
first == 39 || first == 45
|
|
491
|
+
else
|
|
492
|
+
size = if first < 0xE0
|
|
493
|
+
2
|
|
494
|
+
elsif first < 0xF0
|
|
495
|
+
3
|
|
496
|
+
else
|
|
497
|
+
4
|
|
498
|
+
end
|
|
499
|
+
char = text.byteslice(byte_offset, size)
|
|
500
|
+
word_char = char.match?(/\A[\p{L}\p{N}]\z/u) || char == "’"
|
|
501
|
+
end
|
|
502
|
+
if word_char
|
|
503
|
+
token_byte ||= byte_offset
|
|
504
|
+
token_char ||= char_offset
|
|
505
|
+
elsif token_byte
|
|
506
|
+
value = text.byteslice(token_byte...byte_offset)
|
|
507
|
+
words << address_word(value, token_byte, byte_offset, token_char, char_offset)
|
|
508
|
+
token_byte = token_char = nil
|
|
509
|
+
end
|
|
510
|
+
byte_offset += size
|
|
511
|
+
char_offset += 1
|
|
512
|
+
end
|
|
513
|
+
if token_byte
|
|
514
|
+
words << address_word(text.byteslice(token_byte...byte_offset), token_byte, byte_offset,
|
|
515
|
+
token_char, char_offset)
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
anchor_window = nil
|
|
519
|
+
window_anchor_count = 0
|
|
520
|
+
anchor_capped = false
|
|
521
|
+
words.each_with_index do |word, i|
|
|
522
|
+
next unless word[5]
|
|
523
|
+
|
|
524
|
+
digits = word[6]
|
|
525
|
+
next if digits.zero? || digits > 5
|
|
526
|
+
|
|
527
|
+
window = word[1] / WINDOW_BYTES
|
|
528
|
+
if window != anchor_window
|
|
529
|
+
anchor_window = window
|
|
530
|
+
window_anchor_count = 0
|
|
531
|
+
end
|
|
532
|
+
window_anchor_count += 1
|
|
533
|
+
anchor_capped ||= window_anchor_count > ADDRESS_ANCHOR_BUDGET
|
|
534
|
+
|
|
535
|
+
trailing_address = false
|
|
536
|
+
if digits <= 3 && i.positive?
|
|
537
|
+
([i - 4, 0].max..i - 1).reverse_each do |j|
|
|
538
|
+
break if words[j][5]
|
|
539
|
+
if words[j][7] &&
|
|
540
|
+
(!standalone_address_suffix?(words[j]) ||
|
|
541
|
+
(j.positive? && !words[j - 1][5] && !words[j - 1][7] &&
|
|
542
|
+
words[j - 1][8] && words[j - 1][0].bytesize <= 128))
|
|
543
|
+
first = j
|
|
544
|
+
2.times do
|
|
545
|
+
previous = first - 1
|
|
546
|
+
break if previous.negative? || !words[previous][8]
|
|
547
|
+
break unless whitespace_gap?(text, words[previous][2], words[first][1])
|
|
548
|
+
first = previous
|
|
549
|
+
end
|
|
550
|
+
if address_span_allowed?(text, words[first], word)
|
|
551
|
+
add_address_match(out, text, words[first], word)
|
|
552
|
+
trailing_address = true
|
|
553
|
+
end
|
|
554
|
+
break
|
|
555
|
+
end
|
|
556
|
+
end
|
|
557
|
+
end
|
|
558
|
+
|
|
559
|
+
name_or_number_start = word[5] || (!word[7] && word[8] && word[0].bytesize <= 128)
|
|
560
|
+
if !trailing_address && i + 1 < words.length && name_or_number_start &&
|
|
561
|
+
(i.zero? || leading_address_cue?(text, words, i))
|
|
562
|
+
((i + 1)..[i + 5, words.length - 1].min).each do |j|
|
|
563
|
+
break if words[j][5]
|
|
564
|
+
if words[j][7] && (word[5] || (j + 1 < words.length && words[j + 1][5]))
|
|
565
|
+
add_address_match(out, text, word, words[j]) if address_span_allowed?(text, word, words[j])
|
|
566
|
+
break
|
|
567
|
+
end
|
|
568
|
+
end
|
|
569
|
+
end
|
|
570
|
+
end
|
|
571
|
+
anchor_capped || text.bytesize > WINDOW_BYTES
|
|
572
|
+
end
|
|
573
|
+
|
|
574
|
+
def address_word(value, byte_start, byte_end, char_start, char_end)
|
|
575
|
+
bytes = value
|
|
576
|
+
digit_count = 0
|
|
577
|
+
digit_count += 1 while digit_count < bytes.bytesize && bytes.getbyte(digit_count).between?(48, 57)
|
|
578
|
+
numeric = digit_count == bytes.bytesize
|
|
579
|
+
suffix_letter = digit_count.between?(1, 5) && digit_count + 1 == bytes.bytesize &&
|
|
580
|
+
(bytes.getbyte(digit_count).between?(65, 90) || bytes.getbyte(digit_count).between?(97, 122))
|
|
581
|
+
number = numeric || suffix_letter
|
|
582
|
+
digit_count = 0 unless number
|
|
583
|
+
# Numeric anchors cannot be street suffixes. Avoid title-case checks for
|
|
584
|
+
# every number, and pass the already-computed title flag through rather
|
|
585
|
+
# than evaluating it again inside the suffix lookup.
|
|
586
|
+
title_word = !number && street_title_word?(value)
|
|
587
|
+
suffix_word = title_word && street_suffix?(value)
|
|
588
|
+
[value, byte_start, byte_end, char_start, char_end, number, digit_count, suffix_word, title_word]
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
def add_address_match(out, text, first, last)
|
|
592
|
+
value = text.byteslice(first[1]...last[2])
|
|
593
|
+
out << PiiMatch.new(type: :address, start: first[3], finish: last[4], value: value)
|
|
594
|
+
end
|
|
595
|
+
|
|
596
|
+
def address_span_allowed?(text, first, last)
|
|
597
|
+
span = text.byteslice(first[1]...last[2])
|
|
598
|
+
!span.match?(/[|\r\n]/)
|
|
599
|
+
end
|
|
600
|
+
|
|
601
|
+
def leading_address_cue?(text, words, index)
|
|
602
|
+
previous = words[index - 1]
|
|
603
|
+
gap = text.byteslice(previous[2]...words[index][1])
|
|
604
|
+
return true if gap.match?(/[|,.;:!?\n\r]/)
|
|
605
|
+
|
|
606
|
+
%w[to at address send mail live from].include?(previous[0].downcase)
|
|
607
|
+
end
|
|
608
|
+
|
|
609
|
+
def street_suffix?(word)
|
|
610
|
+
lower = word.downcase
|
|
611
|
+
candidates = ADDRESS_STREET_SUFFIXES_BY_END[lower[-2, 2]]
|
|
612
|
+
candidates && candidates.any? { |suffix| lower.end_with?(suffix) }
|
|
613
|
+
end
|
|
614
|
+
|
|
615
|
+
def standalone_address_suffix?(word)
|
|
616
|
+
ADDRESS_STREET_SUFFIXES.include?(word[0].downcase)
|
|
617
|
+
end
|
|
618
|
+
|
|
619
|
+
def whitespace_gap?(text, first, last)
|
|
620
|
+
return false if first >= last
|
|
621
|
+
|
|
622
|
+
while first < last
|
|
623
|
+
byte = text.getbyte(first)
|
|
624
|
+
return false unless byte == 32 || byte.between?(9, 13)
|
|
625
|
+
|
|
626
|
+
first += 1
|
|
627
|
+
end
|
|
628
|
+
true
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
def street_title_word?(word)
|
|
632
|
+
first = word[0]
|
|
633
|
+
first && first == first.upcase && first != first.downcase
|
|
634
|
+
end
|
|
635
|
+
|
|
636
|
+
# Wall-clock degradation is reserved for inputs larger than one core
|
|
637
|
+
# window or a genuinely hung scan. Ordinary short text is governed only by
|
|
638
|
+
# byte windows and the deterministic per-class match-step limit.
|
|
639
|
+
def monotonic_now
|
|
640
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
641
|
+
end
|
|
642
|
+
|
|
643
|
+
# Same 2s wall the tokenizer uses. Sets the flag tokenize_budget_hit?
|
|
644
|
+
# reads, and tells the caller to stop this pass.
|
|
645
|
+
def wall_two_seconds?(text, started)
|
|
646
|
+
return false if text.bytesize <= SHORT_COMPAT_BYTES
|
|
647
|
+
return false if Process.clock_gettime(Process::CLOCK_MONOTONIC) - started < 2.0
|
|
648
|
+
|
|
649
|
+
self.class.note_wall_budget_hit!
|
|
650
|
+
true
|
|
651
|
+
end
|
|
652
|
+
|
|
653
|
+
def emergency_expired?(text, started)
|
|
654
|
+
return false if text.bytesize > ADDRESS_WINDOW_SCAN_BYTES && text.bytesize <= MAX_ADDRESS_RESERVE_BYTES
|
|
655
|
+
|
|
656
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
657
|
+
limit = text.bytesize > WINDOW_BYTES ? EMERGENCY_WALL_SECONDS : SHORT_INPUT_HANG_SECONDS
|
|
658
|
+
elapsed >= limit
|
|
659
|
+
end
|
|
660
|
+
|
|
661
|
+
# Byte boundaries are moved to UTF-8 character boundaries before slicing.
|
|
662
|
+
# Every core is at most WINDOW_BYTES and the right overlap is WINDOW_BYTES.
|
|
663
|
+
def each_window(text)
|
|
664
|
+
total = text.bytesize
|
|
665
|
+
core_start = 0
|
|
666
|
+
while core_start < total
|
|
667
|
+
core_end = utf8_boundary(text, [core_start + WINDOW_BYTES, total].min, backwards: true)
|
|
668
|
+
core_end = total if core_end <= core_start
|
|
669
|
+
window_end = utf8_boundary(text, [core_end + WINDOW_BYTES, total].min, backwards: true)
|
|
670
|
+
window_end = total if window_end < core_end
|
|
671
|
+
yield text.byteslice(core_start...window_end), core_start, core_end
|
|
672
|
+
core_start = core_end
|
|
673
|
+
end
|
|
674
|
+
end
|
|
675
|
+
|
|
676
|
+
def owns_match_start?(global_byte, core_end_byte)
|
|
677
|
+
global_byte < core_end_byte
|
|
678
|
+
end
|
|
679
|
+
|
|
680
|
+
def span_too_long?(span_bytes)
|
|
681
|
+
span_bytes > MAX_SPAN_BYTES
|
|
682
|
+
end
|
|
683
|
+
|
|
684
|
+
def utf8_boundary(text, offset, backwards:)
|
|
685
|
+
return offset if offset >= text.bytesize
|
|
686
|
+
return offset if (text.getbyte(offset) & 0xC0) != 0x80
|
|
687
|
+
|
|
688
|
+
if backwards
|
|
689
|
+
offset -= 1 while offset.positive? && (text.getbyte(offset) & 0xC0) == 0x80
|
|
690
|
+
offset
|
|
691
|
+
else
|
|
692
|
+
offset += 1 while offset < text.bytesize && (text.getbyte(offset) & 0xC0) == 0x80
|
|
693
|
+
offset
|
|
694
|
+
end
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
# These scans are deliberately byte-oriented and have no regex/backtracking.
|
|
698
|
+
# They run over the complete input, including text after regex timeout.
|
|
699
|
+
def linear_matches(text, out, reserve_only: false, deadline: nil)
|
|
700
|
+
bytes = text.b
|
|
701
|
+
ascii_offsets = text.ascii_only?
|
|
702
|
+
lower = bytes.downcase
|
|
703
|
+
digit_run = bytes.bytesize > 64 && bytes.match?(ALL_DIGITS_RE)
|
|
704
|
+
numeric_candidate = !digit_run && bytes.match?(NUMERIC_FALLBACK_RE)
|
|
705
|
+
iban_candidate = !digit_run && bytes.match?(IBAN_SHAPE_RE)
|
|
706
|
+
nid_candidate = !reserve_only && !digit_run && bytes.match?(NATIONAL_ID_SHAPE_RE)
|
|
707
|
+
email_candidate = bytes.include?("@")
|
|
708
|
+
credential_candidate = %w[api key secret token bearer sk-live- sk-test- sk-prod- sk-proj- pk-live- rk-live- ak-live- eyj].any? do |cue|
|
|
709
|
+
lower.include?(cue)
|
|
710
|
+
end
|
|
711
|
+
labeled_id_candidate = NATIONAL_ID_CUES.any? { |cue| lower.include?(cue.b) }
|
|
712
|
+
dob_candidate = DOB_CUES.any? { |cue| lower.include?(cue.b) }
|
|
713
|
+
mrz_candidate = !reserve_only && bytes.include?("<<")
|
|
714
|
+
return false unless digit_run || numeric_candidate || iban_candidate || nid_candidate || email_candidate ||
|
|
715
|
+
credential_candidate || labeled_id_candidate || dob_candidate || mrz_candidate
|
|
716
|
+
|
|
717
|
+
digit_positions = !digit_run && (numeric_candidate || iban_candidate || nid_candidate) ?
|
|
718
|
+
digit_offsets(bytes) : []
|
|
719
|
+
overlong_email = scan_email(bytes, out, deadline: deadline, ascii_offsets: ascii_offsets)
|
|
720
|
+
overlong_number = digit_run
|
|
721
|
+
overlong_number ||= scan_numbers(bytes, digit_positions, out, ascii_offsets: ascii_offsets) if numeric_candidate
|
|
722
|
+
overlong_nid = scan_national_ids(bytes, digit_positions, out, ascii_offsets: ascii_offsets) if nid_candidate
|
|
723
|
+
overlong_iban = scan_iban(bytes, digit_positions, out, ascii_offsets: ascii_offsets) if iban_candidate
|
|
724
|
+
overlong_credential = scan_credentials(bytes, lower, out, ascii_offsets: ascii_offsets)
|
|
725
|
+
overlong_id = scan_labeled_ids(bytes, lower, out, ascii_offsets: ascii_offsets)
|
|
726
|
+
overlong_dob = scan_marked_dob(bytes, lower, out, ascii_offsets: ascii_offsets)
|
|
727
|
+
overlong_mrz = scan_mrz(bytes, out, ascii_offsets: ascii_offsets)
|
|
728
|
+
protected = out.select do |match|
|
|
729
|
+
%i[ccnum iban ssn nid_mrz].include?(match.type) ||
|
|
730
|
+
(match.type == :nid_generic_labelled && match.value.match?(/[A-Za-z]/))
|
|
731
|
+
end
|
|
732
|
+
protected.sort_by!(&:start)
|
|
733
|
+
protected_index = 0
|
|
734
|
+
out.reject! do |match|
|
|
735
|
+
next false unless match.type == :phone
|
|
736
|
+
|
|
737
|
+
protected_index += 1 while protected_index < protected.length && protected[protected_index].finish <= match.start
|
|
738
|
+
other = protected[protected_index]
|
|
739
|
+
other && match.start < other.finish && other.start < match.finish
|
|
740
|
+
end
|
|
741
|
+
overlong_email || overlong_number || overlong_nid || overlong_iban || overlong_credential ||
|
|
742
|
+
overlong_id || overlong_dob || overlong_mrz
|
|
743
|
+
end
|
|
744
|
+
|
|
745
|
+
def digit_offsets(bytes)
|
|
746
|
+
positions = []
|
|
747
|
+
bytes.each_byte.with_index { |byte, index| positions << index if digit?(byte) }
|
|
748
|
+
positions
|
|
749
|
+
end
|
|
750
|
+
|
|
751
|
+
def add_bytes(text, out, type, first, last)
|
|
752
|
+
return if last <= first
|
|
753
|
+
|
|
754
|
+
value = text.byteslice(first...last).force_encoding(text.encoding)
|
|
755
|
+
return unless value.valid_encoding?
|
|
756
|
+
|
|
757
|
+
# Convert the detector's byte offsets to the character offsets consumed
|
|
758
|
+
# by the existing tokenizer API.
|
|
759
|
+
start = if text.encoding == Encoding::BINARY && text.ascii_only?
|
|
760
|
+
first
|
|
761
|
+
else
|
|
762
|
+
prefix = text.byteslice(0...first).force_encoding(text.encoding)
|
|
763
|
+
prefix.valid_encoding? ? prefix.length : first
|
|
764
|
+
end
|
|
765
|
+
out << PiiMatch.new(type: type, start: start, finish: start + value.length, value: value)
|
|
766
|
+
end
|
|
767
|
+
|
|
768
|
+
def scan_email(bytes, out, deadline: nil, ascii_offsets: false)
|
|
769
|
+
overlong = false
|
|
770
|
+
i = 0
|
|
771
|
+
candidates = 0
|
|
772
|
+
while (i = bytes.index("@", i))
|
|
773
|
+
candidates += 1
|
|
774
|
+
raise ReserveDeadlineReached if deadline && (candidates & 1023).zero? && monotonic_now >= deadline
|
|
775
|
+
|
|
776
|
+
left = i
|
|
777
|
+
left -= 1 while left.positive? && email_local_byte?(bytes.getbyte(left - 1)) && i - left < 64
|
|
778
|
+
right = i + 1
|
|
779
|
+
right += 1 while right < bytes.bytesize && email_domain_byte?(bytes.getbyte(right)) && right - i <= 256
|
|
780
|
+
candidate = bytes.byteslice(left...right)
|
|
781
|
+
local, domain = candidate.split("@", 2)
|
|
782
|
+
local_truncated = left.positive? && email_local_byte?(bytes.getbyte(left - 1))
|
|
783
|
+
domain_truncated = right < bytes.bytesize && email_domain_byte?(bytes.getbyte(right))
|
|
784
|
+
overlong ||= local_truncated || domain_truncated
|
|
785
|
+
if !local_truncated && local && domain && !local.empty? && local.bytesize <= 64 && domain.bytesize <= 255 &&
|
|
786
|
+
domain.include?(".") && domain !~ /\.\z|\.\./
|
|
787
|
+
if ascii_offsets
|
|
788
|
+
value = candidate.dup.force_encoding("UTF-8")
|
|
789
|
+
out << PiiMatch.new(type: :email, start: left, finish: right, value: value)
|
|
790
|
+
else
|
|
791
|
+
add_bytes(bytes, out, :email, left, right)
|
|
792
|
+
end
|
|
793
|
+
end
|
|
794
|
+
i += 1
|
|
795
|
+
end
|
|
796
|
+
overlong
|
|
797
|
+
end
|
|
798
|
+
|
|
799
|
+
def email_local_byte?(b)
|
|
800
|
+
b && (b >= 128 || b.between?(48, 57) || b.between?(65, 90) || b.between?(97, 122) || "._%+-".bytes.include?(b))
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
def email_domain_byte?(b)
|
|
804
|
+
b && (b >= 128 || b.between?(48, 57) || b.between?(65, 90) || b.between?(97, 122) || b == 45 || b == 46)
|
|
805
|
+
end
|
|
806
|
+
|
|
807
|
+
def match_continues?(type, byte)
|
|
808
|
+
case type
|
|
809
|
+
when :email
|
|
810
|
+
email_local_byte?(byte) || email_domain_byte?(byte)
|
|
811
|
+
when :phone, :ssn, :ccnum
|
|
812
|
+
# A space or hyphen before a number is a delimiter, not evidence that
|
|
813
|
+
# this window clipped the number. Only an adjacent digit can continue
|
|
814
|
+
# one of these numeric spans across a core boundary.
|
|
815
|
+
digit?(byte)
|
|
816
|
+
when :iban
|
|
817
|
+
letter?(byte) || digit?(byte) || byte == 32
|
|
818
|
+
when :api_key
|
|
819
|
+
credential_byte?(byte)
|
|
820
|
+
when :auth_jwt
|
|
821
|
+
credential_byte?(byte) || byte == 46
|
|
822
|
+
when :nid_generic_labelled, :address, :date_of_birth
|
|
823
|
+
alnum_byte?(byte) || (byte && byte >= 128)
|
|
824
|
+
else
|
|
825
|
+
false
|
|
826
|
+
end
|
|
827
|
+
end
|
|
828
|
+
|
|
829
|
+
def scan_numbers(bytes, digit_positions, out, ascii_offsets: false)
|
|
830
|
+
overlong = false
|
|
831
|
+
position = 0
|
|
832
|
+
while position < digit_positions.length
|
|
833
|
+
i = digit_positions[position]
|
|
834
|
+
if i.positive? && alnum_byte?(bytes.getbyte(i - 1))
|
|
835
|
+
position += 1
|
|
836
|
+
next
|
|
837
|
+
end
|
|
838
|
+
finish = i
|
|
839
|
+
digits = 0
|
|
840
|
+
separator_run = 0
|
|
841
|
+
while finish < bytes.bytesize && finish - i < 64
|
|
842
|
+
b = bytes.getbyte(finish)
|
|
843
|
+
if digit?(b)
|
|
844
|
+
digits += 1
|
|
845
|
+
separator_run = 0
|
|
846
|
+
elsif [32, 45, 46, 40, 41, 43].include?(b)
|
|
847
|
+
separator_run += 1
|
|
848
|
+
break if separator_run > 2
|
|
849
|
+
else
|
|
850
|
+
break
|
|
851
|
+
end
|
|
852
|
+
finish += 1
|
|
853
|
+
end
|
|
854
|
+
candidate_overlong = finish - i >= 64 && finish < bytes.bytesize &&
|
|
855
|
+
(digit?(bytes.getbyte(finish)) || [32, 45, 46].include?(bytes.getbyte(finish)))
|
|
856
|
+
overlong ||= candidate_overlong
|
|
857
|
+
finish -= 1 while finish > i && [32, 45, 46, 40, 41, 43].include?(bytes.getbyte(finish - 1))
|
|
858
|
+
raw = bytes.byteslice(i...finish)
|
|
859
|
+
compact = raw.delete("- .()+")
|
|
860
|
+
if !candidate_overlong && digits == 9 && raw.bytesize <= 11 && valid_ssn?(compact)
|
|
861
|
+
out << byte_match(bytes, :ssn, i, finish, ascii_offsets: ascii_offsets)
|
|
862
|
+
elsif !candidate_overlong && digits.between?(13, 19) && raw.bytesize <= 64 && luhn?(compact) && supported_card_network?(compact)
|
|
863
|
+
out << byte_match(bytes, :ccnum, i, finish, ascii_offsets: ascii_offsets)
|
|
864
|
+
elsif !candidate_overlong && !alnum_byte?(bytes.getbyte(finish)) && phone_candidate?(raw)
|
|
865
|
+
out << byte_match(bytes, :phone, i, finish, ascii_offsets: ascii_offsets)
|
|
866
|
+
end
|
|
867
|
+
position += 1
|
|
868
|
+
position += 1 while position < digit_positions.length && digit_positions[position] < finish
|
|
869
|
+
end
|
|
870
|
+
overlong
|
|
871
|
+
end
|
|
872
|
+
|
|
873
|
+
# Keep the reserve scanner's phone formats no broader than the legacy
|
|
874
|
+
# detector. Arbitrary 7+ digit identifiers are not phone numbers.
|
|
875
|
+
def phone_candidate?(raw)
|
|
876
|
+
match = PATTERNS.fetch(:phone).match(raw)
|
|
877
|
+
match && match.begin(0).zero? && match.end(0) == raw.bytesize
|
|
878
|
+
end
|
|
879
|
+
|
|
880
|
+
# The canonical ruleset marks structured national-ID families with a
|
|
881
|
+
# checksum name. Keep this reserve byte-oriented and bounded; do not run
|
|
882
|
+
# the canonical regexes again after their budget has expired.
|
|
883
|
+
def scan_national_ids(bytes, digit_positions, out, ascii_offsets: false)
|
|
884
|
+
return if digit_positions.empty?
|
|
885
|
+
|
|
886
|
+
position = 0
|
|
887
|
+
while position < digit_positions.length
|
|
888
|
+
first = digit_positions[position]
|
|
889
|
+
probe = first
|
|
890
|
+
candidate_digits = 0
|
|
891
|
+
separator_run = 0
|
|
892
|
+
while probe < bytes.bytesize && probe - first < 64
|
|
893
|
+
byte = bytes.getbyte(probe)
|
|
894
|
+
if digit?(byte)
|
|
895
|
+
candidate_digits += 1
|
|
896
|
+
separator_run = 0
|
|
897
|
+
elsif [32, 45, 46, 47].include?(byte)
|
|
898
|
+
separator_run += 1
|
|
899
|
+
break if separator_run > 1
|
|
900
|
+
else
|
|
901
|
+
break
|
|
902
|
+
end
|
|
903
|
+
probe += 1
|
|
904
|
+
end
|
|
905
|
+
if candidate_digits < 8
|
|
906
|
+
cue_context = bytes.byteslice([first - 40, 0].max...first)
|
|
907
|
+
unless NATIONAL_ID_CHECKSUM_CUE_RE.match?(cue_context)
|
|
908
|
+
position += 1
|
|
909
|
+
position += 1 while position < digit_positions.length && digit_positions[position] < probe
|
|
910
|
+
next
|
|
911
|
+
end
|
|
912
|
+
end
|
|
913
|
+
unless first.zero? || !alnum_byte?(bytes.getbyte(first - 1))
|
|
914
|
+
position += 1
|
|
915
|
+
next
|
|
916
|
+
end
|
|
917
|
+
|
|
918
|
+
last = first
|
|
919
|
+
digits = 0
|
|
920
|
+
separator_run = 0
|
|
921
|
+
while last < bytes.bytesize && last - first < 64
|
|
922
|
+
byte = bytes.getbyte(last)
|
|
923
|
+
if digit?(byte) || letter?(byte)
|
|
924
|
+
digits += 1 if digit?(byte)
|
|
925
|
+
separator_run = 0
|
|
926
|
+
last += 1
|
|
927
|
+
elsif [32, 45, 46, 47].include?(byte)
|
|
928
|
+
separator_run += 1
|
|
929
|
+
break if separator_run > 1
|
|
930
|
+
last += 1
|
|
931
|
+
else
|
|
932
|
+
break
|
|
933
|
+
end
|
|
934
|
+
end
|
|
935
|
+
overlong = last - first >= 64 && last < bytes.bytesize &&
|
|
936
|
+
(digit?(bytes.getbyte(last)) || letter?(bytes.getbyte(last)) ||
|
|
937
|
+
[32, 45, 46, 47].include?(bytes.getbyte(last)))
|
|
938
|
+
unless overlong
|
|
939
|
+
candidate = bytes.byteslice(first...last)
|
|
940
|
+
cue_context = bytes.byteslice([first - 40, 0].max...first).downcase
|
|
941
|
+
if national_id_checksum_valid?(candidate, digits, cue_context)
|
|
942
|
+
out << byte_match(bytes, :nid_generic_labelled, first, last, ascii_offsets: ascii_offsets)
|
|
943
|
+
end
|
|
944
|
+
end
|
|
945
|
+
position += 1
|
|
946
|
+
position += 1 while position < digit_positions.length && digit_positions[position] < last
|
|
947
|
+
end
|
|
948
|
+
token_start = 0
|
|
949
|
+
while token_start < bytes.bytesize
|
|
950
|
+
unless alnum_byte?(bytes.getbyte(token_start)) &&
|
|
951
|
+
(token_start.zero? || !alnum_byte?(bytes.getbyte(token_start - 1)))
|
|
952
|
+
token_start += 1
|
|
953
|
+
next
|
|
954
|
+
end
|
|
955
|
+
token_end = token_start
|
|
956
|
+
token_end += 1 while token_end < bytes.bytesize && alnum_byte?(bytes.getbyte(token_end)) &&
|
|
957
|
+
token_end - token_start < 64
|
|
958
|
+
overlong ||= token_end - token_start == 64 && token_end < bytes.bytesize && alnum_byte?(bytes.getbyte(token_end))
|
|
959
|
+
if token_end - token_start == 18
|
|
960
|
+
value = bytes.byteslice(token_start...token_end).upcase
|
|
961
|
+
if value.match?(/\A[A-Z]{4}\d{6}[HM][A-Z]{5}[A-Z0-9]\d\z/) && valid_mx_curp?(value)
|
|
962
|
+
out << byte_match(bytes, :nid_generic_labelled, token_start, token_end)
|
|
963
|
+
end
|
|
964
|
+
end
|
|
965
|
+
token_start = [token_end, token_start + 1].max
|
|
966
|
+
end
|
|
967
|
+
overlong
|
|
968
|
+
end
|
|
969
|
+
|
|
970
|
+
def alnum_byte?(byte)
|
|
971
|
+
digit?(byte) || letter?(byte)
|
|
972
|
+
end
|
|
973
|
+
|
|
974
|
+
def id_word_byte?(byte)
|
|
975
|
+
alnum_byte?(byte) || byte == 95 || (byte && byte >= 128)
|
|
976
|
+
end
|
|
977
|
+
|
|
978
|
+
def national_id_checksum_valid?(candidate, digits, cue_context)
|
|
979
|
+
compact = candidate.upcase.delete("^A-Z0-9")
|
|
980
|
+
case digits
|
|
981
|
+
when 8
|
|
982
|
+
candidate.include?("-") && valid_cl_rut?(compact)
|
|
983
|
+
when 9
|
|
984
|
+
(candidate.include?("-") && valid_cl_rut?(compact)) ||
|
|
985
|
+
(cue_context.include?("bsn") && valid_bsn?(compact)) ||
|
|
986
|
+
((cue_context.include?("sin") || cue_context.include?("social insurance number")) && valid_luhn?(compact))
|
|
987
|
+
when 10
|
|
988
|
+
valid_cz_sk_rc?(compact) || valid_nid_bare10?(compact)
|
|
989
|
+
when 11
|
|
990
|
+
return valid_pl_pesel?(compact) if cue_context.include?("pesel")
|
|
991
|
+
((candidate.include?("-") || candidate.include?(".")) && valid_cpf?(compact)) ||
|
|
992
|
+
valid_be_nrn?(compact) || valid_nid_bare11?(compact)
|
|
993
|
+
when 12
|
|
994
|
+
return valid_verhoeff?(compact) if %w[aadhaar aadhar uid].any? { |cue| cue_context.include?(cue) }
|
|
995
|
+
return valid_jp_mynumber?(compact) if cue_context.include?("my number")
|
|
996
|
+
valid_verhoeff?(compact) || valid_jp_mynumber?(compact)
|
|
997
|
+
when 13
|
|
998
|
+
valid_nid_bare13?(compact)
|
|
999
|
+
when 16
|
|
1000
|
+
valid_id_nik?(compact)
|
|
1001
|
+
when 17
|
|
1002
|
+
valid_cn_ric?(compact)
|
|
1003
|
+
else
|
|
1004
|
+
false
|
|
1005
|
+
end
|
|
1006
|
+
end
|
|
1007
|
+
|
|
1008
|
+
def valid_verhoeff?(value)
|
|
1009
|
+
d = [[0,1,2,3,4,5,6,7,8,9],[1,2,3,4,0,6,7,8,9,5],[2,3,4,0,1,7,8,9,5,6],
|
|
1010
|
+
[3,4,0,1,2,8,9,5,6,7],[4,0,1,2,3,9,5,6,7,8],[5,9,8,7,6,0,4,3,2,1],
|
|
1011
|
+
[6,5,9,8,7,1,0,4,3,2],[7,6,5,9,8,2,1,0,4,3],[8,7,6,5,9,3,2,1,0,4],
|
|
1012
|
+
[9,8,7,6,5,4,3,2,1,0]]
|
|
1013
|
+
p = [[0,1,2,3,4,5,6,7,8,9],[1,5,7,6,2,8,3,0,9,4],[5,8,0,3,7,9,6,1,4,2],
|
|
1014
|
+
[8,9,1,6,0,4,3,5,2,7],[9,4,5,3,1,2,6,8,7,0],[4,2,8,6,5,7,3,9,0,1],
|
|
1015
|
+
[2,7,9,3,8,0,6,4,1,5],[7,0,4,6,9,1,3,2,5,8]]
|
|
1016
|
+
return false unless value.match?(/\A\d+\z/)
|
|
1017
|
+
c = 0
|
|
1018
|
+
value.reverse.each_char.with_index { |ch, i| c = d[c][p[i % 8][ch.ord - 48]] }
|
|
1019
|
+
c.zero?
|
|
1020
|
+
end
|
|
1021
|
+
|
|
1022
|
+
def valid_bsn?(value)
|
|
1023
|
+
return false unless value.match?(/\A\d{8,9}\z/)
|
|
1024
|
+
digits = value.chars.map(&:to_i)
|
|
1025
|
+
return false unless digits.length == 9
|
|
1026
|
+
(digits.first(8).zip((2..9).to_a.reverse).sum { |n, w| n * w } % 11) == digits[8]
|
|
1027
|
+
end
|
|
1028
|
+
|
|
1029
|
+
def valid_cpf?(value)
|
|
1030
|
+
return false unless value.match?(/\A\d{11}\z/) && value.chars.uniq.length > 1
|
|
1031
|
+
nums = value.chars.map(&:to_i)
|
|
1032
|
+
[9, 10].each do |length|
|
|
1033
|
+
sum = nums.first(length).each_with_index.sum { |n, i| n * (length + 1 - i) }
|
|
1034
|
+
check = (sum * 10) % 11
|
|
1035
|
+
check = 0 if check == 10
|
|
1036
|
+
return false unless check == nums[length]
|
|
1037
|
+
end
|
|
1038
|
+
true
|
|
1039
|
+
end
|
|
1040
|
+
|
|
1041
|
+
def valid_cl_rut?(value)
|
|
1042
|
+
return false unless value.match?(/\A\d{7,8}[\dK]\z/)
|
|
1043
|
+
body = value[0...-1].chars.reverse.map(&:to_i)
|
|
1044
|
+
sum = body.each_with_index.sum { |n, i| n * (i % 6 + 2) }
|
|
1045
|
+
rem = 11 - sum % 11
|
|
1046
|
+
expected = rem == 11 ? "0" : rem == 10 ? "K" : rem.to_s
|
|
1047
|
+
value[-1] == expected
|
|
1048
|
+
end
|
|
1049
|
+
|
|
1050
|
+
def valid_jp_mynumber?(value)
|
|
1051
|
+
return false unless value.match?(/\A\d{12}\z/)
|
|
1052
|
+
nums = value.chars.map(&:to_i)
|
|
1053
|
+
sum = (1..11).sum do |n|
|
|
1054
|
+
weight = n <= 6 ? n + 1 : n - 5
|
|
1055
|
+
nums[11 - n] * weight
|
|
1056
|
+
end
|
|
1057
|
+
remainder = sum % 11
|
|
1058
|
+
check = remainder <= 1 ? 0 : 11 - remainder
|
|
1059
|
+
nums[11] == check
|
|
1060
|
+
end
|
|
1061
|
+
|
|
1062
|
+
def valid_be_nrn?(value)
|
|
1063
|
+
return false unless value.match?(/\A\d{11}\z/)
|
|
1064
|
+
digits = value.to_i
|
|
1065
|
+
(97 - (digits / 100) % 97) == digits % 100 ||
|
|
1066
|
+
(97 - ((2_000_000_000 + digits / 100) % 97)) == digits % 100
|
|
1067
|
+
end
|
|
1068
|
+
|
|
1069
|
+
def valid_cz_sk_rc?(value)
|
|
1070
|
+
return false unless value.match?(/\A\d{10}\z/)
|
|
1071
|
+
d = value.chars.map(&:to_i)
|
|
1072
|
+
encoded_month = d[2] * 10 + d[3]
|
|
1073
|
+
month = case encoded_month
|
|
1074
|
+
when 1..12 then encoded_month
|
|
1075
|
+
when 21..32 then encoded_month - 20
|
|
1076
|
+
when 51..62 then encoded_month - 50
|
|
1077
|
+
when 71..82 then encoded_month - 70
|
|
1078
|
+
else return false
|
|
1079
|
+
end
|
|
1080
|
+
day = d[4] * 10 + d[5]
|
|
1081
|
+
year_valid = [1800, 1900, 2000, 2100].any? do |century|
|
|
1082
|
+
Date.valid_date?(century + d[0] * 10 + d[1], month, day)
|
|
1083
|
+
end
|
|
1084
|
+
year_valid && (value.to_i % 11).zero?
|
|
1085
|
+
end
|
|
1086
|
+
|
|
1087
|
+
def valid_cn_ric?(value)
|
|
1088
|
+
return false unless value.match?(/\A\d{17}[\dX]\z/)
|
|
1089
|
+
weights = [7,9,10,5,8,4,2,1,6,3,7,9,10,5,8,4,2]
|
|
1090
|
+
chars = "10X98765432"
|
|
1091
|
+
sum = value[0, 17].chars.each_with_index.sum { |ch, i| ch.to_i * weights[i] }
|
|
1092
|
+
value[-1] == chars[sum % 11]
|
|
1093
|
+
end
|
|
1094
|
+
|
|
1095
|
+
def valid_mx_curp?(value)
|
|
1096
|
+
return false unless value.match?(/\A[A-Z]{4}\d{6}[HM][A-Z]{5}[A-Z0-9]\d\z/)
|
|
1097
|
+
dictionary = "0123456789ABCDEFGHIJKLMNÑOPQRSTUVWXYZ"
|
|
1098
|
+
sum = value[0, 17].chars.each_with_index.sum do |ch, i|
|
|
1099
|
+
pos = dictionary.index(ch)
|
|
1100
|
+
return false unless pos
|
|
1101
|
+
pos * (18 - i)
|
|
1102
|
+
end
|
|
1103
|
+
value[-1].to_i == (10 - sum % 10) % 10
|
|
1104
|
+
end
|
|
1105
|
+
|
|
1106
|
+
def valid_luhn?(value)
|
|
1107
|
+
value.match?(/\A\d+\z/) && luhn?(value)
|
|
1108
|
+
end
|
|
1109
|
+
|
|
1110
|
+
def valid_nid_bare10?(value)
|
|
1111
|
+
return false unless value.match?(/\A\d{10}\z/)
|
|
1112
|
+
valid_bg_egn?(value) || valid_se_pnr?(value) || valid_at_svnr?(value) || valid_nid_bare10_legacy?(value)
|
|
1113
|
+
end
|
|
1114
|
+
|
|
1115
|
+
def valid_nid_bare10_legacy?(value)
|
|
1116
|
+
d = value.chars.map(&:to_i)
|
|
1117
|
+
month = d[2] * 10 + d[3]
|
|
1118
|
+
month -= 20 if month.between?(21, 32)
|
|
1119
|
+
month -= 40 if month.between?(41, 52)
|
|
1120
|
+
day = d[4] * 10 + d[5]
|
|
1121
|
+
year_valid = [1800, 1900, 2000, 2100].any? do |century|
|
|
1122
|
+
Date.valid_date?(century + d[0] * 10 + d[1], month, day)
|
|
1123
|
+
end
|
|
1124
|
+
return false unless year_valid
|
|
1125
|
+
weights = [2, 4, 8, 5, 10, 9, 7, 3, 6]
|
|
1126
|
+
check = d.first(9).each_with_index.sum { |n, i| n * weights[i] } % 11
|
|
1127
|
+
check = 0 if check == 10
|
|
1128
|
+
check == d[9]
|
|
1129
|
+
end
|
|
1130
|
+
|
|
1131
|
+
def valid_bg_egn?(value)
|
|
1132
|
+
d = value.chars.map(&:to_i)
|
|
1133
|
+
day = d[0] * 10 + d[1]
|
|
1134
|
+
encoded_month = d[2] * 10 + d[3]
|
|
1135
|
+
year_offset, month = case encoded_month
|
|
1136
|
+
when 1..12 then [1900, encoded_month]
|
|
1137
|
+
when 21..32 then [1800, encoded_month - 20]
|
|
1138
|
+
when 41..52 then [2000, encoded_month - 40]
|
|
1139
|
+
when 61..72 then [2100, encoded_month - 60]
|
|
1140
|
+
else return false
|
|
1141
|
+
end
|
|
1142
|
+
return false unless Date.valid_date?(year_offset + d[4] * 10 + d[5], month, day)
|
|
1143
|
+
weights = [2, 4, 8, 5, 10, 9, 7, 3, 6]
|
|
1144
|
+
check = d.first(9).each_with_index.sum { |n, i| n * weights[i] } % 11
|
|
1145
|
+
check = 0 if check == 10
|
|
1146
|
+
check == d[9]
|
|
1147
|
+
end
|
|
1148
|
+
|
|
1149
|
+
def valid_nid_bare11?(value)
|
|
1150
|
+
return false unless value.match?(/\A\d{11}\z/)
|
|
1151
|
+
valid_no_fnr?(value) || valid_hr_oib?(value) || valid_lt_ee_ik?(value) ||
|
|
1152
|
+
valid_tr_tckn?(value) || valid_pl_pesel?(value) || valid_ar_cuit?(value) ||
|
|
1153
|
+
valid_ru_snils?(value) || valid_be_nrn?(value)
|
|
1154
|
+
end
|
|
1155
|
+
|
|
1156
|
+
def valid_no_fnr?(value)
|
|
1157
|
+
return false unless value.match?(/\A\d{11}\z/)
|
|
1158
|
+
d = value.chars.map(&:to_i)
|
|
1159
|
+
day, month = d[0] * 10 + d[1], d[2] * 10 + d[3]
|
|
1160
|
+
day -= 40 if day.between?(41, 71)
|
|
1161
|
+
month -= 40 if month.between?(41, 52)
|
|
1162
|
+
return false unless valid_day_month?(day, month)
|
|
1163
|
+
w1 = [3,7,6,1,8,9,4,5,2]
|
|
1164
|
+
k1 = 11 - (0...9).sum { |i| d[i] * w1[i] } % 11
|
|
1165
|
+
k1 = 0 if k1 == 11
|
|
1166
|
+
return false if k1 == 10 || d[9] != k1
|
|
1167
|
+
w2 = [5,4,3,2,7,6,5,4,3,2]
|
|
1168
|
+
k2 = 11 - (0...10).sum { |i| d[i] * w2[i] } % 11
|
|
1169
|
+
k2 = 0 if k2 == 11
|
|
1170
|
+
k2 != 10 && d[10] == k2
|
|
1171
|
+
end
|
|
1172
|
+
|
|
1173
|
+
def valid_nid_bare13?(value)
|
|
1174
|
+
return false unless value.match?(/\A\d{13}\z/)
|
|
1175
|
+
valid_jmbg?(value) || valid_ro_cnp?(value) || valid_za_id?(value) || valid_kr_rrn?(value)
|
|
1176
|
+
end
|
|
1177
|
+
|
|
1178
|
+
def valid_jmbg?(value)
|
|
1179
|
+
d = value.chars.map(&:to_i)
|
|
1180
|
+
day, month = d[0] * 10 + d[1], d[2] * 10 + d[3]
|
|
1181
|
+
return false unless valid_day_month?(day, month)
|
|
1182
|
+
w = [7, 6, 5, 4, 3, 2, 7, 6, 5, 4, 3, 2]
|
|
1183
|
+
sum = d.first(12).each_with_index.sum { |n, i| n * w[i] }
|
|
1184
|
+
check = 11 - sum % 11
|
|
1185
|
+
check = 0 if check >= 10
|
|
1186
|
+
check == d[12]
|
|
1187
|
+
end
|
|
1188
|
+
|
|
1189
|
+
def valid_day_month?(day, month)
|
|
1190
|
+
day.between?(1, 31) && month.between?(1, 12)
|
|
1191
|
+
end
|
|
1192
|
+
|
|
1193
|
+
def valid_se_pnr?(value)
|
|
1194
|
+
d = value.chars.map(&:to_i)
|
|
1195
|
+
month = d[2] * 10 + d[3]
|
|
1196
|
+
day = d[4] * 10 + d[5]
|
|
1197
|
+
valid_day_month?(day, month) && luhn?(value)
|
|
1198
|
+
end
|
|
1199
|
+
|
|
1200
|
+
def valid_at_svnr?(value)
|
|
1201
|
+
d = value.chars.map(&:to_i)
|
|
1202
|
+
return false unless valid_day_month?(d[4] * 10 + d[5], d[6] * 10 + d[7])
|
|
1203
|
+
indices = [0, 1, 2, 4, 5, 6, 7, 8, 9]
|
|
1204
|
+
weights = [3, 7, 9, 5, 8, 4, 2, 1, 6]
|
|
1205
|
+
check = indices.each_with_index.sum { |idx, i| d[idx] * weights[i] } % 11
|
|
1206
|
+
check != 10 && check == d[3]
|
|
1207
|
+
end
|
|
1208
|
+
|
|
1209
|
+
def valid_hr_oib?(value)
|
|
1210
|
+
d = value.chars.map(&:to_i)
|
|
1211
|
+
m = 10
|
|
1212
|
+
10.times do |i|
|
|
1213
|
+
m = (m + d[i]) % 10
|
|
1214
|
+
m = 10 if m.zero?
|
|
1215
|
+
m = (m * 2) % 11
|
|
1216
|
+
end
|
|
1217
|
+
check = 11 - m
|
|
1218
|
+
check = 0 if check == 10
|
|
1219
|
+
check == d[10]
|
|
1220
|
+
end
|
|
1221
|
+
|
|
1222
|
+
def valid_lt_ee_ik?(value)
|
|
1223
|
+
d = value.chars.map(&:to_i)
|
|
1224
|
+
return false unless d[0].between?(1, 8) && valid_day_month?(d[5] * 10 + d[6], d[3] * 10 + d[4])
|
|
1225
|
+
w1 = [1,2,3,4,5,6,7,8,9,1]
|
|
1226
|
+
check = d.first(10).each_with_index.sum { |n, i| n * w1[i] } % 11
|
|
1227
|
+
if check == 10
|
|
1228
|
+
w2 = [3,4,5,6,7,8,9,1,2,3]
|
|
1229
|
+
check = d.first(10).each_with_index.sum { |n, i| n * w2[i] } % 11
|
|
1230
|
+
check = 0 if check == 10
|
|
1231
|
+
end
|
|
1232
|
+
check == d[10]
|
|
1233
|
+
end
|
|
1234
|
+
|
|
1235
|
+
def valid_tr_tckn?(value)
|
|
1236
|
+
d = value.chars.map(&:to_i)
|
|
1237
|
+
return false if d[0].zero? || d.first(10).sum % 10 != d[10]
|
|
1238
|
+
odd = [0,2,4,6,8].sum { |i| d[i] }
|
|
1239
|
+
even = [1,3,5,7].sum { |i| d[i] }
|
|
1240
|
+
((odd * 7 - even) % 10) == d[9]
|
|
1241
|
+
end
|
|
1242
|
+
|
|
1243
|
+
def valid_pl_pesel?(value)
|
|
1244
|
+
d = value.chars.map(&:to_i)
|
|
1245
|
+
month_raw = d[2] * 10 + d[3]
|
|
1246
|
+
month = [0,20,40,60,80].filter_map { |offset| month_raw - offset if (1 + offset..12 + offset).cover?(month_raw) }.first
|
|
1247
|
+
return false unless month && valid_day_month?(d[4] * 10 + d[5], month)
|
|
1248
|
+
w = [1,3,7,9,1,3,7,9,1,3]
|
|
1249
|
+
check = (10 - d.first(10).each_with_index.sum { |n, i| n * w[i] } % 10) % 10
|
|
1250
|
+
check == d[10]
|
|
1251
|
+
end
|
|
1252
|
+
|
|
1253
|
+
def valid_ar_cuit?(value)
|
|
1254
|
+
d = value.chars.map(&:to_i)
|
|
1255
|
+
w = [5,4,3,2,7,6,5,4,3,2]
|
|
1256
|
+
sum = d.first(10).each_with_index.sum { |n, i| n * w[i] }
|
|
1257
|
+
check = 11 - sum % 11
|
|
1258
|
+
check = 0 if check == 11
|
|
1259
|
+
check = 9 if check == 10
|
|
1260
|
+
check == d[10]
|
|
1261
|
+
end
|
|
1262
|
+
|
|
1263
|
+
def valid_ru_snils?(value)
|
|
1264
|
+
d = value.chars.map(&:to_i)
|
|
1265
|
+
sum = d.first(9).each_with_index.sum { |n, i| n * (9 - i) }
|
|
1266
|
+
check = sum < 100 ? sum : sum % 101
|
|
1267
|
+
check = 0 if check == 100
|
|
1268
|
+
check == d[9] * 10 + d[10]
|
|
1269
|
+
end
|
|
1270
|
+
|
|
1271
|
+
def valid_ro_cnp?(value)
|
|
1272
|
+
d = value.chars.map(&:to_i)
|
|
1273
|
+
return false unless d[0].between?(1, 9) && valid_day_month?(d[5] * 10 + d[6], d[3] * 10 + d[4])
|
|
1274
|
+
w = [2,7,9,1,4,6,3,5,8,2,7,9]
|
|
1275
|
+
check = d.first(12).each_with_index.sum { |n, i| n * w[i] } % 11
|
|
1276
|
+
check = 1 if check == 10
|
|
1277
|
+
check == d[12]
|
|
1278
|
+
end
|
|
1279
|
+
|
|
1280
|
+
def valid_za_id?(value)
|
|
1281
|
+
d = value.chars.map(&:to_i)
|
|
1282
|
+
valid_day_month?(d[4] * 10 + d[5], d[2] * 10 + d[3]) && luhn?(value)
|
|
1283
|
+
end
|
|
1284
|
+
|
|
1285
|
+
def valid_kr_rrn?(value)
|
|
1286
|
+
d = value.chars.map(&:to_i)
|
|
1287
|
+
return false unless d[6].between?(1, 8) && valid_day_month?(d[4] * 10 + d[5], d[2] * 10 + d[3])
|
|
1288
|
+
w = [2,3,4,5,6,7,8,9,2,3,4,5]
|
|
1289
|
+
check = (11 - d.first(12).each_with_index.sum { |n, i| n * w[i] } % 11) % 10
|
|
1290
|
+
check == d[12]
|
|
1291
|
+
end
|
|
1292
|
+
|
|
1293
|
+
def valid_id_nik?(value)
|
|
1294
|
+
return false unless value.match?(/\A\d{16}\z/)
|
|
1295
|
+
d = value.chars.map(&:to_i)
|
|
1296
|
+
province = d[0] * 10 + d[1]
|
|
1297
|
+
day = d[6] * 10 + d[7]
|
|
1298
|
+
day -= 40 if day > 40
|
|
1299
|
+
valid_day_month?(day, d[8] * 10 + d[9]) && province.between?(11, 96)
|
|
1300
|
+
end
|
|
1301
|
+
|
|
1302
|
+
def byte_match(bytes, type, first, last, ascii_offsets: false)
|
|
1303
|
+
value = bytes.byteslice(first...last).dup.force_encoding("UTF-8")
|
|
1304
|
+
start = ascii_offsets ? first : bytes.byteslice(0...first).force_encoding("UTF-8").scrub.length
|
|
1305
|
+
PiiMatch.new(type: type, start: start, finish: start + value.length, value: value)
|
|
1306
|
+
end
|
|
1307
|
+
|
|
1308
|
+
def digit?(b)
|
|
1309
|
+
b && b.between?(48, 57)
|
|
1310
|
+
end
|
|
1311
|
+
|
|
1312
|
+
def valid_ssn?(digits)
|
|
1313
|
+
digits.bytesize == 9 && digits[0, 3].to_i.between?(1, 899) && digits[0, 3] != "666" &&
|
|
1314
|
+
digits[3, 2] != "00" && digits[5, 4] != "0000"
|
|
1315
|
+
end
|
|
1316
|
+
|
|
1317
|
+
def merge_card_matches!(text, matches)
|
|
1318
|
+
cards = card_matches(text)
|
|
1319
|
+
return if cards.empty?
|
|
1320
|
+
|
|
1321
|
+
# Cards from one left-to-right run are disjoint and ordered by start.
|
|
1322
|
+
# A linear any?-scan is quadratic on dense prompts (about 10k cards
|
|
1323
|
+
# times 20k other spans on 1 MiB) and does not return inside the
|
|
1324
|
+
# customer budget. One binary search per span keeps the same overlap
|
|
1325
|
+
# rule: drop a span that overlaps a card unless the span strictly
|
|
1326
|
+
# encloses that card.
|
|
1327
|
+
cards.sort_by!(&:start)
|
|
1328
|
+
matches.reject! do |match|
|
|
1329
|
+
card_overlap_rejects?(cards, match)
|
|
1330
|
+
end
|
|
1331
|
+
matches.concat(cards)
|
|
1332
|
+
end
|
|
1333
|
+
|
|
1334
|
+
def card_overlap_rejects?(cards, match)
|
|
1335
|
+
lo = 0
|
|
1336
|
+
hi = cards.length
|
|
1337
|
+
while lo < hi
|
|
1338
|
+
mid = (lo + hi) / 2
|
|
1339
|
+
if cards[mid].start < match.finish
|
|
1340
|
+
lo = mid + 1
|
|
1341
|
+
else
|
|
1342
|
+
hi = mid
|
|
1343
|
+
end
|
|
1344
|
+
end
|
|
1345
|
+
index = lo - 1
|
|
1346
|
+
while index >= 0 && cards[index].finish > match.start
|
|
1347
|
+
card = cards[index]
|
|
1348
|
+
if match.start < card.finish && match.finish > card.start
|
|
1349
|
+
encloses = match.start <= card.start && match.finish >= card.finish &&
|
|
1350
|
+
(match.finish - match.start) > (card.finish - card.start)
|
|
1351
|
+
return true unless encloses
|
|
1352
|
+
end
|
|
1353
|
+
index -= 1
|
|
1354
|
+
end
|
|
1355
|
+
false
|
|
1356
|
+
end
|
|
1357
|
+
|
|
1358
|
+
# One alternation, not a downcase copy. The 50 KiB fixture misses every
|
|
1359
|
+
# branch. Mixed case is listed for the cues the byte scanner folds.
|
|
1360
|
+
CREDENTIAL_CUE_LITERALS = [
|
|
1361
|
+
"eyJ", "sk-", "pk-", "rk-", "ak-", "AKIA", "akia",
|
|
1362
|
+
"api key", "api_key", "api-key", "API key", "API_KEY", "API-KEY", "Api key",
|
|
1363
|
+
"secret", "Secret", "SECRET", "token", "Token", "TOKEN",
|
|
1364
|
+
"bearer", "Bearer", "BEARER",
|
|
1365
|
+
].freeze
|
|
1366
|
+
|
|
1367
|
+
# Same literals as the alternation, via String#include?. The alternation
|
|
1368
|
+
# walked the whole 50 KiB fixture on every call and missed every branch.
|
|
1369
|
+
def short_credential_cue?(text)
|
|
1370
|
+
CREDENTIAL_CUE_LITERALS.any? { |literal| text.include?(literal) }
|
|
1371
|
+
end
|
|
1372
|
+
|
|
1373
|
+
CARD_SEPARATORS = [" ", "\t", "\n", "\r", "\r\n", "-", ".", "\u00a0", "\u2010", "\u2011", "\u2012", "\u2013", "\u2014", "\u2015", "\u2212", "\u00b7", "\u2027"].freeze
|
|
1374
|
+
CARD_SEPARATOR_BYTES = CARD_SEPARATORS.each_with_object({}) { |sep, lookup|
|
|
1375
|
+
lookup[sep.ord] = true if sep.bytesize == 1
|
|
1376
|
+
}.freeze
|
|
1377
|
+
|
|
1378
|
+
def card_ascii_word?(byte)
|
|
1379
|
+
(byte >= 48 && byte <= 57) || (byte >= 65 && byte <= 90) || (byte >= 97 && byte <= 122) || byte == 95
|
|
1380
|
+
end
|
|
1381
|
+
|
|
1382
|
+
def card_digit_runs(text)
|
|
1383
|
+
runs = []
|
|
1384
|
+
if text.ascii_only?
|
|
1385
|
+
n = text.bytesize
|
|
1386
|
+
i = 0
|
|
1387
|
+
while i < n
|
|
1388
|
+
b = text.getbyte(i)
|
|
1389
|
+
unless b >= 48 && b <= 57
|
|
1390
|
+
i += 1
|
|
1391
|
+
next
|
|
1392
|
+
end
|
|
1393
|
+
j = i + 1
|
|
1394
|
+
j += 1 while j < n && (c = text.getbyte(j)) >= 48 && c <= 57
|
|
1395
|
+
runs << [i, j, text.byteslice(i, j - i)]
|
|
1396
|
+
i = j
|
|
1397
|
+
end
|
|
1398
|
+
else
|
|
1399
|
+
text.scan(/[0-9]+/) do
|
|
1400
|
+
match = Regexp.last_match
|
|
1401
|
+
runs << [match.begin(0), match.end(0), match[0]]
|
|
1402
|
+
end
|
|
1403
|
+
end
|
|
1404
|
+
runs
|
|
1405
|
+
end
|
|
1406
|
+
|
|
1407
|
+
def card_matches(text)
|
|
1408
|
+
# Digit runs are collected with a linear scan. The old prefilter
|
|
1409
|
+
# /[0-9](?:[^0-9]{0,3}[0-9]){12}/ backtracks on dense digits, and
|
|
1410
|
+
# String#scan of the same runs cost seconds per mebibyte. ASCII
|
|
1411
|
+
# offsets match the character offsets scan would have produced.
|
|
1412
|
+
# \d(?:\D{0,3}\d){12} fails in linear time on prose that cannot hold a
|
|
1413
|
+
# PAN (the 50 KiB gate). A hit still uses the linear run scanner.
|
|
1414
|
+
return [] unless text.match?(/\d(?:\D{0,3}\d){12}/)
|
|
1415
|
+
|
|
1416
|
+
runs = card_digit_runs(text)
|
|
1417
|
+
found = []
|
|
1418
|
+
index = 0
|
|
1419
|
+
ascii = text.ascii_only?
|
|
1420
|
+
while index < runs.length
|
|
1421
|
+
start, ending, digits = runs[index]
|
|
1422
|
+
groups = [[start, ending, digits]]
|
|
1423
|
+
last = index
|
|
1424
|
+
while last + 1 < runs.length
|
|
1425
|
+
nxt_start, nxt_end, nxt_digits = runs[last + 1]
|
|
1426
|
+
gap = text[ending...nxt_start]
|
|
1427
|
+
break unless CARD_SEPARATORS.include?(gap)
|
|
1428
|
+
|
|
1429
|
+
groups << [nxt_start, nxt_end, nxt_digits]
|
|
1430
|
+
ending = nxt_end
|
|
1431
|
+
last += 1
|
|
1432
|
+
end
|
|
1433
|
+
if groups.map { |group| group[2].length } == [8, 4, 4, 4, 12]
|
|
1434
|
+
# UUID shape. Do not cut a 16-digit Luhn window out of it.
|
|
1435
|
+
index = last + 1
|
|
1436
|
+
next
|
|
1437
|
+
end
|
|
1438
|
+
# A chain shorter than the shortest PAN cannot contain one.
|
|
1439
|
+
digit_count = 0
|
|
1440
|
+
groups.each { |group| digit_count += group[2].length }
|
|
1441
|
+
if digit_count < 13
|
|
1442
|
+
index = last + 1
|
|
1443
|
+
next
|
|
1444
|
+
end
|
|
1445
|
+
card_chain_segments(text, groups, ascii).each do |seg_start, seg_end|
|
|
1446
|
+
found << PiiMatch.new(type: :ccnum, start: seg_start, finish: seg_end, value: text[seg_start...seg_end])
|
|
1447
|
+
end
|
|
1448
|
+
index = last + 1
|
|
1449
|
+
end
|
|
1450
|
+
found.concat(double_separator_cards(text, ascii)) if text_may_have_double_separator?(text)
|
|
1451
|
+
found
|
|
1452
|
+
end
|
|
1453
|
+
|
|
1454
|
+
def text_may_have_double_separator?(text)
|
|
1455
|
+
return true if text.include?(" ")
|
|
1456
|
+
# ". " is in ordinary sentences. Only a digit before the separator can
|
|
1457
|
+
# be the doubled gap the card rule counts.
|
|
1458
|
+
return true if text.match?(/[0-9][\t\n\r.\-] /)
|
|
1459
|
+
return false if text.ascii_only?
|
|
1460
|
+
|
|
1461
|
+
CARD_SEPARATORS.any? { |sep| sep != "\r\n" && sep != " " && text.include?("#{sep} ") }
|
|
1462
|
+
end
|
|
1463
|
+
|
|
1464
|
+
def double_separator_cards(text, ascii)
|
|
1465
|
+
return double_separator_cards_ascii(text) if ascii
|
|
1466
|
+
|
|
1467
|
+
found = []
|
|
1468
|
+
n = text.length
|
|
1469
|
+
i = 0
|
|
1470
|
+
while i < n
|
|
1471
|
+
unless digit_char?(text[i])
|
|
1472
|
+
i += 1
|
|
1473
|
+
next
|
|
1474
|
+
end
|
|
1475
|
+
start = i
|
|
1476
|
+
digits = +""
|
|
1477
|
+
doubles = 0
|
|
1478
|
+
finish = i
|
|
1479
|
+
j = i
|
|
1480
|
+
while j < n
|
|
1481
|
+
ch = text[j]
|
|
1482
|
+
if digit_char?(ch)
|
|
1483
|
+
digits << ch
|
|
1484
|
+
j += 1
|
|
1485
|
+
finish = j
|
|
1486
|
+
next
|
|
1487
|
+
end
|
|
1488
|
+
unless ch && CARD_SEPARATORS.include?(ch)
|
|
1489
|
+
break
|
|
1490
|
+
end
|
|
1491
|
+
after = j + 1
|
|
1492
|
+
if after < n && digit_char?(text[after])
|
|
1493
|
+
j = after
|
|
1494
|
+
next
|
|
1495
|
+
end
|
|
1496
|
+
if after < n && text[after] == " " && after + 1 < n && digit_char?(text[after + 1])
|
|
1497
|
+
doubles += 1
|
|
1498
|
+
j = after + 1
|
|
1499
|
+
next
|
|
1500
|
+
end
|
|
1501
|
+
break
|
|
1502
|
+
end
|
|
1503
|
+
if doubles == 1 && digits.length.between?(13, 19) && supported_card_network?(digits) && luhn?(digits) &&
|
|
1504
|
+
card_span_bounded?(text, start, finish, ascii)
|
|
1505
|
+
found << PiiMatch.new(type: :ccnum, start: start, finish: finish, value: text[start...finish])
|
|
1506
|
+
end
|
|
1507
|
+
i = j > i ? j : i + 1
|
|
1508
|
+
end
|
|
1509
|
+
found
|
|
1510
|
+
end
|
|
1511
|
+
|
|
1512
|
+
def digit_char?(ch)
|
|
1513
|
+
!ch.nil? && ch.bytesize == 1 && (b = ch.ord) >= 48 && b <= 57
|
|
1514
|
+
end
|
|
1515
|
+
|
|
1516
|
+
# The 50 KiB prose gate includes ". " in almost every sentence. A
|
|
1517
|
+
# per-character Regexp#match? walk blew the 0.050 s median. Byte scans
|
|
1518
|
+
# keep the same span rule: exactly one (separator, space) inside a
|
|
1519
|
+
# 13-19 digit candidate.
|
|
1520
|
+
def double_separator_cards_ascii(text)
|
|
1521
|
+
found = []
|
|
1522
|
+
n = text.bytesize
|
|
1523
|
+
sep = CARD_SEPARATOR_BYTES
|
|
1524
|
+
i = 0
|
|
1525
|
+
while i < n
|
|
1526
|
+
b = text.getbyte(i)
|
|
1527
|
+
unless b >= 48 && b <= 57
|
|
1528
|
+
i += 1
|
|
1529
|
+
next
|
|
1530
|
+
end
|
|
1531
|
+
start = i
|
|
1532
|
+
digit_count = 0
|
|
1533
|
+
doubles = 0
|
|
1534
|
+
finish = i
|
|
1535
|
+
j = i
|
|
1536
|
+
while j < n
|
|
1537
|
+
b = text.getbyte(j)
|
|
1538
|
+
if b >= 48 && b <= 57
|
|
1539
|
+
digit_count += 1
|
|
1540
|
+
j += 1
|
|
1541
|
+
finish = j
|
|
1542
|
+
next
|
|
1543
|
+
end
|
|
1544
|
+
unless sep[b]
|
|
1545
|
+
break
|
|
1546
|
+
end
|
|
1547
|
+
nxt = j + 1 < n ? text.getbyte(j + 1) : nil
|
|
1548
|
+
if nxt && nxt >= 48 && nxt <= 57
|
|
1549
|
+
j += 1
|
|
1550
|
+
next
|
|
1551
|
+
end
|
|
1552
|
+
if nxt == 32 && j + 2 < n && (c = text.getbyte(j + 2)) >= 48 && c <= 57
|
|
1553
|
+
doubles += 1
|
|
1554
|
+
j += 2
|
|
1555
|
+
next
|
|
1556
|
+
end
|
|
1557
|
+
break
|
|
1558
|
+
end
|
|
1559
|
+
if doubles == 1 && digit_count.between?(13, 19)
|
|
1560
|
+
digits = +""
|
|
1561
|
+
k = start
|
|
1562
|
+
while k < finish
|
|
1563
|
+
b = text.getbyte(k)
|
|
1564
|
+
digits << b.chr if b >= 48 && b <= 57
|
|
1565
|
+
k += 1
|
|
1566
|
+
end
|
|
1567
|
+
if supported_card_network?(digits) && luhn?(digits) &&
|
|
1568
|
+
card_span_bounded?(text, start, finish, true)
|
|
1569
|
+
found << PiiMatch.new(type: :ccnum, start: start, finish: finish, value: text.byteslice(start, finish - start))
|
|
1570
|
+
end
|
|
1571
|
+
end
|
|
1572
|
+
i = j > i ? j : i + 1
|
|
1573
|
+
end
|
|
1574
|
+
found
|
|
1575
|
+
end
|
|
1576
|
+
|
|
1577
|
+
def card_span_bounded?(text, start, finish, ascii)
|
|
1578
|
+
if ascii
|
|
1579
|
+
left = start.zero? || !card_ascii_word?(text.getbyte(start - 1))
|
|
1580
|
+
right = finish == text.length || !card_ascii_word?(text.getbyte(finish))
|
|
1581
|
+
return left && right
|
|
1582
|
+
end
|
|
1583
|
+
left = start.zero? ? nil : text[start - 1]
|
|
1584
|
+
right = finish == text.length ? nil : text[finish]
|
|
1585
|
+
(left.nil? || !left.match?(/[\p{L}\p{M}\p{N}_]/u)) &&
|
|
1586
|
+
(right.nil? || !right.match?(/[\p{L}\p{M}\p{N}_]/u))
|
|
1587
|
+
end
|
|
1588
|
+
|
|
1589
|
+
# Canonical grouping, or the whole separator chain when that chain does
|
|
1590
|
+
# not already contain a shorter canonical PAN. Single-digit slices are not.
|
|
1591
|
+
def card_window_eligible?(groups, from, to_inclusive)
|
|
1592
|
+
return true if canonical_window?(groups, from, to_inclusive)
|
|
1593
|
+
return false unless from.zero? && to_inclusive == groups.length - 1
|
|
1594
|
+
|
|
1595
|
+
!chain_has_canonical_pan?(groups)
|
|
1596
|
+
end
|
|
1597
|
+
|
|
1598
|
+
def canonical_window?(groups, from, to_inclusive)
|
|
1599
|
+
lengths = (from..to_inclusive).map { |index| groups[index][2].length }
|
|
1600
|
+
case lengths.length
|
|
1601
|
+
when 1
|
|
1602
|
+
lengths[0].between?(13, 19)
|
|
1603
|
+
when 2
|
|
1604
|
+
lengths == [6, 10]
|
|
1605
|
+
when 3
|
|
1606
|
+
lengths == [4, 6, 5]
|
|
1607
|
+
when 4
|
|
1608
|
+
lengths == [4, 4, 4, 4] || lengths == [4, 4, 4, 3]
|
|
1609
|
+
when 5
|
|
1610
|
+
lengths == [4, 4, 4, 4, 3]
|
|
1611
|
+
else
|
|
1612
|
+
false
|
|
1613
|
+
end
|
|
1614
|
+
end
|
|
1615
|
+
|
|
1616
|
+
def chain_has_canonical_pan?(groups)
|
|
1617
|
+
count = groups.length
|
|
1618
|
+
count.times do |i|
|
|
1619
|
+
digits = +""
|
|
1620
|
+
i.upto(count - 1) do |j|
|
|
1621
|
+
digits << groups[j][2]
|
|
1622
|
+
break if digits.length > 19
|
|
1623
|
+
next unless digits.length >= 13 && canonical_window?(groups, i, j)
|
|
1624
|
+
return true if supported_card_network?(digits) && luhn?(digits)
|
|
1625
|
+
end
|
|
1626
|
+
end
|
|
1627
|
+
false
|
|
1628
|
+
end
|
|
1629
|
+
|
|
1630
|
+
# groups: [start, finish, digits]. Returns [start, finish] spans.
|
|
1631
|
+
def card_chain_segments(text, groups, ascii)
|
|
1632
|
+
count = groups.length
|
|
1633
|
+
suffix = Array.new(count + 1, 0)
|
|
1634
|
+
take_to = Array.new(count, -1)
|
|
1635
|
+
(count - 1).downto(0) do |i|
|
|
1636
|
+
best = suffix[i + 1]
|
|
1637
|
+
best_to = -1
|
|
1638
|
+
digits = +""
|
|
1639
|
+
i.upto(count - 1) do |j|
|
|
1640
|
+
digits << groups[j][2]
|
|
1641
|
+
length = digits.length
|
|
1642
|
+
break if length > 19
|
|
1643
|
+
next unless length >= 13 && card_window_eligible?(groups, i, j) && supported_card_network?(digits) && luhn?(digits)
|
|
1644
|
+
next unless card_span_bounded?(text, groups[i][0], groups[j][1], ascii)
|
|
1645
|
+
|
|
1646
|
+
score = length + suffix[j + 1]
|
|
1647
|
+
if score > best
|
|
1648
|
+
best = score
|
|
1649
|
+
best_to = j + 1
|
|
1650
|
+
end
|
|
1651
|
+
end
|
|
1652
|
+
suffix[i] = best
|
|
1653
|
+
take_to[i] = best_to
|
|
1654
|
+
end
|
|
1655
|
+
segments = []
|
|
1656
|
+
i = 0
|
|
1657
|
+
while i < count
|
|
1658
|
+
to = take_to[i]
|
|
1659
|
+
if to < 0
|
|
1660
|
+
i += 1
|
|
1661
|
+
next
|
|
1662
|
+
end
|
|
1663
|
+
segments << [groups[i][0], groups[to - 1][1]]
|
|
1664
|
+
i = to
|
|
1665
|
+
end
|
|
1666
|
+
segments
|
|
1667
|
+
end
|
|
1668
|
+
|
|
1669
|
+
def supported_card_network?(digits)
|
|
1670
|
+
length = digits.length
|
|
1671
|
+
return false if length < 2
|
|
1672
|
+
|
|
1673
|
+
first2 = digits[0, 2].to_i
|
|
1674
|
+
first3 = digits[0, 3].to_i
|
|
1675
|
+
first4 = digits[0, 4].to_i
|
|
1676
|
+
first6 = digits[0, 6].to_i
|
|
1677
|
+
return true if digits.start_with?("4") && [13, 16, 19].include?(length)
|
|
1678
|
+
return true if length == 16 && ((51..55).cover?(first2) || (2221..2720).cover?(first4))
|
|
1679
|
+
return true if length == 15 && ["34", "37"].include?(digits[0, 2])
|
|
1680
|
+
return true if length == 14 && ((300..305).cover?(first3) || [36, 38, 39].include?(first2))
|
|
1681
|
+
return true if [16, 19].include?(length) && (digits.start_with?("6011") || digits.start_with?("65"))
|
|
1682
|
+
return true if length == 16 && ((644..649).cover?(first3) || (622126..622925).cover?(first6))
|
|
1683
|
+
return true if (16..19).cover?(length) && (3528..3589).cover?(first4)
|
|
1684
|
+
|
|
1685
|
+
(16..19).cover?(length) && digits.start_with?("62")
|
|
1686
|
+
end
|
|
1687
|
+
|
|
1688
|
+
def luhn?(digits)
|
|
1689
|
+
sum = 0
|
|
1690
|
+
parity = digits.length % 2
|
|
1691
|
+
digits.each_char.with_index do |ch, index|
|
|
1692
|
+
n = ch.ord - 48
|
|
1693
|
+
if index % 2 == parity
|
|
1694
|
+
n *= 2
|
|
1695
|
+
n -= 9 if n > 9
|
|
1696
|
+
end
|
|
1697
|
+
sum += n
|
|
1698
|
+
end
|
|
1699
|
+
(sum % 10).zero?
|
|
1700
|
+
end
|
|
1701
|
+
|
|
1702
|
+
def scan_iban(bytes, digit_positions, out, ascii_offsets: false)
|
|
1703
|
+
overlong = false
|
|
1704
|
+
index = 0
|
|
1705
|
+
while index < digit_positions.length
|
|
1706
|
+
control_end = digit_positions[index]
|
|
1707
|
+
i = control_end - 3
|
|
1708
|
+
unless i >= 0 && letter?(bytes.getbyte(i)) && letter?(bytes.getbyte(i + 1)) &&
|
|
1709
|
+
digit?(bytes.getbyte(i + 2)) && digit?(bytes.getbyte(i + 3))
|
|
1710
|
+
index += 1
|
|
1711
|
+
next
|
|
1712
|
+
end
|
|
1713
|
+
finish = i
|
|
1714
|
+
compact = +""
|
|
1715
|
+
separator_run = 0
|
|
1716
|
+
while finish < bytes.bytesize && finish - i < 68
|
|
1717
|
+
b = bytes.getbyte(finish)
|
|
1718
|
+
if letter?(b) || digit?(b)
|
|
1719
|
+
compact << b
|
|
1720
|
+
separator_run = 0
|
|
1721
|
+
elsif b == 32 && !compact.empty?
|
|
1722
|
+
separator_run += 1
|
|
1723
|
+
break if separator_run > 1
|
|
1724
|
+
else
|
|
1725
|
+
break
|
|
1726
|
+
end
|
|
1727
|
+
finish += 1
|
|
1728
|
+
end
|
|
1729
|
+
if compact.bytesize.between?(15, 34) && iban_mod97(compact) == 1
|
|
1730
|
+
last = finish
|
|
1731
|
+
last -= 1 while last > i && bytes.getbyte(last - 1) == 32
|
|
1732
|
+
out << byte_match(bytes, :iban, i, last, ascii_offsets: ascii_offsets)
|
|
1733
|
+
elsif compact.bytesize > 34
|
|
1734
|
+
overlong = true
|
|
1735
|
+
end
|
|
1736
|
+
index += 1
|
|
1737
|
+
index += 1 while index < digit_positions.length && digit_positions[index] < finish
|
|
1738
|
+
end
|
|
1739
|
+
overlong
|
|
1740
|
+
end
|
|
1741
|
+
|
|
1742
|
+
def letter?(b)
|
|
1743
|
+
b && (b.between?(65, 90) || b.between?(97, 122))
|
|
1744
|
+
end
|
|
1745
|
+
|
|
1746
|
+
def iban_mod97(compact)
|
|
1747
|
+
moved = compact.byteslice(4..) + compact.byteslice(0, 4)
|
|
1748
|
+
remainder = 0
|
|
1749
|
+
moved.each_byte do |b|
|
|
1750
|
+
if digit?(b)
|
|
1751
|
+
remainder = (remainder * 10 + b - 48) % 97
|
|
1752
|
+
else
|
|
1753
|
+
value = b >= 97 ? b - 87 : b - 55
|
|
1754
|
+
remainder = (remainder * 100 + value) % 97
|
|
1755
|
+
end
|
|
1756
|
+
end
|
|
1757
|
+
remainder
|
|
1758
|
+
end
|
|
1759
|
+
|
|
1760
|
+
def scan_credentials(bytes, lower, out, ascii_offsets: false)
|
|
1761
|
+
overlong = false
|
|
1762
|
+
["sk-live-", "sk-test-", "sk-prod-", "sk-proj-", "pk-live-", "rk-live-", "ak-live-"].each do |prefix|
|
|
1763
|
+
offset = 0
|
|
1764
|
+
while (at = bytes.index(prefix, offset))
|
|
1765
|
+
finish = at + prefix.bytesize
|
|
1766
|
+
finish += 1 while finish < bytes.bytesize && credential_byte?(bytes.getbyte(finish))
|
|
1767
|
+
overlong ||= finish - at > MAX_SPAN_BYTES
|
|
1768
|
+
out << byte_match(bytes, :api_key, at, finish, ascii_offsets: ascii_offsets) if finish - at > prefix.bytesize
|
|
1769
|
+
offset = [finish, at + 1].max
|
|
1770
|
+
end
|
|
1771
|
+
end
|
|
1772
|
+
overlong ||= scan_labeled_token(bytes, lower, out, ascii_offsets: ascii_offsets)
|
|
1773
|
+
overlong ||= scan_jwt(bytes, out, ascii_offsets: ascii_offsets)
|
|
1774
|
+
overlong
|
|
1775
|
+
end
|
|
1776
|
+
|
|
1777
|
+
def credential_byte?(b)
|
|
1778
|
+
b && (letter?(b) || digit?(b) || b == 45 || b == 95)
|
|
1779
|
+
end
|
|
1780
|
+
|
|
1781
|
+
def scan_labeled_token(bytes, lower, out, ascii_offsets: false)
|
|
1782
|
+
overlong = false
|
|
1783
|
+
["api key", "api_key", "secret", "token", "bearer"].each do |cue|
|
|
1784
|
+
offset = 0
|
|
1785
|
+
while (at = lower.index(cue.b, offset))
|
|
1786
|
+
p = at + cue.bytesize
|
|
1787
|
+
p += 1 while p < bytes.bytesize && [32, 58, 61, 34, 39].include?(bytes.getbyte(p))
|
|
1788
|
+
finish = p
|
|
1789
|
+
finish += 1 while finish < bytes.bytesize && credential_byte?(bytes.getbyte(finish))
|
|
1790
|
+
overlong ||= finish - p > MAX_SPAN_BYTES
|
|
1791
|
+
out << byte_match(bytes, :api_key, at, finish, ascii_offsets: ascii_offsets) if finish - p >= 16
|
|
1792
|
+
offset = [finish, at + 1].max
|
|
1793
|
+
end
|
|
1794
|
+
end
|
|
1795
|
+
overlong
|
|
1796
|
+
end
|
|
1797
|
+
|
|
1798
|
+
def scan_jwt(bytes, out, ascii_offsets: false)
|
|
1799
|
+
overlong = false
|
|
1800
|
+
offset = 0
|
|
1801
|
+
while (at = bytes.index("eyJ", offset))
|
|
1802
|
+
finish = at
|
|
1803
|
+
dots = 0
|
|
1804
|
+
while finish < bytes.bytesize && (credential_byte?(bytes.getbyte(finish)) || bytes.getbyte(finish) == 46)
|
|
1805
|
+
dots += 1 if bytes.getbyte(finish) == 46
|
|
1806
|
+
finish += 1
|
|
1807
|
+
end
|
|
1808
|
+
overlong ||= finish - at > MAX_SPAN_BYTES
|
|
1809
|
+
out << byte_match(bytes, :auth_jwt, at, finish, ascii_offsets: ascii_offsets) if dots == 2 && finish - at > 20
|
|
1810
|
+
offset = [finish, at + 1].max
|
|
1811
|
+
end
|
|
1812
|
+
overlong
|
|
1813
|
+
end
|
|
1814
|
+
|
|
1815
|
+
def scan_labeled_ids(bytes, lower, out, ascii_offsets: false)
|
|
1816
|
+
overlong = false
|
|
1817
|
+
NATIONAL_ID_CUES.each do |cue|
|
|
1818
|
+
offset = 0
|
|
1819
|
+
while (at = lower.index(cue.b, offset))
|
|
1820
|
+
if at.positive? && id_word_byte?(bytes.getbyte(at - 1))
|
|
1821
|
+
offset = at + cue.bytesize
|
|
1822
|
+
next
|
|
1823
|
+
end
|
|
1824
|
+
p = at + cue.bytesize
|
|
1825
|
+
p += 1 while p < bytes.bytesize && !digit?(bytes.getbyte(p)) && p - at < 64
|
|
1826
|
+
if p - at >= 64 && !digit?(bytes.getbyte(p))
|
|
1827
|
+
overlong = true
|
|
1828
|
+
offset = p
|
|
1829
|
+
next
|
|
1830
|
+
end
|
|
1831
|
+
finish = p
|
|
1832
|
+
while finish < bytes.bytesize && finish - at < 64
|
|
1833
|
+
if digit?(bytes.getbyte(finish))
|
|
1834
|
+
finish += 1
|
|
1835
|
+
elsif [32, 45, 46].include?(bytes.getbyte(finish)) &&
|
|
1836
|
+
digit?(bytes.getbyte(finish + 1))
|
|
1837
|
+
finish += 1
|
|
1838
|
+
else
|
|
1839
|
+
break
|
|
1840
|
+
end
|
|
1841
|
+
end
|
|
1842
|
+
next_byte = bytes.getbyte(finish)
|
|
1843
|
+
candidate_overlong = finish - at >= 64 && finish < bytes.bytesize &&
|
|
1844
|
+
(digit?(next_byte) ||
|
|
1845
|
+
([32, 45, 46].include?(next_byte) && digit?(bytes.getbyte(finish + 1))))
|
|
1846
|
+
gap_end = finish
|
|
1847
|
+
gap_end += 1 while gap_end < bytes.bytesize && gap_end - finish < 64 &&
|
|
1848
|
+
[32, 45, 46].include?(bytes.getbyte(gap_end))
|
|
1849
|
+
overlong ||= candidate_overlong ||
|
|
1850
|
+
(gap_end > finish + 1 && digit?(bytes.getbyte(gap_end)))
|
|
1851
|
+
compact = bytes.byteslice(p...finish).delete("- .")
|
|
1852
|
+
out << byte_match(bytes, :nid_generic_labelled, at, finish, ascii_offsets: ascii_offsets) if !candidate_overlong && compact.bytesize.between?(5, 18)
|
|
1853
|
+
offset = [finish, at + 1].max
|
|
1854
|
+
end
|
|
1855
|
+
end
|
|
1856
|
+
overlong
|
|
1857
|
+
end
|
|
1858
|
+
|
|
1859
|
+
def scan_marked_dob(bytes, lower, out, ascii_offsets: false)
|
|
1860
|
+
overlong = false
|
|
1861
|
+
["dob", "date of birth", "birth date", "born"].each do |cue|
|
|
1862
|
+
offset = 0
|
|
1863
|
+
while (at = lower.index(cue.b, offset))
|
|
1864
|
+
p = at + cue.bytesize
|
|
1865
|
+
p += 1 while p < bytes.bytesize && [32, 58, 35, 46, 44, 45].include?(bytes.getbyte(p)) && p - at < 64
|
|
1866
|
+
finish = p
|
|
1867
|
+
while finish < bytes.bytesize && finish - p < 32
|
|
1868
|
+
if digit?(bytes.getbyte(finish))
|
|
1869
|
+
finish += 1
|
|
1870
|
+
elsif [47, 45, 46, 32].include?(bytes.getbyte(finish)) && digit?(bytes.getbyte(finish + 1))
|
|
1871
|
+
finish += 1
|
|
1872
|
+
else
|
|
1873
|
+
break
|
|
1874
|
+
end
|
|
1875
|
+
end
|
|
1876
|
+
candidate_overlong = finish - p >= 32 && finish < bytes.bytesize &&
|
|
1877
|
+
(digit?(bytes.getbyte(finish)) ||
|
|
1878
|
+
([47, 45, 46, 32].include?(bytes.getbyte(finish)) && digit?(bytes.getbyte(finish + 1))))
|
|
1879
|
+
overlong ||= candidate_overlong
|
|
1880
|
+
finish -= 1 while finish > p && [47, 45, 46, 32].include?(bytes.getbyte(finish - 1))
|
|
1881
|
+
digits = bytes.byteslice(p...finish).delete("- /.")
|
|
1882
|
+
date_text = bytes.byteslice(p...finish)
|
|
1883
|
+
out << byte_match(bytes, :date_of_birth, at, finish, ascii_offsets: ascii_offsets) if !candidate_overlong && valid_dob_date?(date_text, digits)
|
|
1884
|
+
offset = [finish, at + 1].max
|
|
1885
|
+
end
|
|
1886
|
+
end
|
|
1887
|
+
overlong
|
|
1888
|
+
end
|
|
1889
|
+
|
|
1890
|
+
def valid_dob_date?(date_text, digits)
|
|
1891
|
+
parts = date_text.split(/[\/\.\-\s]+/).reject(&:empty?)
|
|
1892
|
+
if parts.length == 3 && parts.all? { |part| part.match?(/\A\d{1,4}\z/) }
|
|
1893
|
+
a, b, c = parts.map(&:to_i)
|
|
1894
|
+
[[a, b, c], [a, c, b], [b, a, c], [b, c, a], [c, a, b], [c, b, a]].any? do |year, month, day|
|
|
1895
|
+
year.between?(1, 9999) && Date.valid_date?(year, month, day)
|
|
1896
|
+
end
|
|
1897
|
+
elsif digits.bytesize == 8 && digits.match?(/\A\d{8}\z/)
|
|
1898
|
+
year, month, day = digits[0, 4].to_i, digits[4, 2].to_i, digits[6, 2].to_i
|
|
1899
|
+
Date.valid_date?(year, month, day)
|
|
1900
|
+
elsif digits.bytesize == 6 && digits.match?(/\A\d{6}\z/)
|
|
1901
|
+
day, month, year = digits[0, 2].to_i, digits[2, 2].to_i, digits[4, 2].to_i
|
|
1902
|
+
[1900 + year, 2000 + year].any? { |full_year| Date.valid_date?(full_year, month, day) }
|
|
1903
|
+
else
|
|
1904
|
+
false
|
|
1905
|
+
end
|
|
1906
|
+
end
|
|
1907
|
+
|
|
1908
|
+
def scan_mrz(bytes, out, ascii_offsets: false)
|
|
1909
|
+
overlong = false
|
|
1910
|
+
line_start = 0
|
|
1911
|
+
while line_start < bytes.bytesize
|
|
1912
|
+
line_end = bytes.index("\n", line_start) || bytes.bytesize
|
|
1913
|
+
line = bytes.byteslice(line_start...line_end).delete_suffix("\r")
|
|
1914
|
+
if line.include?("<<") && line.bytesize > MRZ_MAX_BYTES
|
|
1915
|
+
overlong = true
|
|
1916
|
+
elsif line.bytesize.between?(30, MRZ_MAX_BYTES) && line.include?("<<") && line.each_byte.all? { |b| b == 60 || letter?(b) || digit?(b) }
|
|
1917
|
+
out << byte_match(bytes, :nid_mrz, line_start, line_start + line.bytesize, ascii_offsets: ascii_offsets)
|
|
1918
|
+
end
|
|
1919
|
+
line_start = line_end + 1
|
|
1920
|
+
end
|
|
1921
|
+
overlong
|
|
1922
|
+
end
|
|
1923
|
+
end
|
|
1924
|
+
end
|
|
1925
|
+
end
|