cloakapi 1.4.1-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +160 -0
  3. data/LICENSE +202 -0
  4. data/NOTICE +18 -0
  5. data/README.md +407 -0
  6. data/THIRD-PARTY-NOTICES.md +22 -0
  7. data/TRADEMARKS.md +19 -0
  8. data/examples/fake_gateway.rb +78 -0
  9. data/examples/hello.rb +64 -0
  10. data/lib/cloakapi/_lib/linux-x86_64/libcloak_engine.so +0 -0
  11. data/lib/cloakapi/client.rb +1272 -0
  12. data/lib/cloakapi/degraded.rb +77 -0
  13. data/lib/cloakapi/disclosure.rb +252 -0
  14. data/lib/cloakapi/engine.rb +702 -0
  15. data/lib/cloakapi/errors.rb +65 -0
  16. data/lib/cloakapi/pii/bundled_rules.rb +670 -0
  17. data/lib/cloakapi/pii/card_spans.rb +300 -0
  18. data/lib/cloakapi/pii/detector.rb +1925 -0
  19. data/lib/cloakapi/pii/patterns.json +2238 -0
  20. data/lib/cloakapi/pii/pii_match.rb +7 -0
  21. data/lib/cloakapi/pii/pii_type.rb +7 -0
  22. data/lib/cloakapi/pii/ruleset.rb +65 -0
  23. data/lib/cloakapi/pii/tokenizer.rb +411 -0
  24. data/lib/cloakapi/posture.rb +82 -0
  25. data/lib/cloakapi/receipts.rb +406 -0
  26. data/lib/cloakapi/resources/chat.rb +82 -0
  27. data/lib/cloakapi/resources/messages.rb +90 -0
  28. data/lib/cloakapi/resources/models.rb +25 -0
  29. data/lib/cloakapi/resources/receipts.rb +162 -0
  30. data/lib/cloakapi/resources/reconstruct.rb +120 -0
  31. data/lib/cloakapi/resources/request_strings.rb +485 -0
  32. data/lib/cloakapi/resources/reviewable.rb +163 -0
  33. data/lib/cloakapi/resources/tokens.rb +28 -0
  34. data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
  35. data/lib/cloakapi/routing.rb +500 -0
  36. data/lib/cloakapi/streaming.rb +196 -0
  37. data/lib/cloakapi/surrogate_mode.rb +233 -0
  38. data/lib/cloakapi/version.rb +15 -0
  39. data/lib/cloakapi/warnings.rb +137 -0
  40. data/lib/cloakapi.rb +37 -0
  41. data/licenses/Apache-2.0.txt +202 -0
  42. data/licenses/CC-BY-4.0.txt +104 -0
  43. data/licenses/MIT-Faker.txt +21 -0
  44. data/licenses/MIT.txt +21 -0
  45. metadata +150 -0
@@ -0,0 +1,1925 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "ruleset"
4
+ require "date"
5
+
6
+ module CloakAPI
7
+ module Pii
8
+ class Detector
9
+ WINDOW_BYTES = 65_536
10
+ MAX_WINDOWS_PER_PASS = 32
11
+ REGEX_STEP_BUDGET = 4_096
12
+ EMERGENCY_WALL_SECONDS = 3.75
13
+ SHORT_INPUT_HANG_SECONDS = 2.0
14
+ REGEX_TIMEOUT_SECONDS = 0.25
15
+ MAX_SPAN_BYTES = 65_536
16
+ ADDRESS_CANONICAL_SCAN_BYTES = 4_096
17
+ ADDRESS_WINDOW_SCAN_BYTES = 1_048_576
18
+ MAX_ADDRESS_RESERVE_BYTES = 8_388_608
19
+ LINEAR_RESERVE_SECONDS = 3.0
20
+ LINEAR_RESERVE_CHUNK_BYTES = 262_144
21
+ LINEAR_RESERVE_OVERLAP_BYTES = 65_536
22
+ SHORT_COMPAT_BYTES = 65_536
23
+ ADDRESS_ANCHOR_BUDGET = 256
24
+ ADDRESS_STREET_SUFFIXES = %w[
25
+ straße strasse straat street gata gatan gaten gate gade veien vegen vägen vei veg vej weg
26
+ avenue ave avenida rue via road rd lane ln gasse allee platz plein torg torget plass square
27
+ boulevard blvd crescent gardens terrace parade court ct drive dr place pl wharf mews quay way
28
+ st sq carrer calle rua strada ulica intie tie torvet gränd gracht väg allée
29
+ ].freeze
30
+ ADDRESS_STREET_SUFFIXES_BY_END = ADDRESS_STREET_SUFFIXES.group_by { |suffix| suffix[-2, 2] }
31
+ .transform_values(&:freeze).freeze
32
+ MRZ_MAX_BYTES = 512
33
+ NATIONAL_ID_CUES = [
34
+ "national id", "national identification", "national identification number", "national number",
35
+ "identity number", "id number", "id no", "id no.", "nid", "personal id", "personal number",
36
+ "id-nummer", "id-nr", "nasjonal id", "ausweisnummer", "personalausweis", "nationale id",
37
+ "numéro national", "numéro d'identité", "carte d'identité", "pièce d'identité",
38
+ "número de identidad", "identificación nacional", "cédula", "numero di identità",
39
+ "documento d'identità", "carta d'identità", "identiteitsnummer", "nationaal id",
40
+ "número de identidade", "cartão de identidade", "identificação nacional", "social security",
41
+ "nino", "nric", "personnummer", "fødselsnummer",
42
+ "fodselsnummer", "kennitala", "cpr", "tax id", "driver licence", "driver license",
43
+ "resident card", "citizen card"
44
+ ].uniq.freeze
45
+ API_KEY_CUES = %w[
46
+ api key secret token bearer sk-live- sk-test- sk-prod- sk-proj- pk-live- rk-live- ak-live-
47
+ ].freeze
48
+ API_KEY_CUE_RE = Regexp.new(API_KEY_CUES.map { |cue| Regexp.escape(cue) }.join("|"), Regexp::IGNORECASE)
49
+ NATIONAL_ID_CUE_RE = Regexp.new(NATIONAL_ID_CUES.map { |cue| Regexp.escape(cue) }.join("|"), Regexp::IGNORECASE)
50
+ NATIONAL_ID_CHECKSUM_CUE_RE = /(?:bsn|sin|social[ ]+insurance[ ]+number|aadhaar|aadhar|uid|my[ ]+number)/i
51
+ NATIONAL_ID_SHAPE_RE = /(?:\b\d(?:[ .\/-]?\d){7,16}[Xx]?\b|\b[A-Z]{4}\d{6}[HM][A-Z]{5}[A-Z0-9]\d\b)/
52
+ NUMERIC_FALLBACK_RE = /\d(?:[ \t().+\-]?\d){6,18}/
53
+ IBAN_SHAPE_RE = /\b[A-Za-z]{2}\d{2}/
54
+ ALL_DIGITS_RE = /\A[0-9]{65,}\z/
55
+ DOB_CUES = %w[dob date of birth birth date born geboren nacido nacida nascita née né natød natø].freeze
56
+ DOB_CUE_RE = Regexp.new(DOB_CUES.map { |cue| Regexp.escape(cue) }.join("|"), Regexp::IGNORECASE)
57
+ OPTIONAL_CUE_RE = Regexp.new((API_KEY_CUES + NATIONAL_ID_CUES + DOB_CUES).uniq
58
+ .map { |cue| Regexp.escape(cue) }.join("|"), Regexp::IGNORECASE)
59
+ LINEAR_WARNING = "Detection was partial because some pattern checks were skipped. The fallback scans " \
60
+ "e-mail, phone, card, IBAN, SSN, labelled ID, marked DOB, credentials and MRZ; " \
61
+ "address and other regex-only matches may be missed."
62
+
63
+ ADDRESS_RE = Ruleset.regexp("ADDRESS")
64
+ PATTERNS = {
65
+ email: /\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b/,
66
+ phone: /\b(\+?1[\s\-.]?)?\(?\d{3}\)?[\s\-.]?\d{3}[\s\-.]?\d{4}\b/,
67
+ ssn: /\b[0-8]\d{2}[-\s]?\d{2}[-\s]?\d{4}\b/,
68
+ iban: /\b[A-Z]{2}[0-9]{2}(?: ?[A-Z0-9]{1,4}){2,7}\b/,
69
+ ip: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/,
70
+ ccnum: /\b(?:4[0-9]{3} ?[0-9]{4} ?[0-9]{4} ?[0-9]{4}|4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{2} ?[0-9]{4} ?[0-9]{4} ?[0-9]{4}|5[1-5][0-9]{14}|3[47][0-9]{2} ?[0-9]{6} ?[0-9]{5}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11})\b/,
71
+ api_key: Ruleset.regexp("API_KEY"),
72
+ auth_jwt: Ruleset.regexp("AUTH_JWT"),
73
+ nid_generic_labelled: Ruleset.regexp("NID_GENERIC_LABELLED"),
74
+ date_of_birth: Ruleset.regexp("DATE_OF_BIRTH"),
75
+ address: ADDRESS_RE,
76
+ }.freeze
77
+ LEGACY_SHORT_PATTERNS = PATTERNS.slice(:email, :phone, :ssn, :iban, :ip, :ccnum, :address).freeze
78
+ BOUNDED_PATTERNS = PATTERNS.transform_values do |regexp|
79
+ Regexp.new(regexp.source, regexp.options, timeout: REGEX_TIMEOUT_SECONDS)
80
+ end.freeze
81
+
82
+ Detection = Data.define(:matches, :degraded, :coverage, :warning)
83
+ ReserveDeadlineReached = Class.new(StandardError)
84
+
85
+ # The tokenize 2s wall clock also covers detect. A mebibyte of
86
+ # email+card finished detect in about 3s with tokenize_budget_hit? still
87
+ # false, because the emergency cap is 3.75s and nothing recorded 2s.
88
+ def self.reset_wall_budget_hit!
89
+ @wall_budget_hit = false
90
+ end
91
+
92
+ def self.note_wall_budget_hit!
93
+ @wall_budget_hit = true
94
+ end
95
+
96
+ def self.wall_budget_hit?
97
+ @wall_budget_hit == true
98
+ end
99
+
100
+ # Compatibility API: callers which only need spans still receive an Array.
101
+ def detect(text)
102
+ self.class.reset_wall_budget_hit!
103
+ if text.is_a?(String) && text.bytesize <= SHORT_COMPAT_BYTES && text.valid_encoding? &&
104
+ (text.encoding == Encoding::UTF_8 || text.ascii_only?)
105
+ legacy_short_matches(text)
106
+ else
107
+ detect_with_status(text).matches
108
+ end
109
+ end
110
+
111
+ def legacy_short_matches(text)
112
+ matches = []
113
+ LEGACY_SHORT_PATTERNS.each do |type, regexp|
114
+ next if type == :ccnum
115
+
116
+ text.scan(regexp) do
117
+ match = Regexp.last_match
118
+ matches << PiiMatch.new(type: type, start: match.begin(0), finish: match.end(0), value: match[0])
119
+ end
120
+ end
121
+ # Texts at or under 64 KiB never reach the regex credential pass.
122
+ # The byte scanner is the same reserve used above that line, and it
123
+ # runs only when a credential cue is present so the 50 KiB gate does
124
+ # not pay for it on ordinary prose.
125
+ if short_credential_cue?(text)
126
+ bytes = text.b
127
+ scan_credentials(bytes, bytes.downcase, matches, ascii_offsets: text.ascii_only?)
128
+ end
129
+ merge_card_matches!(text, matches)
130
+ matches.uniq { |m| [m.type, m.start, m.finish] }.sort_by(&:start)
131
+ end
132
+
133
+ # Scans bounded byte windows. A match belongs to the window whose core
134
+ # contains its starting character offset; overlap is context, not ownership.
135
+ # Ruby regex timeouts are per Regexp instance, so no process-global timeout
136
+ # is mutated while concurrent SDK calls are running.
137
+ def detect_with_status(text, step_budget: REGEX_STEP_BUDGET)
138
+ self.class.reset_wall_budget_hit!
139
+ unless text.is_a?(String)
140
+ return Detection.new(
141
+ matches: [], degraded: true, coverage: "rules_only",
142
+ warning: "Detection was partial because input was not a String."
143
+ )
144
+ end
145
+
146
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
147
+ step_budget = [[Integer(step_budget), 0].max, REGEX_STEP_BUDGET].min
148
+ matches = []
149
+ needs_reserve = false
150
+ address_reserve_needed = false
151
+ scanned_end_byte = 0
152
+ reserve_start_byte = nil
153
+ reserve_unscanned_bytes = nil
154
+ ascii_input = text.ascii_only?
155
+ scannable_encoding = text.valid_encoding? && (text.encoding == Encoding::UTF_8 || ascii_input)
156
+ degraded = !scannable_encoding
157
+ if scannable_encoding && text.bytesize > SHORT_COMPAT_BYTES &&
158
+ text.bytesize <= ADDRESS_WINDOW_SCAN_BYTES
159
+ return scan_single_large_window(text, started)
160
+ end
161
+ if scannable_encoding
162
+ base_chars = 0
163
+ windows_processed = 0
164
+ each_window(text) do |window, base_byte, core_end_byte|
165
+ windows_processed += 1
166
+ scanned_end_byte = core_end_byte
167
+ BOUNDED_PATTERNS.each do |type, regexp|
168
+ if type == :address &&
169
+ ((text.bytesize > ADDRESS_CANONICAL_SCAN_BYTES &&
170
+ text.bytesize <= SHORT_COMPAT_BYTES) || text.bytesize > ADDRESS_WINDOW_SCAN_BYTES)
171
+ degraded = true
172
+ needs_reserve = true
173
+ address_reserve_needed = text.bytesize <= MAX_ADDRESS_RESERVE_BYTES
174
+ next
175
+ end
176
+ possible = pattern_possible?(type, window)
177
+ unless possible || (text.bytesize <= ADDRESS_CANONICAL_SCAN_BYTES &&
178
+ %i[nid_generic_labelled date_of_birth].include?(type))
179
+ # A cue is only a performance hint. Unless it is proven complete
180
+ # for the canonical expression, skipping its scan must be visible.
181
+ degraded = true if text.bytesize > ADDRESS_CANONICAL_SCAN_BYTES
182
+ next
183
+ end
184
+ if emergency_expired?(text, started)
185
+ degraded = true
186
+ needs_reserve = true
187
+ reserve_start_byte ||= base_byte
188
+ break
189
+ end
190
+
191
+ begin
192
+ cursor = 0
193
+ steps = 0
194
+ while (match = regexp.match(window, cursor))
195
+ if emergency_expired?(text, started) || steps >= step_budget
196
+ degraded = true
197
+ needs_reserve = true
198
+ reserve_start_byte ||= base_byte
199
+ break
200
+ end
201
+ steps += 1
202
+ start_byte = ascii_input ? match.begin(0) : match.pre_match.bytesize
203
+ finish_byte = start_byte + match[0].bytesize
204
+ global_byte = base_byte + start_byte
205
+ global_finish_byte = base_byte + finish_byte
206
+ clipped_left = start_byte.zero? && base_byte.positive? &&
207
+ match_continues?(type, text.getbyte(base_byte - 1))
208
+ clipped_right = finish_byte == window.bytesize && global_finish_byte < text.bytesize &&
209
+ match_continues?(type, text.getbyte(global_finish_byte))
210
+ if clipped_left || clipped_right
211
+ degraded = true
212
+ needs_reserve = true
213
+ reserve_start_byte ||= base_byte
214
+ cursor = match.end(0)
215
+ next
216
+ end
217
+ if owns_match_start?(global_byte, core_end_byte)
218
+ if span_too_long?(finish_byte - start_byte)
219
+ degraded = true
220
+ needs_reserve = true
221
+ reserve_start_byte ||= base_byte
222
+ end
223
+ start_char = base_chars + match.begin(0)
224
+ matches << PiiMatch.new(type: type, start: start_char,
225
+ finish: start_char + match[0].length, value: match[0])
226
+ end
227
+ cursor = match.end(0)
228
+ cursor += 1 if match.begin(0) == match.end(0)
229
+ end
230
+ if type == :nid_generic_labelled
231
+ # The canonical ruleset deliberately has a narrower cue set
232
+ # than this SDK's labelled-ID reserve (for example, it does
233
+ # not include the Norwegian "fødselsnummer" cue). When a
234
+ # cue is present but the canonical regex has no match, use
235
+ # the same bounded, label-anchored fallback used by the
236
+ # linear reserve on this window instead of silently skipping
237
+ # the cue in the prefix before a large-input reserve starts.
238
+ local_matches = []
239
+ scan_labeled_ids(window.b, window.b.downcase, local_matches,
240
+ ascii_offsets: window.ascii_only?)
241
+ local_matches.each do |local_match|
242
+ local_start_byte = window.byteslice(0...local_match.start).bytesize
243
+ global_start_byte = base_byte + local_start_byte
244
+ next unless owns_match_start?(global_start_byte, core_end_byte)
245
+
246
+ start_char = base_chars + local_match.start
247
+ matches << PiiMatch.new(type: local_match.type, start: start_char,
248
+ finish: start_char + local_match.value.length,
249
+ value: local_match.value)
250
+ end
251
+ end
252
+ rescue Regexp::TimeoutError
253
+ degraded = true
254
+ needs_reserve = true
255
+ reserve_start_byte ||= base_byte
256
+ break
257
+ end
258
+ end
259
+ base_chars += text.byteslice(base_byte...core_end_byte).length
260
+ if text.bytesize > MAX_ADDRESS_RESERVE_BYTES &&
261
+ windows_processed >= MAX_WINDOWS_PER_PASS && core_end_byte < text.bytesize
262
+ degraded = true
263
+ needs_reserve = true
264
+ break
265
+ end
266
+ break if emergency_expired?(text, started)
267
+ end
268
+ end
269
+
270
+ if scannable_encoding && (needs_reserve || degraded) && text.bytesize > SHORT_COMPAT_BYTES &&
271
+ text.bytesize <= MAX_ADDRESS_RESERVE_BYTES
272
+ degraded = true
273
+ linear_matches(text, matches)
274
+ elsif needs_reserve && text.bytesize > SHORT_COMPAT_BYTES
275
+ degraded = true
276
+ reserve_unscanned_bytes = scan_linear_reserve_chunks(
277
+ text, reserve_start_byte || scanned_end_byte, matches, deadline: started + EMERGENCY_WALL_SECONDS
278
+ )
279
+ elsif needs_reserve || text.bytesize <= SHORT_COMPAT_BYTES
280
+ linear_degraded = linear_matches(text, matches)
281
+ degraded ||= linear_degraded
282
+ end
283
+ address_degraded = false
284
+ if scannable_encoding && (text.bytesize <= SHORT_COMPAT_BYTES || address_reserve_needed ||
285
+ (text.bytesize <= MAX_ADDRESS_RESERVE_BYTES && needs_reserve &&
286
+ !emergency_expired?(text, started)))
287
+ address_degraded = scan_address_reserve(text, matches)
288
+ end
289
+ degraded ||= address_degraded
290
+ # A second full-text card walk on multi-megabyte input blows the R12
291
+ # six-second deadline. Windows above 1 MiB keep the regex card class.
292
+ merge_card_matches!(text, matches) if text.bytesize <= ADDRESS_WINDOW_SCAN_BYTES
293
+ matches = matches.uniq { |m| [m.type, m.start, m.finish] }.sort_by(&:start)
294
+ warning = if !scannable_encoding
295
+ "Detection was partial because input encoding could not be scanned as UTF-8."
296
+ elsif degraded
297
+ warning = LINEAR_WARNING
298
+ if text.bytesize > MAX_ADDRESS_RESERVE_BYTES && needs_reserve
299
+ warning += " Address patterns were not scanned for input larger than 8 MiB."
300
+ end
301
+ if reserve_unscanned_bytes
302
+ first, last = reserve_unscanned_bytes
303
+ warning += " Linear reserve stopped; bytes #{first}..#{last} were not scanned. " \
304
+ "The end offset is exclusive (interval [#{first}, #{last})); " \
305
+ "equivalently, bytes #{first}..#{last - 1} were not scanned. " \
306
+ "That final byte offset is inclusive."
307
+ end
308
+ warning
309
+ end
310
+ Detection.new(matches: matches, degraded: degraded, coverage: "rules_only", warning: warning)
311
+ end
312
+
313
+ private
314
+
315
+ def scan_single_large_window(text, started)
316
+ matches = []
317
+ degraded = false
318
+ needs_reserve = false
319
+ LEGACY_SHORT_PATTERNS.each_key do |type|
320
+ next if type == :ccnum
321
+
322
+ if wall_two_seconds?(text, started)
323
+ degraded = true
324
+ break
325
+ end
326
+ if emergency_expired?(text, started)
327
+ degraded = true
328
+ needs_reserve = true
329
+ break
330
+ end
331
+
332
+ elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
333
+ regexp = Regexp.new(PATTERNS.fetch(type).source, PATTERNS.fetch(type).options,
334
+ timeout: [[2.0 - elapsed, EMERGENCY_WALL_SECONDS - elapsed].min, 0.001].max)
335
+ begin
336
+ text.to_enum(:scan, regexp).each do
337
+ match = Regexp.last_match
338
+ matches << PiiMatch.new(type: type, start: match.begin(0), finish: match.end(0), value: match[0])
339
+ if wall_two_seconds?(text, started)
340
+ degraded = true
341
+ break
342
+ end
343
+ if emergency_expired?(text, started)
344
+ degraded = true
345
+ needs_reserve = true
346
+ break
347
+ end
348
+ end
349
+ rescue Regexp::TimeoutError
350
+ degraded = true
351
+ needs_reserve = true
352
+ break
353
+ end
354
+ end
355
+
356
+ optional_cues_present = text.match?(OPTIONAL_CUE_RE)
357
+ %i[api_key auth_jwt nid_generic_labelled date_of_birth].each do |type|
358
+ if wall_two_seconds?(text, started)
359
+ degraded = true
360
+ break
361
+ end
362
+ if emergency_expired?(text, started)
363
+ degraded = true
364
+ needs_reserve = true
365
+ break
366
+ end
367
+ cue_type = %i[api_key nid_generic_labelled date_of_birth].include?(type)
368
+ unless (!cue_type || optional_cues_present) && pattern_possible?(type, text)
369
+ degraded = true
370
+ next
371
+ end
372
+
373
+ regexp = BOUNDED_PATTERNS.fetch(type)
374
+ begin
375
+ text.to_enum(:scan, regexp).each do
376
+ match = Regexp.last_match
377
+ matches << PiiMatch.new(type: type, start: match.begin(0), finish: match.end(0), value: match[0])
378
+ if wall_two_seconds?(text, started)
379
+ degraded = true
380
+ break
381
+ end
382
+ if emergency_expired?(text, started)
383
+ degraded = true
384
+ needs_reserve = true
385
+ break
386
+ end
387
+ end
388
+ rescue Regexp::TimeoutError
389
+ degraded = true
390
+ needs_reserve = true
391
+ break
392
+ end
393
+ end
394
+
395
+ if pattern_possible?(:nid_generic_labelled, text)
396
+ scan_labeled_ids(text.b, text.b.downcase, matches, ascii_offsets: text.ascii_only?)
397
+ end
398
+
399
+ if needs_reserve && text.bytesize <= MAX_ADDRESS_RESERVE_BYTES
400
+ linear_degraded = linear_matches(text, matches)
401
+ degraded ||= linear_degraded
402
+ end
403
+
404
+ merge_card_matches!(text, matches)
405
+ if wall_two_seconds?(text, started)
406
+ degraded = true
407
+ end
408
+ matches = matches.uniq { |match| [match.type, match.start, match.finish] }.sort_by(&:start)
409
+ warning = degraded ? LINEAR_WARNING : nil
410
+ Detection.new(matches: matches, degraded: degraded, coverage: "rules_only", warning: warning)
411
+ end
412
+
413
+ # The normal window pass owns this prefix. When its cap or deadline is
414
+ # reached on very large inputs, scan the remainder with the bounded,
415
+ # byte-oriented reserve in chunks. Overlap allows tokens crossing chunk
416
+ # boundaries; duplicate spans are removed by the caller.
417
+ def scan_linear_reserve_chunks(text, scanned_end_byte, out, deadline:)
418
+ total = text.bytesize
419
+ reserve_started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
420
+ deadline = [deadline, reserve_started + LINEAR_RESERVE_SECONDS].min
421
+ core_start = [scanned_end_byte - LINEAR_RESERVE_OVERLAP_BYTES, 0].max
422
+ core_start = utf8_boundary(text, core_start, backwards: true)
423
+ ascii_offsets = text.ascii_only?
424
+ base_chars = ascii_offsets ? core_start : text.byteslice(0...core_start).length
425
+ first_allowed = ascii_offsets ? [scanned_end_byte, core_start].min :
426
+ text.byteslice(0...[scanned_end_byte, core_start].min).length
427
+ while core_start < total
428
+ return [core_start, total] if monotonic_now >= deadline
429
+
430
+ core_end = utf8_boundary(text, [core_start + LINEAR_RESERVE_CHUNK_BYTES, total].min, backwards: true)
431
+ core_end = total if core_end <= core_start
432
+ chunk_end = utf8_boundary(text, [core_end + LINEAR_RESERVE_OVERLAP_BYTES, total].min, backwards: true)
433
+ chunk_end = total if chunk_end < core_end
434
+ chunk = text.byteslice(core_start...chunk_end)
435
+ chunk_matches = []
436
+ begin
437
+ linear_matches(chunk, chunk_matches, reserve_only: true, deadline: deadline)
438
+ rescue ReserveDeadlineReached
439
+ return [core_start, total]
440
+ end
441
+ core_chars = base_chars + (ascii_offsets ? core_end - core_start : text.byteslice(core_start...core_end).length)
442
+ chunk_matches.each do |match|
443
+ absolute_start = base_chars + match.start
444
+ next if absolute_start < first_allowed
445
+ next if absolute_start >= core_chars
446
+
447
+ out << PiiMatch.new(type: match.type, start: absolute_start,
448
+ finish: absolute_start + match.value.length, value: match.value)
449
+ end
450
+ base_chars = core_chars
451
+ core_start = core_end
452
+ end
453
+ nil
454
+ end
455
+
456
+ def pattern_possible?(type, window)
457
+ case type
458
+ when :email then window.include?("@")
459
+ when :phone then window.match?(/\d{3}/)
460
+ when :ssn then window.match?(/\d{4}/)
461
+ when :iban then window.match?(/\b[A-Z]{2}\d{2}/)
462
+ when :ip then window.count(".") >= 3 || window.include?(":")
463
+ when :ccnum then window.match?(/\b\d(?:[ \-.]?\d){12,18}\b/)
464
+ when :api_key
465
+ window.match?(API_KEY_CUE_RE)
466
+ when :auth_jwt then window.include?("eyJ") && window.count(".") >= 2
467
+ when :nid_generic_labelled
468
+ window.match?(NATIONAL_ID_CUE_RE)
469
+ when :date_of_birth
470
+ window.match?(DOB_CUE_RE)
471
+ else true
472
+ end
473
+ end
474
+
475
+ # A linear Unicode-token reserve for numbered addresses. The canonical
476
+ # expression remains useful on short text; larger text uses this scanner
477
+ # and an explicit anchor budget instead of spending unbounded regex work.
478
+ def scan_address_reserve(text, out)
479
+ words = []
480
+ byte_offset = 0
481
+ char_offset = 0
482
+ total_bytes = text.bytesize
483
+ token_byte = nil
484
+ token_char = nil
485
+ while byte_offset < total_bytes
486
+ first = text.getbyte(byte_offset)
487
+ if first < 0x80
488
+ size = 1
489
+ word_char = first.between?(48, 57) || first.between?(65, 90) || first.between?(97, 122) ||
490
+ first == 39 || first == 45
491
+ else
492
+ size = if first < 0xE0
493
+ 2
494
+ elsif first < 0xF0
495
+ 3
496
+ else
497
+ 4
498
+ end
499
+ char = text.byteslice(byte_offset, size)
500
+ word_char = char.match?(/\A[\p{L}\p{N}]\z/u) || char == "’"
501
+ end
502
+ if word_char
503
+ token_byte ||= byte_offset
504
+ token_char ||= char_offset
505
+ elsif token_byte
506
+ value = text.byteslice(token_byte...byte_offset)
507
+ words << address_word(value, token_byte, byte_offset, token_char, char_offset)
508
+ token_byte = token_char = nil
509
+ end
510
+ byte_offset += size
511
+ char_offset += 1
512
+ end
513
+ if token_byte
514
+ words << address_word(text.byteslice(token_byte...byte_offset), token_byte, byte_offset,
515
+ token_char, char_offset)
516
+ end
517
+
518
+ anchor_window = nil
519
+ window_anchor_count = 0
520
+ anchor_capped = false
521
+ words.each_with_index do |word, i|
522
+ next unless word[5]
523
+
524
+ digits = word[6]
525
+ next if digits.zero? || digits > 5
526
+
527
+ window = word[1] / WINDOW_BYTES
528
+ if window != anchor_window
529
+ anchor_window = window
530
+ window_anchor_count = 0
531
+ end
532
+ window_anchor_count += 1
533
+ anchor_capped ||= window_anchor_count > ADDRESS_ANCHOR_BUDGET
534
+
535
+ trailing_address = false
536
+ if digits <= 3 && i.positive?
537
+ ([i - 4, 0].max..i - 1).reverse_each do |j|
538
+ break if words[j][5]
539
+ if words[j][7] &&
540
+ (!standalone_address_suffix?(words[j]) ||
541
+ (j.positive? && !words[j - 1][5] && !words[j - 1][7] &&
542
+ words[j - 1][8] && words[j - 1][0].bytesize <= 128))
543
+ first = j
544
+ 2.times do
545
+ previous = first - 1
546
+ break if previous.negative? || !words[previous][8]
547
+ break unless whitespace_gap?(text, words[previous][2], words[first][1])
548
+ first = previous
549
+ end
550
+ if address_span_allowed?(text, words[first], word)
551
+ add_address_match(out, text, words[first], word)
552
+ trailing_address = true
553
+ end
554
+ break
555
+ end
556
+ end
557
+ end
558
+
559
+ name_or_number_start = word[5] || (!word[7] && word[8] && word[0].bytesize <= 128)
560
+ if !trailing_address && i + 1 < words.length && name_or_number_start &&
561
+ (i.zero? || leading_address_cue?(text, words, i))
562
+ ((i + 1)..[i + 5, words.length - 1].min).each do |j|
563
+ break if words[j][5]
564
+ if words[j][7] && (word[5] || (j + 1 < words.length && words[j + 1][5]))
565
+ add_address_match(out, text, word, words[j]) if address_span_allowed?(text, word, words[j])
566
+ break
567
+ end
568
+ end
569
+ end
570
+ end
571
+ anchor_capped || text.bytesize > WINDOW_BYTES
572
+ end
573
+
574
+ def address_word(value, byte_start, byte_end, char_start, char_end)
575
+ bytes = value
576
+ digit_count = 0
577
+ digit_count += 1 while digit_count < bytes.bytesize && bytes.getbyte(digit_count).between?(48, 57)
578
+ numeric = digit_count == bytes.bytesize
579
+ suffix_letter = digit_count.between?(1, 5) && digit_count + 1 == bytes.bytesize &&
580
+ (bytes.getbyte(digit_count).between?(65, 90) || bytes.getbyte(digit_count).between?(97, 122))
581
+ number = numeric || suffix_letter
582
+ digit_count = 0 unless number
583
+ # Numeric anchors cannot be street suffixes. Avoid title-case checks for
584
+ # every number, and pass the already-computed title flag through rather
585
+ # than evaluating it again inside the suffix lookup.
586
+ title_word = !number && street_title_word?(value)
587
+ suffix_word = title_word && street_suffix?(value)
588
+ [value, byte_start, byte_end, char_start, char_end, number, digit_count, suffix_word, title_word]
589
+ end
590
+
591
+ def add_address_match(out, text, first, last)
592
+ value = text.byteslice(first[1]...last[2])
593
+ out << PiiMatch.new(type: :address, start: first[3], finish: last[4], value: value)
594
+ end
595
+
596
+ def address_span_allowed?(text, first, last)
597
+ span = text.byteslice(first[1]...last[2])
598
+ !span.match?(/[|\r\n]/)
599
+ end
600
+
601
+ def leading_address_cue?(text, words, index)
602
+ previous = words[index - 1]
603
+ gap = text.byteslice(previous[2]...words[index][1])
604
+ return true if gap.match?(/[|,.;:!?\n\r]/)
605
+
606
+ %w[to at address send mail live from].include?(previous[0].downcase)
607
+ end
608
+
609
+ def street_suffix?(word)
610
+ lower = word.downcase
611
+ candidates = ADDRESS_STREET_SUFFIXES_BY_END[lower[-2, 2]]
612
+ candidates && candidates.any? { |suffix| lower.end_with?(suffix) }
613
+ end
614
+
615
+ def standalone_address_suffix?(word)
616
+ ADDRESS_STREET_SUFFIXES.include?(word[0].downcase)
617
+ end
618
+
619
+ def whitespace_gap?(text, first, last)
620
+ return false if first >= last
621
+
622
+ while first < last
623
+ byte = text.getbyte(first)
624
+ return false unless byte == 32 || byte.between?(9, 13)
625
+
626
+ first += 1
627
+ end
628
+ true
629
+ end
630
+
631
+ def street_title_word?(word)
632
+ first = word[0]
633
+ first && first == first.upcase && first != first.downcase
634
+ end
635
+
636
+ # Wall-clock degradation is reserved for inputs larger than one core
637
+ # window or a genuinely hung scan. Ordinary short text is governed only by
638
+ # byte windows and the deterministic per-class match-step limit.
639
+ def monotonic_now
640
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
641
+ end
642
+
643
+ # Same 2s wall the tokenizer uses. Sets the flag tokenize_budget_hit?
644
+ # reads, and tells the caller to stop this pass.
645
+ def wall_two_seconds?(text, started)
646
+ return false if text.bytesize <= SHORT_COMPAT_BYTES
647
+ return false if Process.clock_gettime(Process::CLOCK_MONOTONIC) - started < 2.0
648
+
649
+ self.class.note_wall_budget_hit!
650
+ true
651
+ end
652
+
653
+ def emergency_expired?(text, started)
654
+ return false if text.bytesize > ADDRESS_WINDOW_SCAN_BYTES && text.bytesize <= MAX_ADDRESS_RESERVE_BYTES
655
+
656
+ elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
657
+ limit = text.bytesize > WINDOW_BYTES ? EMERGENCY_WALL_SECONDS : SHORT_INPUT_HANG_SECONDS
658
+ elapsed >= limit
659
+ end
660
+
661
+ # Byte boundaries are moved to UTF-8 character boundaries before slicing.
662
+ # Every core is at most WINDOW_BYTES and the right overlap is WINDOW_BYTES.
663
+ def each_window(text)
664
+ total = text.bytesize
665
+ core_start = 0
666
+ while core_start < total
667
+ core_end = utf8_boundary(text, [core_start + WINDOW_BYTES, total].min, backwards: true)
668
+ core_end = total if core_end <= core_start
669
+ window_end = utf8_boundary(text, [core_end + WINDOW_BYTES, total].min, backwards: true)
670
+ window_end = total if window_end < core_end
671
+ yield text.byteslice(core_start...window_end), core_start, core_end
672
+ core_start = core_end
673
+ end
674
+ end
675
+
676
+ def owns_match_start?(global_byte, core_end_byte)
677
+ global_byte < core_end_byte
678
+ end
679
+
680
+ def span_too_long?(span_bytes)
681
+ span_bytes > MAX_SPAN_BYTES
682
+ end
683
+
684
+ def utf8_boundary(text, offset, backwards:)
685
+ return offset if offset >= text.bytesize
686
+ return offset if (text.getbyte(offset) & 0xC0) != 0x80
687
+
688
+ if backwards
689
+ offset -= 1 while offset.positive? && (text.getbyte(offset) & 0xC0) == 0x80
690
+ offset
691
+ else
692
+ offset += 1 while offset < text.bytesize && (text.getbyte(offset) & 0xC0) == 0x80
693
+ offset
694
+ end
695
+ end
696
+
697
+ # These scans are deliberately byte-oriented and have no regex/backtracking.
698
+ # They run over the complete input, including text after regex timeout.
699
+ def linear_matches(text, out, reserve_only: false, deadline: nil)
700
+ bytes = text.b
701
+ ascii_offsets = text.ascii_only?
702
+ lower = bytes.downcase
703
+ digit_run = bytes.bytesize > 64 && bytes.match?(ALL_DIGITS_RE)
704
+ numeric_candidate = !digit_run && bytes.match?(NUMERIC_FALLBACK_RE)
705
+ iban_candidate = !digit_run && bytes.match?(IBAN_SHAPE_RE)
706
+ nid_candidate = !reserve_only && !digit_run && bytes.match?(NATIONAL_ID_SHAPE_RE)
707
+ email_candidate = bytes.include?("@")
708
+ credential_candidate = %w[api key secret token bearer sk-live- sk-test- sk-prod- sk-proj- pk-live- rk-live- ak-live- eyj].any? do |cue|
709
+ lower.include?(cue)
710
+ end
711
+ labeled_id_candidate = NATIONAL_ID_CUES.any? { |cue| lower.include?(cue.b) }
712
+ dob_candidate = DOB_CUES.any? { |cue| lower.include?(cue.b) }
713
+ mrz_candidate = !reserve_only && bytes.include?("<<")
714
+ return false unless digit_run || numeric_candidate || iban_candidate || nid_candidate || email_candidate ||
715
+ credential_candidate || labeled_id_candidate || dob_candidate || mrz_candidate
716
+
717
+ digit_positions = !digit_run && (numeric_candidate || iban_candidate || nid_candidate) ?
718
+ digit_offsets(bytes) : []
719
+ overlong_email = scan_email(bytes, out, deadline: deadline, ascii_offsets: ascii_offsets)
720
+ overlong_number = digit_run
721
+ overlong_number ||= scan_numbers(bytes, digit_positions, out, ascii_offsets: ascii_offsets) if numeric_candidate
722
+ overlong_nid = scan_national_ids(bytes, digit_positions, out, ascii_offsets: ascii_offsets) if nid_candidate
723
+ overlong_iban = scan_iban(bytes, digit_positions, out, ascii_offsets: ascii_offsets) if iban_candidate
724
+ overlong_credential = scan_credentials(bytes, lower, out, ascii_offsets: ascii_offsets)
725
+ overlong_id = scan_labeled_ids(bytes, lower, out, ascii_offsets: ascii_offsets)
726
+ overlong_dob = scan_marked_dob(bytes, lower, out, ascii_offsets: ascii_offsets)
727
+ overlong_mrz = scan_mrz(bytes, out, ascii_offsets: ascii_offsets)
728
+ protected = out.select do |match|
729
+ %i[ccnum iban ssn nid_mrz].include?(match.type) ||
730
+ (match.type == :nid_generic_labelled && match.value.match?(/[A-Za-z]/))
731
+ end
732
+ protected.sort_by!(&:start)
733
+ protected_index = 0
734
+ out.reject! do |match|
735
+ next false unless match.type == :phone
736
+
737
+ protected_index += 1 while protected_index < protected.length && protected[protected_index].finish <= match.start
738
+ other = protected[protected_index]
739
+ other && match.start < other.finish && other.start < match.finish
740
+ end
741
+ overlong_email || overlong_number || overlong_nid || overlong_iban || overlong_credential ||
742
+ overlong_id || overlong_dob || overlong_mrz
743
+ end
744
+
745
+ def digit_offsets(bytes)
746
+ positions = []
747
+ bytes.each_byte.with_index { |byte, index| positions << index if digit?(byte) }
748
+ positions
749
+ end
750
+
751
+ def add_bytes(text, out, type, first, last)
752
+ return if last <= first
753
+
754
+ value = text.byteslice(first...last).force_encoding(text.encoding)
755
+ return unless value.valid_encoding?
756
+
757
+ # Convert the detector's byte offsets to the character offsets consumed
758
+ # by the existing tokenizer API.
759
+ start = if text.encoding == Encoding::BINARY && text.ascii_only?
760
+ first
761
+ else
762
+ prefix = text.byteslice(0...first).force_encoding(text.encoding)
763
+ prefix.valid_encoding? ? prefix.length : first
764
+ end
765
+ out << PiiMatch.new(type: type, start: start, finish: start + value.length, value: value)
766
+ end
767
+
768
+ def scan_email(bytes, out, deadline: nil, ascii_offsets: false)
769
+ overlong = false
770
+ i = 0
771
+ candidates = 0
772
+ while (i = bytes.index("@", i))
773
+ candidates += 1
774
+ raise ReserveDeadlineReached if deadline && (candidates & 1023).zero? && monotonic_now >= deadline
775
+
776
+ left = i
777
+ left -= 1 while left.positive? && email_local_byte?(bytes.getbyte(left - 1)) && i - left < 64
778
+ right = i + 1
779
+ right += 1 while right < bytes.bytesize && email_domain_byte?(bytes.getbyte(right)) && right - i <= 256
780
+ candidate = bytes.byteslice(left...right)
781
+ local, domain = candidate.split("@", 2)
782
+ local_truncated = left.positive? && email_local_byte?(bytes.getbyte(left - 1))
783
+ domain_truncated = right < bytes.bytesize && email_domain_byte?(bytes.getbyte(right))
784
+ overlong ||= local_truncated || domain_truncated
785
+ if !local_truncated && local && domain && !local.empty? && local.bytesize <= 64 && domain.bytesize <= 255 &&
786
+ domain.include?(".") && domain !~ /\.\z|\.\./
787
+ if ascii_offsets
788
+ value = candidate.dup.force_encoding("UTF-8")
789
+ out << PiiMatch.new(type: :email, start: left, finish: right, value: value)
790
+ else
791
+ add_bytes(bytes, out, :email, left, right)
792
+ end
793
+ end
794
+ i += 1
795
+ end
796
+ overlong
797
+ end
798
+
799
+ def email_local_byte?(b)
800
+ b && (b >= 128 || b.between?(48, 57) || b.between?(65, 90) || b.between?(97, 122) || "._%+-".bytes.include?(b))
801
+ end
802
+
803
+ def email_domain_byte?(b)
804
+ b && (b >= 128 || b.between?(48, 57) || b.between?(65, 90) || b.between?(97, 122) || b == 45 || b == 46)
805
+ end
806
+
807
+ def match_continues?(type, byte)
808
+ case type
809
+ when :email
810
+ email_local_byte?(byte) || email_domain_byte?(byte)
811
+ when :phone, :ssn, :ccnum
812
+ # A space or hyphen before a number is a delimiter, not evidence that
813
+ # this window clipped the number. Only an adjacent digit can continue
814
+ # one of these numeric spans across a core boundary.
815
+ digit?(byte)
816
+ when :iban
817
+ letter?(byte) || digit?(byte) || byte == 32
818
+ when :api_key
819
+ credential_byte?(byte)
820
+ when :auth_jwt
821
+ credential_byte?(byte) || byte == 46
822
+ when :nid_generic_labelled, :address, :date_of_birth
823
+ alnum_byte?(byte) || (byte && byte >= 128)
824
+ else
825
+ false
826
+ end
827
+ end
828
+
829
+ def scan_numbers(bytes, digit_positions, out, ascii_offsets: false)
830
+ overlong = false
831
+ position = 0
832
+ while position < digit_positions.length
833
+ i = digit_positions[position]
834
+ if i.positive? && alnum_byte?(bytes.getbyte(i - 1))
835
+ position += 1
836
+ next
837
+ end
838
+ finish = i
839
+ digits = 0
840
+ separator_run = 0
841
+ while finish < bytes.bytesize && finish - i < 64
842
+ b = bytes.getbyte(finish)
843
+ if digit?(b)
844
+ digits += 1
845
+ separator_run = 0
846
+ elsif [32, 45, 46, 40, 41, 43].include?(b)
847
+ separator_run += 1
848
+ break if separator_run > 2
849
+ else
850
+ break
851
+ end
852
+ finish += 1
853
+ end
854
+ candidate_overlong = finish - i >= 64 && finish < bytes.bytesize &&
855
+ (digit?(bytes.getbyte(finish)) || [32, 45, 46].include?(bytes.getbyte(finish)))
856
+ overlong ||= candidate_overlong
857
+ finish -= 1 while finish > i && [32, 45, 46, 40, 41, 43].include?(bytes.getbyte(finish - 1))
858
+ raw = bytes.byteslice(i...finish)
859
+ compact = raw.delete("- .()+")
860
+ if !candidate_overlong && digits == 9 && raw.bytesize <= 11 && valid_ssn?(compact)
861
+ out << byte_match(bytes, :ssn, i, finish, ascii_offsets: ascii_offsets)
862
+ elsif !candidate_overlong && digits.between?(13, 19) && raw.bytesize <= 64 && luhn?(compact) && supported_card_network?(compact)
863
+ out << byte_match(bytes, :ccnum, i, finish, ascii_offsets: ascii_offsets)
864
+ elsif !candidate_overlong && !alnum_byte?(bytes.getbyte(finish)) && phone_candidate?(raw)
865
+ out << byte_match(bytes, :phone, i, finish, ascii_offsets: ascii_offsets)
866
+ end
867
+ position += 1
868
+ position += 1 while position < digit_positions.length && digit_positions[position] < finish
869
+ end
870
+ overlong
871
+ end
872
+
873
+ # Keep the reserve scanner's phone formats no broader than the legacy
874
+ # detector. Arbitrary 7+ digit identifiers are not phone numbers.
875
+ def phone_candidate?(raw)
876
+ match = PATTERNS.fetch(:phone).match(raw)
877
+ match && match.begin(0).zero? && match.end(0) == raw.bytesize
878
+ end
879
+
880
+ # The canonical ruleset marks structured national-ID families with a
881
+ # checksum name. Keep this reserve byte-oriented and bounded; do not run
882
+ # the canonical regexes again after their budget has expired.
883
+ def scan_national_ids(bytes, digit_positions, out, ascii_offsets: false)
884
+ return if digit_positions.empty?
885
+
886
+ position = 0
887
+ while position < digit_positions.length
888
+ first = digit_positions[position]
889
+ probe = first
890
+ candidate_digits = 0
891
+ separator_run = 0
892
+ while probe < bytes.bytesize && probe - first < 64
893
+ byte = bytes.getbyte(probe)
894
+ if digit?(byte)
895
+ candidate_digits += 1
896
+ separator_run = 0
897
+ elsif [32, 45, 46, 47].include?(byte)
898
+ separator_run += 1
899
+ break if separator_run > 1
900
+ else
901
+ break
902
+ end
903
+ probe += 1
904
+ end
905
+ if candidate_digits < 8
906
+ cue_context = bytes.byteslice([first - 40, 0].max...first)
907
+ unless NATIONAL_ID_CHECKSUM_CUE_RE.match?(cue_context)
908
+ position += 1
909
+ position += 1 while position < digit_positions.length && digit_positions[position] < probe
910
+ next
911
+ end
912
+ end
913
+ unless first.zero? || !alnum_byte?(bytes.getbyte(first - 1))
914
+ position += 1
915
+ next
916
+ end
917
+
918
+ last = first
919
+ digits = 0
920
+ separator_run = 0
921
+ while last < bytes.bytesize && last - first < 64
922
+ byte = bytes.getbyte(last)
923
+ if digit?(byte) || letter?(byte)
924
+ digits += 1 if digit?(byte)
925
+ separator_run = 0
926
+ last += 1
927
+ elsif [32, 45, 46, 47].include?(byte)
928
+ separator_run += 1
929
+ break if separator_run > 1
930
+ last += 1
931
+ else
932
+ break
933
+ end
934
+ end
935
+ overlong = last - first >= 64 && last < bytes.bytesize &&
936
+ (digit?(bytes.getbyte(last)) || letter?(bytes.getbyte(last)) ||
937
+ [32, 45, 46, 47].include?(bytes.getbyte(last)))
938
+ unless overlong
939
+ candidate = bytes.byteslice(first...last)
940
+ cue_context = bytes.byteslice([first - 40, 0].max...first).downcase
941
+ if national_id_checksum_valid?(candidate, digits, cue_context)
942
+ out << byte_match(bytes, :nid_generic_labelled, first, last, ascii_offsets: ascii_offsets)
943
+ end
944
+ end
945
+ position += 1
946
+ position += 1 while position < digit_positions.length && digit_positions[position] < last
947
+ end
948
+ token_start = 0
949
+ while token_start < bytes.bytesize
950
+ unless alnum_byte?(bytes.getbyte(token_start)) &&
951
+ (token_start.zero? || !alnum_byte?(bytes.getbyte(token_start - 1)))
952
+ token_start += 1
953
+ next
954
+ end
955
+ token_end = token_start
956
+ token_end += 1 while token_end < bytes.bytesize && alnum_byte?(bytes.getbyte(token_end)) &&
957
+ token_end - token_start < 64
958
+ overlong ||= token_end - token_start == 64 && token_end < bytes.bytesize && alnum_byte?(bytes.getbyte(token_end))
959
+ if token_end - token_start == 18
960
+ value = bytes.byteslice(token_start...token_end).upcase
961
+ if value.match?(/\A[A-Z]{4}\d{6}[HM][A-Z]{5}[A-Z0-9]\d\z/) && valid_mx_curp?(value)
962
+ out << byte_match(bytes, :nid_generic_labelled, token_start, token_end)
963
+ end
964
+ end
965
+ token_start = [token_end, token_start + 1].max
966
+ end
967
+ overlong
968
+ end
969
+
970
+ def alnum_byte?(byte)
971
+ digit?(byte) || letter?(byte)
972
+ end
973
+
974
+ def id_word_byte?(byte)
975
+ alnum_byte?(byte) || byte == 95 || (byte && byte >= 128)
976
+ end
977
+
978
+ def national_id_checksum_valid?(candidate, digits, cue_context)
979
+ compact = candidate.upcase.delete("^A-Z0-9")
980
+ case digits
981
+ when 8
982
+ candidate.include?("-") && valid_cl_rut?(compact)
983
+ when 9
984
+ (candidate.include?("-") && valid_cl_rut?(compact)) ||
985
+ (cue_context.include?("bsn") && valid_bsn?(compact)) ||
986
+ ((cue_context.include?("sin") || cue_context.include?("social insurance number")) && valid_luhn?(compact))
987
+ when 10
988
+ valid_cz_sk_rc?(compact) || valid_nid_bare10?(compact)
989
+ when 11
990
+ return valid_pl_pesel?(compact) if cue_context.include?("pesel")
991
+ ((candidate.include?("-") || candidate.include?(".")) && valid_cpf?(compact)) ||
992
+ valid_be_nrn?(compact) || valid_nid_bare11?(compact)
993
+ when 12
994
+ return valid_verhoeff?(compact) if %w[aadhaar aadhar uid].any? { |cue| cue_context.include?(cue) }
995
+ return valid_jp_mynumber?(compact) if cue_context.include?("my number")
996
+ valid_verhoeff?(compact) || valid_jp_mynumber?(compact)
997
+ when 13
998
+ valid_nid_bare13?(compact)
999
+ when 16
1000
+ valid_id_nik?(compact)
1001
+ when 17
1002
+ valid_cn_ric?(compact)
1003
+ else
1004
+ false
1005
+ end
1006
+ end
1007
+
1008
+ def valid_verhoeff?(value)
1009
+ d = [[0,1,2,3,4,5,6,7,8,9],[1,2,3,4,0,6,7,8,9,5],[2,3,4,0,1,7,8,9,5,6],
1010
+ [3,4,0,1,2,8,9,5,6,7],[4,0,1,2,3,9,5,6,7,8],[5,9,8,7,6,0,4,3,2,1],
1011
+ [6,5,9,8,7,1,0,4,3,2],[7,6,5,9,8,2,1,0,4,3],[8,7,6,5,9,3,2,1,0,4],
1012
+ [9,8,7,6,5,4,3,2,1,0]]
1013
+ p = [[0,1,2,3,4,5,6,7,8,9],[1,5,7,6,2,8,3,0,9,4],[5,8,0,3,7,9,6,1,4,2],
1014
+ [8,9,1,6,0,4,3,5,2,7],[9,4,5,3,1,2,6,8,7,0],[4,2,8,6,5,7,3,9,0,1],
1015
+ [2,7,9,3,8,0,6,4,1,5],[7,0,4,6,9,1,3,2,5,8]]
1016
+ return false unless value.match?(/\A\d+\z/)
1017
+ c = 0
1018
+ value.reverse.each_char.with_index { |ch, i| c = d[c][p[i % 8][ch.ord - 48]] }
1019
+ c.zero?
1020
+ end
1021
+
1022
+ def valid_bsn?(value)
1023
+ return false unless value.match?(/\A\d{8,9}\z/)
1024
+ digits = value.chars.map(&:to_i)
1025
+ return false unless digits.length == 9
1026
+ (digits.first(8).zip((2..9).to_a.reverse).sum { |n, w| n * w } % 11) == digits[8]
1027
+ end
1028
+
1029
+ def valid_cpf?(value)
1030
+ return false unless value.match?(/\A\d{11}\z/) && value.chars.uniq.length > 1
1031
+ nums = value.chars.map(&:to_i)
1032
+ [9, 10].each do |length|
1033
+ sum = nums.first(length).each_with_index.sum { |n, i| n * (length + 1 - i) }
1034
+ check = (sum * 10) % 11
1035
+ check = 0 if check == 10
1036
+ return false unless check == nums[length]
1037
+ end
1038
+ true
1039
+ end
1040
+
1041
+ def valid_cl_rut?(value)
1042
+ return false unless value.match?(/\A\d{7,8}[\dK]\z/)
1043
+ body = value[0...-1].chars.reverse.map(&:to_i)
1044
+ sum = body.each_with_index.sum { |n, i| n * (i % 6 + 2) }
1045
+ rem = 11 - sum % 11
1046
+ expected = rem == 11 ? "0" : rem == 10 ? "K" : rem.to_s
1047
+ value[-1] == expected
1048
+ end
1049
+
1050
+ def valid_jp_mynumber?(value)
1051
+ return false unless value.match?(/\A\d{12}\z/)
1052
+ nums = value.chars.map(&:to_i)
1053
+ sum = (1..11).sum do |n|
1054
+ weight = n <= 6 ? n + 1 : n - 5
1055
+ nums[11 - n] * weight
1056
+ end
1057
+ remainder = sum % 11
1058
+ check = remainder <= 1 ? 0 : 11 - remainder
1059
+ nums[11] == check
1060
+ end
1061
+
1062
+ def valid_be_nrn?(value)
1063
+ return false unless value.match?(/\A\d{11}\z/)
1064
+ digits = value.to_i
1065
+ (97 - (digits / 100) % 97) == digits % 100 ||
1066
+ (97 - ((2_000_000_000 + digits / 100) % 97)) == digits % 100
1067
+ end
1068
+
1069
+ def valid_cz_sk_rc?(value)
1070
+ return false unless value.match?(/\A\d{10}\z/)
1071
+ d = value.chars.map(&:to_i)
1072
+ encoded_month = d[2] * 10 + d[3]
1073
+ month = case encoded_month
1074
+ when 1..12 then encoded_month
1075
+ when 21..32 then encoded_month - 20
1076
+ when 51..62 then encoded_month - 50
1077
+ when 71..82 then encoded_month - 70
1078
+ else return false
1079
+ end
1080
+ day = d[4] * 10 + d[5]
1081
+ year_valid = [1800, 1900, 2000, 2100].any? do |century|
1082
+ Date.valid_date?(century + d[0] * 10 + d[1], month, day)
1083
+ end
1084
+ year_valid && (value.to_i % 11).zero?
1085
+ end
1086
+
1087
+ def valid_cn_ric?(value)
1088
+ return false unless value.match?(/\A\d{17}[\dX]\z/)
1089
+ weights = [7,9,10,5,8,4,2,1,6,3,7,9,10,5,8,4,2]
1090
+ chars = "10X98765432"
1091
+ sum = value[0, 17].chars.each_with_index.sum { |ch, i| ch.to_i * weights[i] }
1092
+ value[-1] == chars[sum % 11]
1093
+ end
1094
+
1095
+ def valid_mx_curp?(value)
1096
+ return false unless value.match?(/\A[A-Z]{4}\d{6}[HM][A-Z]{5}[A-Z0-9]\d\z/)
1097
+ dictionary = "0123456789ABCDEFGHIJKLMNÑOPQRSTUVWXYZ"
1098
+ sum = value[0, 17].chars.each_with_index.sum do |ch, i|
1099
+ pos = dictionary.index(ch)
1100
+ return false unless pos
1101
+ pos * (18 - i)
1102
+ end
1103
+ value[-1].to_i == (10 - sum % 10) % 10
1104
+ end
1105
+
1106
+ def valid_luhn?(value)
1107
+ value.match?(/\A\d+\z/) && luhn?(value)
1108
+ end
1109
+
1110
+ def valid_nid_bare10?(value)
1111
+ return false unless value.match?(/\A\d{10}\z/)
1112
+ valid_bg_egn?(value) || valid_se_pnr?(value) || valid_at_svnr?(value) || valid_nid_bare10_legacy?(value)
1113
+ end
1114
+
1115
+ def valid_nid_bare10_legacy?(value)
1116
+ d = value.chars.map(&:to_i)
1117
+ month = d[2] * 10 + d[3]
1118
+ month -= 20 if month.between?(21, 32)
1119
+ month -= 40 if month.between?(41, 52)
1120
+ day = d[4] * 10 + d[5]
1121
+ year_valid = [1800, 1900, 2000, 2100].any? do |century|
1122
+ Date.valid_date?(century + d[0] * 10 + d[1], month, day)
1123
+ end
1124
+ return false unless year_valid
1125
+ weights = [2, 4, 8, 5, 10, 9, 7, 3, 6]
1126
+ check = d.first(9).each_with_index.sum { |n, i| n * weights[i] } % 11
1127
+ check = 0 if check == 10
1128
+ check == d[9]
1129
+ end
1130
+
1131
+ def valid_bg_egn?(value)
1132
+ d = value.chars.map(&:to_i)
1133
+ day = d[0] * 10 + d[1]
1134
+ encoded_month = d[2] * 10 + d[3]
1135
+ year_offset, month = case encoded_month
1136
+ when 1..12 then [1900, encoded_month]
1137
+ when 21..32 then [1800, encoded_month - 20]
1138
+ when 41..52 then [2000, encoded_month - 40]
1139
+ when 61..72 then [2100, encoded_month - 60]
1140
+ else return false
1141
+ end
1142
+ return false unless Date.valid_date?(year_offset + d[4] * 10 + d[5], month, day)
1143
+ weights = [2, 4, 8, 5, 10, 9, 7, 3, 6]
1144
+ check = d.first(9).each_with_index.sum { |n, i| n * weights[i] } % 11
1145
+ check = 0 if check == 10
1146
+ check == d[9]
1147
+ end
1148
+
1149
+ def valid_nid_bare11?(value)
1150
+ return false unless value.match?(/\A\d{11}\z/)
1151
+ valid_no_fnr?(value) || valid_hr_oib?(value) || valid_lt_ee_ik?(value) ||
1152
+ valid_tr_tckn?(value) || valid_pl_pesel?(value) || valid_ar_cuit?(value) ||
1153
+ valid_ru_snils?(value) || valid_be_nrn?(value)
1154
+ end
1155
+
1156
+ def valid_no_fnr?(value)
1157
+ return false unless value.match?(/\A\d{11}\z/)
1158
+ d = value.chars.map(&:to_i)
1159
+ day, month = d[0] * 10 + d[1], d[2] * 10 + d[3]
1160
+ day -= 40 if day.between?(41, 71)
1161
+ month -= 40 if month.between?(41, 52)
1162
+ return false unless valid_day_month?(day, month)
1163
+ w1 = [3,7,6,1,8,9,4,5,2]
1164
+ k1 = 11 - (0...9).sum { |i| d[i] * w1[i] } % 11
1165
+ k1 = 0 if k1 == 11
1166
+ return false if k1 == 10 || d[9] != k1
1167
+ w2 = [5,4,3,2,7,6,5,4,3,2]
1168
+ k2 = 11 - (0...10).sum { |i| d[i] * w2[i] } % 11
1169
+ k2 = 0 if k2 == 11
1170
+ k2 != 10 && d[10] == k2
1171
+ end
1172
+
1173
+ def valid_nid_bare13?(value)
1174
+ return false unless value.match?(/\A\d{13}\z/)
1175
+ valid_jmbg?(value) || valid_ro_cnp?(value) || valid_za_id?(value) || valid_kr_rrn?(value)
1176
+ end
1177
+
1178
+ def valid_jmbg?(value)
1179
+ d = value.chars.map(&:to_i)
1180
+ day, month = d[0] * 10 + d[1], d[2] * 10 + d[3]
1181
+ return false unless valid_day_month?(day, month)
1182
+ w = [7, 6, 5, 4, 3, 2, 7, 6, 5, 4, 3, 2]
1183
+ sum = d.first(12).each_with_index.sum { |n, i| n * w[i] }
1184
+ check = 11 - sum % 11
1185
+ check = 0 if check >= 10
1186
+ check == d[12]
1187
+ end
1188
+
1189
+ def valid_day_month?(day, month)
1190
+ day.between?(1, 31) && month.between?(1, 12)
1191
+ end
1192
+
1193
+ def valid_se_pnr?(value)
1194
+ d = value.chars.map(&:to_i)
1195
+ month = d[2] * 10 + d[3]
1196
+ day = d[4] * 10 + d[5]
1197
+ valid_day_month?(day, month) && luhn?(value)
1198
+ end
1199
+
1200
+ def valid_at_svnr?(value)
1201
+ d = value.chars.map(&:to_i)
1202
+ return false unless valid_day_month?(d[4] * 10 + d[5], d[6] * 10 + d[7])
1203
+ indices = [0, 1, 2, 4, 5, 6, 7, 8, 9]
1204
+ weights = [3, 7, 9, 5, 8, 4, 2, 1, 6]
1205
+ check = indices.each_with_index.sum { |idx, i| d[idx] * weights[i] } % 11
1206
+ check != 10 && check == d[3]
1207
+ end
1208
+
1209
+ def valid_hr_oib?(value)
1210
+ d = value.chars.map(&:to_i)
1211
+ m = 10
1212
+ 10.times do |i|
1213
+ m = (m + d[i]) % 10
1214
+ m = 10 if m.zero?
1215
+ m = (m * 2) % 11
1216
+ end
1217
+ check = 11 - m
1218
+ check = 0 if check == 10
1219
+ check == d[10]
1220
+ end
1221
+
1222
+ def valid_lt_ee_ik?(value)
1223
+ d = value.chars.map(&:to_i)
1224
+ return false unless d[0].between?(1, 8) && valid_day_month?(d[5] * 10 + d[6], d[3] * 10 + d[4])
1225
+ w1 = [1,2,3,4,5,6,7,8,9,1]
1226
+ check = d.first(10).each_with_index.sum { |n, i| n * w1[i] } % 11
1227
+ if check == 10
1228
+ w2 = [3,4,5,6,7,8,9,1,2,3]
1229
+ check = d.first(10).each_with_index.sum { |n, i| n * w2[i] } % 11
1230
+ check = 0 if check == 10
1231
+ end
1232
+ check == d[10]
1233
+ end
1234
+
1235
+ def valid_tr_tckn?(value)
1236
+ d = value.chars.map(&:to_i)
1237
+ return false if d[0].zero? || d.first(10).sum % 10 != d[10]
1238
+ odd = [0,2,4,6,8].sum { |i| d[i] }
1239
+ even = [1,3,5,7].sum { |i| d[i] }
1240
+ ((odd * 7 - even) % 10) == d[9]
1241
+ end
1242
+
1243
+ def valid_pl_pesel?(value)
1244
+ d = value.chars.map(&:to_i)
1245
+ month_raw = d[2] * 10 + d[3]
1246
+ month = [0,20,40,60,80].filter_map { |offset| month_raw - offset if (1 + offset..12 + offset).cover?(month_raw) }.first
1247
+ return false unless month && valid_day_month?(d[4] * 10 + d[5], month)
1248
+ w = [1,3,7,9,1,3,7,9,1,3]
1249
+ check = (10 - d.first(10).each_with_index.sum { |n, i| n * w[i] } % 10) % 10
1250
+ check == d[10]
1251
+ end
1252
+
1253
+ def valid_ar_cuit?(value)
1254
+ d = value.chars.map(&:to_i)
1255
+ w = [5,4,3,2,7,6,5,4,3,2]
1256
+ sum = d.first(10).each_with_index.sum { |n, i| n * w[i] }
1257
+ check = 11 - sum % 11
1258
+ check = 0 if check == 11
1259
+ check = 9 if check == 10
1260
+ check == d[10]
1261
+ end
1262
+
1263
+ def valid_ru_snils?(value)
1264
+ d = value.chars.map(&:to_i)
1265
+ sum = d.first(9).each_with_index.sum { |n, i| n * (9 - i) }
1266
+ check = sum < 100 ? sum : sum % 101
1267
+ check = 0 if check == 100
1268
+ check == d[9] * 10 + d[10]
1269
+ end
1270
+
1271
+ def valid_ro_cnp?(value)
1272
+ d = value.chars.map(&:to_i)
1273
+ return false unless d[0].between?(1, 9) && valid_day_month?(d[5] * 10 + d[6], d[3] * 10 + d[4])
1274
+ w = [2,7,9,1,4,6,3,5,8,2,7,9]
1275
+ check = d.first(12).each_with_index.sum { |n, i| n * w[i] } % 11
1276
+ check = 1 if check == 10
1277
+ check == d[12]
1278
+ end
1279
+
1280
+ def valid_za_id?(value)
1281
+ d = value.chars.map(&:to_i)
1282
+ valid_day_month?(d[4] * 10 + d[5], d[2] * 10 + d[3]) && luhn?(value)
1283
+ end
1284
+
1285
+ def valid_kr_rrn?(value)
1286
+ d = value.chars.map(&:to_i)
1287
+ return false unless d[6].between?(1, 8) && valid_day_month?(d[4] * 10 + d[5], d[2] * 10 + d[3])
1288
+ w = [2,3,4,5,6,7,8,9,2,3,4,5]
1289
+ check = (11 - d.first(12).each_with_index.sum { |n, i| n * w[i] } % 11) % 10
1290
+ check == d[12]
1291
+ end
1292
+
1293
+ def valid_id_nik?(value)
1294
+ return false unless value.match?(/\A\d{16}\z/)
1295
+ d = value.chars.map(&:to_i)
1296
+ province = d[0] * 10 + d[1]
1297
+ day = d[6] * 10 + d[7]
1298
+ day -= 40 if day > 40
1299
+ valid_day_month?(day, d[8] * 10 + d[9]) && province.between?(11, 96)
1300
+ end
1301
+
1302
+ def byte_match(bytes, type, first, last, ascii_offsets: false)
1303
+ value = bytes.byteslice(first...last).dup.force_encoding("UTF-8")
1304
+ start = ascii_offsets ? first : bytes.byteslice(0...first).force_encoding("UTF-8").scrub.length
1305
+ PiiMatch.new(type: type, start: start, finish: start + value.length, value: value)
1306
+ end
1307
+
1308
+ def digit?(b)
1309
+ b && b.between?(48, 57)
1310
+ end
1311
+
1312
+ def valid_ssn?(digits)
1313
+ digits.bytesize == 9 && digits[0, 3].to_i.between?(1, 899) && digits[0, 3] != "666" &&
1314
+ digits[3, 2] != "00" && digits[5, 4] != "0000"
1315
+ end
1316
+
1317
+ def merge_card_matches!(text, matches)
1318
+ cards = card_matches(text)
1319
+ return if cards.empty?
1320
+
1321
+ # Cards from one left-to-right run are disjoint and ordered by start.
1322
+ # A linear any?-scan is quadratic on dense prompts (about 10k cards
1323
+ # times 20k other spans on 1 MiB) and does not return inside the
1324
+ # customer budget. One binary search per span keeps the same overlap
1325
+ # rule: drop a span that overlaps a card unless the span strictly
1326
+ # encloses that card.
1327
+ cards.sort_by!(&:start)
1328
+ matches.reject! do |match|
1329
+ card_overlap_rejects?(cards, match)
1330
+ end
1331
+ matches.concat(cards)
1332
+ end
1333
+
1334
+ def card_overlap_rejects?(cards, match)
1335
+ lo = 0
1336
+ hi = cards.length
1337
+ while lo < hi
1338
+ mid = (lo + hi) / 2
1339
+ if cards[mid].start < match.finish
1340
+ lo = mid + 1
1341
+ else
1342
+ hi = mid
1343
+ end
1344
+ end
1345
+ index = lo - 1
1346
+ while index >= 0 && cards[index].finish > match.start
1347
+ card = cards[index]
1348
+ if match.start < card.finish && match.finish > card.start
1349
+ encloses = match.start <= card.start && match.finish >= card.finish &&
1350
+ (match.finish - match.start) > (card.finish - card.start)
1351
+ return true unless encloses
1352
+ end
1353
+ index -= 1
1354
+ end
1355
+ false
1356
+ end
1357
+
1358
+ # One alternation, not a downcase copy. The 50 KiB fixture misses every
1359
+ # branch. Mixed case is listed for the cues the byte scanner folds.
1360
+ CREDENTIAL_CUE_LITERALS = [
1361
+ "eyJ", "sk-", "pk-", "rk-", "ak-", "AKIA", "akia",
1362
+ "api key", "api_key", "api-key", "API key", "API_KEY", "API-KEY", "Api key",
1363
+ "secret", "Secret", "SECRET", "token", "Token", "TOKEN",
1364
+ "bearer", "Bearer", "BEARER",
1365
+ ].freeze
1366
+
1367
+ # Same literals as the alternation, via String#include?. The alternation
1368
+ # walked the whole 50 KiB fixture on every call and missed every branch.
1369
+ def short_credential_cue?(text)
1370
+ CREDENTIAL_CUE_LITERALS.any? { |literal| text.include?(literal) }
1371
+ end
1372
+
1373
+ CARD_SEPARATORS = [" ", "\t", "\n", "\r", "\r\n", "-", ".", "\u00a0", "\u2010", "\u2011", "\u2012", "\u2013", "\u2014", "\u2015", "\u2212", "\u00b7", "\u2027"].freeze
1374
+ CARD_SEPARATOR_BYTES = CARD_SEPARATORS.each_with_object({}) { |sep, lookup|
1375
+ lookup[sep.ord] = true if sep.bytesize == 1
1376
+ }.freeze
1377
+
1378
+ def card_ascii_word?(byte)
1379
+ (byte >= 48 && byte <= 57) || (byte >= 65 && byte <= 90) || (byte >= 97 && byte <= 122) || byte == 95
1380
+ end
1381
+
1382
+ def card_digit_runs(text)
1383
+ runs = []
1384
+ if text.ascii_only?
1385
+ n = text.bytesize
1386
+ i = 0
1387
+ while i < n
1388
+ b = text.getbyte(i)
1389
+ unless b >= 48 && b <= 57
1390
+ i += 1
1391
+ next
1392
+ end
1393
+ j = i + 1
1394
+ j += 1 while j < n && (c = text.getbyte(j)) >= 48 && c <= 57
1395
+ runs << [i, j, text.byteslice(i, j - i)]
1396
+ i = j
1397
+ end
1398
+ else
1399
+ text.scan(/[0-9]+/) do
1400
+ match = Regexp.last_match
1401
+ runs << [match.begin(0), match.end(0), match[0]]
1402
+ end
1403
+ end
1404
+ runs
1405
+ end
1406
+
1407
+ def card_matches(text)
1408
+ # Digit runs are collected with a linear scan. The old prefilter
1409
+ # /[0-9](?:[^0-9]{0,3}[0-9]){12}/ backtracks on dense digits, and
1410
+ # String#scan of the same runs cost seconds per mebibyte. ASCII
1411
+ # offsets match the character offsets scan would have produced.
1412
+ # \d(?:\D{0,3}\d){12} fails in linear time on prose that cannot hold a
1413
+ # PAN (the 50 KiB gate). A hit still uses the linear run scanner.
1414
+ return [] unless text.match?(/\d(?:\D{0,3}\d){12}/)
1415
+
1416
+ runs = card_digit_runs(text)
1417
+ found = []
1418
+ index = 0
1419
+ ascii = text.ascii_only?
1420
+ while index < runs.length
1421
+ start, ending, digits = runs[index]
1422
+ groups = [[start, ending, digits]]
1423
+ last = index
1424
+ while last + 1 < runs.length
1425
+ nxt_start, nxt_end, nxt_digits = runs[last + 1]
1426
+ gap = text[ending...nxt_start]
1427
+ break unless CARD_SEPARATORS.include?(gap)
1428
+
1429
+ groups << [nxt_start, nxt_end, nxt_digits]
1430
+ ending = nxt_end
1431
+ last += 1
1432
+ end
1433
+ if groups.map { |group| group[2].length } == [8, 4, 4, 4, 12]
1434
+ # UUID shape. Do not cut a 16-digit Luhn window out of it.
1435
+ index = last + 1
1436
+ next
1437
+ end
1438
+ # A chain shorter than the shortest PAN cannot contain one.
1439
+ digit_count = 0
1440
+ groups.each { |group| digit_count += group[2].length }
1441
+ if digit_count < 13
1442
+ index = last + 1
1443
+ next
1444
+ end
1445
+ card_chain_segments(text, groups, ascii).each do |seg_start, seg_end|
1446
+ found << PiiMatch.new(type: :ccnum, start: seg_start, finish: seg_end, value: text[seg_start...seg_end])
1447
+ end
1448
+ index = last + 1
1449
+ end
1450
+ found.concat(double_separator_cards(text, ascii)) if text_may_have_double_separator?(text)
1451
+ found
1452
+ end
1453
+
1454
+ def text_may_have_double_separator?(text)
1455
+ return true if text.include?(" ")
1456
+ # ". " is in ordinary sentences. Only a digit before the separator can
1457
+ # be the doubled gap the card rule counts.
1458
+ return true if text.match?(/[0-9][\t\n\r.\-] /)
1459
+ return false if text.ascii_only?
1460
+
1461
+ CARD_SEPARATORS.any? { |sep| sep != "\r\n" && sep != " " && text.include?("#{sep} ") }
1462
+ end
1463
+
1464
+ def double_separator_cards(text, ascii)
1465
+ return double_separator_cards_ascii(text) if ascii
1466
+
1467
+ found = []
1468
+ n = text.length
1469
+ i = 0
1470
+ while i < n
1471
+ unless digit_char?(text[i])
1472
+ i += 1
1473
+ next
1474
+ end
1475
+ start = i
1476
+ digits = +""
1477
+ doubles = 0
1478
+ finish = i
1479
+ j = i
1480
+ while j < n
1481
+ ch = text[j]
1482
+ if digit_char?(ch)
1483
+ digits << ch
1484
+ j += 1
1485
+ finish = j
1486
+ next
1487
+ end
1488
+ unless ch && CARD_SEPARATORS.include?(ch)
1489
+ break
1490
+ end
1491
+ after = j + 1
1492
+ if after < n && digit_char?(text[after])
1493
+ j = after
1494
+ next
1495
+ end
1496
+ if after < n && text[after] == " " && after + 1 < n && digit_char?(text[after + 1])
1497
+ doubles += 1
1498
+ j = after + 1
1499
+ next
1500
+ end
1501
+ break
1502
+ end
1503
+ if doubles == 1 && digits.length.between?(13, 19) && supported_card_network?(digits) && luhn?(digits) &&
1504
+ card_span_bounded?(text, start, finish, ascii)
1505
+ found << PiiMatch.new(type: :ccnum, start: start, finish: finish, value: text[start...finish])
1506
+ end
1507
+ i = j > i ? j : i + 1
1508
+ end
1509
+ found
1510
+ end
1511
+
1512
+ def digit_char?(ch)
1513
+ !ch.nil? && ch.bytesize == 1 && (b = ch.ord) >= 48 && b <= 57
1514
+ end
1515
+
1516
+ # The 50 KiB prose gate includes ". " in almost every sentence. A
1517
+ # per-character Regexp#match? walk blew the 0.050 s median. Byte scans
1518
+ # keep the same span rule: exactly one (separator, space) inside a
1519
+ # 13-19 digit candidate.
1520
+ def double_separator_cards_ascii(text)
1521
+ found = []
1522
+ n = text.bytesize
1523
+ sep = CARD_SEPARATOR_BYTES
1524
+ i = 0
1525
+ while i < n
1526
+ b = text.getbyte(i)
1527
+ unless b >= 48 && b <= 57
1528
+ i += 1
1529
+ next
1530
+ end
1531
+ start = i
1532
+ digit_count = 0
1533
+ doubles = 0
1534
+ finish = i
1535
+ j = i
1536
+ while j < n
1537
+ b = text.getbyte(j)
1538
+ if b >= 48 && b <= 57
1539
+ digit_count += 1
1540
+ j += 1
1541
+ finish = j
1542
+ next
1543
+ end
1544
+ unless sep[b]
1545
+ break
1546
+ end
1547
+ nxt = j + 1 < n ? text.getbyte(j + 1) : nil
1548
+ if nxt && nxt >= 48 && nxt <= 57
1549
+ j += 1
1550
+ next
1551
+ end
1552
+ if nxt == 32 && j + 2 < n && (c = text.getbyte(j + 2)) >= 48 && c <= 57
1553
+ doubles += 1
1554
+ j += 2
1555
+ next
1556
+ end
1557
+ break
1558
+ end
1559
+ if doubles == 1 && digit_count.between?(13, 19)
1560
+ digits = +""
1561
+ k = start
1562
+ while k < finish
1563
+ b = text.getbyte(k)
1564
+ digits << b.chr if b >= 48 && b <= 57
1565
+ k += 1
1566
+ end
1567
+ if supported_card_network?(digits) && luhn?(digits) &&
1568
+ card_span_bounded?(text, start, finish, true)
1569
+ found << PiiMatch.new(type: :ccnum, start: start, finish: finish, value: text.byteslice(start, finish - start))
1570
+ end
1571
+ end
1572
+ i = j > i ? j : i + 1
1573
+ end
1574
+ found
1575
+ end
1576
+
1577
+ def card_span_bounded?(text, start, finish, ascii)
1578
+ if ascii
1579
+ left = start.zero? || !card_ascii_word?(text.getbyte(start - 1))
1580
+ right = finish == text.length || !card_ascii_word?(text.getbyte(finish))
1581
+ return left && right
1582
+ end
1583
+ left = start.zero? ? nil : text[start - 1]
1584
+ right = finish == text.length ? nil : text[finish]
1585
+ (left.nil? || !left.match?(/[\p{L}\p{M}\p{N}_]/u)) &&
1586
+ (right.nil? || !right.match?(/[\p{L}\p{M}\p{N}_]/u))
1587
+ end
1588
+
1589
+ # Canonical grouping, or the whole separator chain when that chain does
1590
+ # not already contain a shorter canonical PAN. Single-digit slices are not.
1591
+ def card_window_eligible?(groups, from, to_inclusive)
1592
+ return true if canonical_window?(groups, from, to_inclusive)
1593
+ return false unless from.zero? && to_inclusive == groups.length - 1
1594
+
1595
+ !chain_has_canonical_pan?(groups)
1596
+ end
1597
+
1598
+ def canonical_window?(groups, from, to_inclusive)
1599
+ lengths = (from..to_inclusive).map { |index| groups[index][2].length }
1600
+ case lengths.length
1601
+ when 1
1602
+ lengths[0].between?(13, 19)
1603
+ when 2
1604
+ lengths == [6, 10]
1605
+ when 3
1606
+ lengths == [4, 6, 5]
1607
+ when 4
1608
+ lengths == [4, 4, 4, 4] || lengths == [4, 4, 4, 3]
1609
+ when 5
1610
+ lengths == [4, 4, 4, 4, 3]
1611
+ else
1612
+ false
1613
+ end
1614
+ end
1615
+
1616
+ def chain_has_canonical_pan?(groups)
1617
+ count = groups.length
1618
+ count.times do |i|
1619
+ digits = +""
1620
+ i.upto(count - 1) do |j|
1621
+ digits << groups[j][2]
1622
+ break if digits.length > 19
1623
+ next unless digits.length >= 13 && canonical_window?(groups, i, j)
1624
+ return true if supported_card_network?(digits) && luhn?(digits)
1625
+ end
1626
+ end
1627
+ false
1628
+ end
1629
+
1630
+ # groups: [start, finish, digits]. Returns [start, finish] spans.
1631
+ def card_chain_segments(text, groups, ascii)
1632
+ count = groups.length
1633
+ suffix = Array.new(count + 1, 0)
1634
+ take_to = Array.new(count, -1)
1635
+ (count - 1).downto(0) do |i|
1636
+ best = suffix[i + 1]
1637
+ best_to = -1
1638
+ digits = +""
1639
+ i.upto(count - 1) do |j|
1640
+ digits << groups[j][2]
1641
+ length = digits.length
1642
+ break if length > 19
1643
+ next unless length >= 13 && card_window_eligible?(groups, i, j) && supported_card_network?(digits) && luhn?(digits)
1644
+ next unless card_span_bounded?(text, groups[i][0], groups[j][1], ascii)
1645
+
1646
+ score = length + suffix[j + 1]
1647
+ if score > best
1648
+ best = score
1649
+ best_to = j + 1
1650
+ end
1651
+ end
1652
+ suffix[i] = best
1653
+ take_to[i] = best_to
1654
+ end
1655
+ segments = []
1656
+ i = 0
1657
+ while i < count
1658
+ to = take_to[i]
1659
+ if to < 0
1660
+ i += 1
1661
+ next
1662
+ end
1663
+ segments << [groups[i][0], groups[to - 1][1]]
1664
+ i = to
1665
+ end
1666
+ segments
1667
+ end
1668
+
1669
+ def supported_card_network?(digits)
1670
+ length = digits.length
1671
+ return false if length < 2
1672
+
1673
+ first2 = digits[0, 2].to_i
1674
+ first3 = digits[0, 3].to_i
1675
+ first4 = digits[0, 4].to_i
1676
+ first6 = digits[0, 6].to_i
1677
+ return true if digits.start_with?("4") && [13, 16, 19].include?(length)
1678
+ return true if length == 16 && ((51..55).cover?(first2) || (2221..2720).cover?(first4))
1679
+ return true if length == 15 && ["34", "37"].include?(digits[0, 2])
1680
+ return true if length == 14 && ((300..305).cover?(first3) || [36, 38, 39].include?(first2))
1681
+ return true if [16, 19].include?(length) && (digits.start_with?("6011") || digits.start_with?("65"))
1682
+ return true if length == 16 && ((644..649).cover?(first3) || (622126..622925).cover?(first6))
1683
+ return true if (16..19).cover?(length) && (3528..3589).cover?(first4)
1684
+
1685
+ (16..19).cover?(length) && digits.start_with?("62")
1686
+ end
1687
+
1688
+ def luhn?(digits)
1689
+ sum = 0
1690
+ parity = digits.length % 2
1691
+ digits.each_char.with_index do |ch, index|
1692
+ n = ch.ord - 48
1693
+ if index % 2 == parity
1694
+ n *= 2
1695
+ n -= 9 if n > 9
1696
+ end
1697
+ sum += n
1698
+ end
1699
+ (sum % 10).zero?
1700
+ end
1701
+
1702
+ def scan_iban(bytes, digit_positions, out, ascii_offsets: false)
1703
+ overlong = false
1704
+ index = 0
1705
+ while index < digit_positions.length
1706
+ control_end = digit_positions[index]
1707
+ i = control_end - 3
1708
+ unless i >= 0 && letter?(bytes.getbyte(i)) && letter?(bytes.getbyte(i + 1)) &&
1709
+ digit?(bytes.getbyte(i + 2)) && digit?(bytes.getbyte(i + 3))
1710
+ index += 1
1711
+ next
1712
+ end
1713
+ finish = i
1714
+ compact = +""
1715
+ separator_run = 0
1716
+ while finish < bytes.bytesize && finish - i < 68
1717
+ b = bytes.getbyte(finish)
1718
+ if letter?(b) || digit?(b)
1719
+ compact << b
1720
+ separator_run = 0
1721
+ elsif b == 32 && !compact.empty?
1722
+ separator_run += 1
1723
+ break if separator_run > 1
1724
+ else
1725
+ break
1726
+ end
1727
+ finish += 1
1728
+ end
1729
+ if compact.bytesize.between?(15, 34) && iban_mod97(compact) == 1
1730
+ last = finish
1731
+ last -= 1 while last > i && bytes.getbyte(last - 1) == 32
1732
+ out << byte_match(bytes, :iban, i, last, ascii_offsets: ascii_offsets)
1733
+ elsif compact.bytesize > 34
1734
+ overlong = true
1735
+ end
1736
+ index += 1
1737
+ index += 1 while index < digit_positions.length && digit_positions[index] < finish
1738
+ end
1739
+ overlong
1740
+ end
1741
+
1742
+ def letter?(b)
1743
+ b && (b.between?(65, 90) || b.between?(97, 122))
1744
+ end
1745
+
1746
+ def iban_mod97(compact)
1747
+ moved = compact.byteslice(4..) + compact.byteslice(0, 4)
1748
+ remainder = 0
1749
+ moved.each_byte do |b|
1750
+ if digit?(b)
1751
+ remainder = (remainder * 10 + b - 48) % 97
1752
+ else
1753
+ value = b >= 97 ? b - 87 : b - 55
1754
+ remainder = (remainder * 100 + value) % 97
1755
+ end
1756
+ end
1757
+ remainder
1758
+ end
1759
+
1760
+ def scan_credentials(bytes, lower, out, ascii_offsets: false)
1761
+ overlong = false
1762
+ ["sk-live-", "sk-test-", "sk-prod-", "sk-proj-", "pk-live-", "rk-live-", "ak-live-"].each do |prefix|
1763
+ offset = 0
1764
+ while (at = bytes.index(prefix, offset))
1765
+ finish = at + prefix.bytesize
1766
+ finish += 1 while finish < bytes.bytesize && credential_byte?(bytes.getbyte(finish))
1767
+ overlong ||= finish - at > MAX_SPAN_BYTES
1768
+ out << byte_match(bytes, :api_key, at, finish, ascii_offsets: ascii_offsets) if finish - at > prefix.bytesize
1769
+ offset = [finish, at + 1].max
1770
+ end
1771
+ end
1772
+ overlong ||= scan_labeled_token(bytes, lower, out, ascii_offsets: ascii_offsets)
1773
+ overlong ||= scan_jwt(bytes, out, ascii_offsets: ascii_offsets)
1774
+ overlong
1775
+ end
1776
+
1777
+ def credential_byte?(b)
1778
+ b && (letter?(b) || digit?(b) || b == 45 || b == 95)
1779
+ end
1780
+
1781
+ def scan_labeled_token(bytes, lower, out, ascii_offsets: false)
1782
+ overlong = false
1783
+ ["api key", "api_key", "secret", "token", "bearer"].each do |cue|
1784
+ offset = 0
1785
+ while (at = lower.index(cue.b, offset))
1786
+ p = at + cue.bytesize
1787
+ p += 1 while p < bytes.bytesize && [32, 58, 61, 34, 39].include?(bytes.getbyte(p))
1788
+ finish = p
1789
+ finish += 1 while finish < bytes.bytesize && credential_byte?(bytes.getbyte(finish))
1790
+ overlong ||= finish - p > MAX_SPAN_BYTES
1791
+ out << byte_match(bytes, :api_key, at, finish, ascii_offsets: ascii_offsets) if finish - p >= 16
1792
+ offset = [finish, at + 1].max
1793
+ end
1794
+ end
1795
+ overlong
1796
+ end
1797
+
1798
+ def scan_jwt(bytes, out, ascii_offsets: false)
1799
+ overlong = false
1800
+ offset = 0
1801
+ while (at = bytes.index("eyJ", offset))
1802
+ finish = at
1803
+ dots = 0
1804
+ while finish < bytes.bytesize && (credential_byte?(bytes.getbyte(finish)) || bytes.getbyte(finish) == 46)
1805
+ dots += 1 if bytes.getbyte(finish) == 46
1806
+ finish += 1
1807
+ end
1808
+ overlong ||= finish - at > MAX_SPAN_BYTES
1809
+ out << byte_match(bytes, :auth_jwt, at, finish, ascii_offsets: ascii_offsets) if dots == 2 && finish - at > 20
1810
+ offset = [finish, at + 1].max
1811
+ end
1812
+ overlong
1813
+ end
1814
+
1815
+ def scan_labeled_ids(bytes, lower, out, ascii_offsets: false)
1816
+ overlong = false
1817
+ NATIONAL_ID_CUES.each do |cue|
1818
+ offset = 0
1819
+ while (at = lower.index(cue.b, offset))
1820
+ if at.positive? && id_word_byte?(bytes.getbyte(at - 1))
1821
+ offset = at + cue.bytesize
1822
+ next
1823
+ end
1824
+ p = at + cue.bytesize
1825
+ p += 1 while p < bytes.bytesize && !digit?(bytes.getbyte(p)) && p - at < 64
1826
+ if p - at >= 64 && !digit?(bytes.getbyte(p))
1827
+ overlong = true
1828
+ offset = p
1829
+ next
1830
+ end
1831
+ finish = p
1832
+ while finish < bytes.bytesize && finish - at < 64
1833
+ if digit?(bytes.getbyte(finish))
1834
+ finish += 1
1835
+ elsif [32, 45, 46].include?(bytes.getbyte(finish)) &&
1836
+ digit?(bytes.getbyte(finish + 1))
1837
+ finish += 1
1838
+ else
1839
+ break
1840
+ end
1841
+ end
1842
+ next_byte = bytes.getbyte(finish)
1843
+ candidate_overlong = finish - at >= 64 && finish < bytes.bytesize &&
1844
+ (digit?(next_byte) ||
1845
+ ([32, 45, 46].include?(next_byte) && digit?(bytes.getbyte(finish + 1))))
1846
+ gap_end = finish
1847
+ gap_end += 1 while gap_end < bytes.bytesize && gap_end - finish < 64 &&
1848
+ [32, 45, 46].include?(bytes.getbyte(gap_end))
1849
+ overlong ||= candidate_overlong ||
1850
+ (gap_end > finish + 1 && digit?(bytes.getbyte(gap_end)))
1851
+ compact = bytes.byteslice(p...finish).delete("- .")
1852
+ out << byte_match(bytes, :nid_generic_labelled, at, finish, ascii_offsets: ascii_offsets) if !candidate_overlong && compact.bytesize.between?(5, 18)
1853
+ offset = [finish, at + 1].max
1854
+ end
1855
+ end
1856
+ overlong
1857
+ end
1858
+
1859
+ def scan_marked_dob(bytes, lower, out, ascii_offsets: false)
1860
+ overlong = false
1861
+ ["dob", "date of birth", "birth date", "born"].each do |cue|
1862
+ offset = 0
1863
+ while (at = lower.index(cue.b, offset))
1864
+ p = at + cue.bytesize
1865
+ p += 1 while p < bytes.bytesize && [32, 58, 35, 46, 44, 45].include?(bytes.getbyte(p)) && p - at < 64
1866
+ finish = p
1867
+ while finish < bytes.bytesize && finish - p < 32
1868
+ if digit?(bytes.getbyte(finish))
1869
+ finish += 1
1870
+ elsif [47, 45, 46, 32].include?(bytes.getbyte(finish)) && digit?(bytes.getbyte(finish + 1))
1871
+ finish += 1
1872
+ else
1873
+ break
1874
+ end
1875
+ end
1876
+ candidate_overlong = finish - p >= 32 && finish < bytes.bytesize &&
1877
+ (digit?(bytes.getbyte(finish)) ||
1878
+ ([47, 45, 46, 32].include?(bytes.getbyte(finish)) && digit?(bytes.getbyte(finish + 1))))
1879
+ overlong ||= candidate_overlong
1880
+ finish -= 1 while finish > p && [47, 45, 46, 32].include?(bytes.getbyte(finish - 1))
1881
+ digits = bytes.byteslice(p...finish).delete("- /.")
1882
+ date_text = bytes.byteslice(p...finish)
1883
+ out << byte_match(bytes, :date_of_birth, at, finish, ascii_offsets: ascii_offsets) if !candidate_overlong && valid_dob_date?(date_text, digits)
1884
+ offset = [finish, at + 1].max
1885
+ end
1886
+ end
1887
+ overlong
1888
+ end
1889
+
1890
+ def valid_dob_date?(date_text, digits)
1891
+ parts = date_text.split(/[\/\.\-\s]+/).reject(&:empty?)
1892
+ if parts.length == 3 && parts.all? { |part| part.match?(/\A\d{1,4}\z/) }
1893
+ a, b, c = parts.map(&:to_i)
1894
+ [[a, b, c], [a, c, b], [b, a, c], [b, c, a], [c, a, b], [c, b, a]].any? do |year, month, day|
1895
+ year.between?(1, 9999) && Date.valid_date?(year, month, day)
1896
+ end
1897
+ elsif digits.bytesize == 8 && digits.match?(/\A\d{8}\z/)
1898
+ year, month, day = digits[0, 4].to_i, digits[4, 2].to_i, digits[6, 2].to_i
1899
+ Date.valid_date?(year, month, day)
1900
+ elsif digits.bytesize == 6 && digits.match?(/\A\d{6}\z/)
1901
+ day, month, year = digits[0, 2].to_i, digits[2, 2].to_i, digits[4, 2].to_i
1902
+ [1900 + year, 2000 + year].any? { |full_year| Date.valid_date?(full_year, month, day) }
1903
+ else
1904
+ false
1905
+ end
1906
+ end
1907
+
1908
+ def scan_mrz(bytes, out, ascii_offsets: false)
1909
+ overlong = false
1910
+ line_start = 0
1911
+ while line_start < bytes.bytesize
1912
+ line_end = bytes.index("\n", line_start) || bytes.bytesize
1913
+ line = bytes.byteslice(line_start...line_end).delete_suffix("\r")
1914
+ if line.include?("<<") && line.bytesize > MRZ_MAX_BYTES
1915
+ overlong = true
1916
+ elsif line.bytesize.between?(30, MRZ_MAX_BYTES) && line.include?("<<") && line.each_byte.all? { |b| b == 60 || letter?(b) || digit?(b) }
1917
+ out << byte_match(bytes, :nid_mrz, line_start, line_start + line.bytesize, ascii_offsets: ascii_offsets)
1918
+ end
1919
+ line_start = line_end + 1
1920
+ end
1921
+ overlong
1922
+ end
1923
+ end
1924
+ end
1925
+ end