cloakapi 1.4.1-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +160 -0
- data/LICENSE +202 -0
- data/NOTICE +18 -0
- data/README.md +407 -0
- data/THIRD-PARTY-NOTICES.md +22 -0
- data/TRADEMARKS.md +19 -0
- data/examples/fake_gateway.rb +78 -0
- data/examples/hello.rb +64 -0
- data/lib/cloakapi/_lib/linux-x86_64/libcloak_engine.so +0 -0
- data/lib/cloakapi/client.rb +1272 -0
- data/lib/cloakapi/degraded.rb +77 -0
- data/lib/cloakapi/disclosure.rb +252 -0
- data/lib/cloakapi/engine.rb +702 -0
- data/lib/cloakapi/errors.rb +65 -0
- data/lib/cloakapi/pii/bundled_rules.rb +670 -0
- data/lib/cloakapi/pii/card_spans.rb +300 -0
- data/lib/cloakapi/pii/detector.rb +1925 -0
- data/lib/cloakapi/pii/patterns.json +2238 -0
- data/lib/cloakapi/pii/pii_match.rb +7 -0
- data/lib/cloakapi/pii/pii_type.rb +7 -0
- data/lib/cloakapi/pii/ruleset.rb +65 -0
- data/lib/cloakapi/pii/tokenizer.rb +411 -0
- data/lib/cloakapi/posture.rb +82 -0
- data/lib/cloakapi/receipts.rb +406 -0
- data/lib/cloakapi/resources/chat.rb +82 -0
- data/lib/cloakapi/resources/messages.rb +90 -0
- data/lib/cloakapi/resources/models.rb +25 -0
- data/lib/cloakapi/resources/receipts.rb +162 -0
- data/lib/cloakapi/resources/reconstruct.rb +120 -0
- data/lib/cloakapi/resources/request_strings.rb +485 -0
- data/lib/cloakapi/resources/reviewable.rb +163 -0
- data/lib/cloakapi/resources/tokens.rb +28 -0
- data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
- data/lib/cloakapi/routing.rb +500 -0
- data/lib/cloakapi/streaming.rb +196 -0
- data/lib/cloakapi/surrogate_mode.rb +233 -0
- data/lib/cloakapi/version.rb +15 -0
- data/lib/cloakapi/warnings.rb +137 -0
- data/lib/cloakapi.rb +37 -0
- data/licenses/Apache-2.0.txt +202 -0
- data/licenses/CC-BY-4.0.txt +104 -0
- data/licenses/MIT-Faker.txt +21 -0
- data/licenses/MIT.txt +21 -0
- metadata +150 -0
|
@@ -0,0 +1,670 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "ruleset"
|
|
4
|
+
require_relative "pii_match"
|
|
5
|
+
require_relative "detector"
|
|
6
|
+
require_relative "card_spans"
|
|
7
|
+
|
|
8
|
+
module CloakAPI
|
|
9
|
+
module Pii
|
|
10
|
+
# Every rule type of the bundled rule set, applied by the built-in rules lane.
|
|
11
|
+
#
|
|
12
|
+
# Owner decision of 2026-11-12 (DECISIONS.md, "JA B3 A"): the fallback of
|
|
13
|
+
# every SDK runs all rule types that come with it. The rules themselves are
|
|
14
|
+
# not changed here, only applied: each pattern of
|
|
15
|
+
# +lib/cloakapi/pii/patterns.json+ is compiled as written, a type that names
|
|
16
|
+
# a check digit is accepted only when the check passes, and the file's own
|
|
17
|
+
# +precedence+ list settles which type names a value that several types hit.
|
|
18
|
+
#
|
|
19
|
+
# "Applied as written" has a measure: the reference rules lane, the
|
|
20
|
+
# data-driven TypeScript runner (packages/pii-patterns-ts), which runs each
|
|
21
|
+
# pattern as a JavaScript regular expression without the Unicode flag.
|
|
22
|
+
# Ruby's engine reads four things differently, so {translate} spells them
|
|
23
|
+
# out; nothing else of a pattern is touched, and
|
|
24
|
+
# spec/pii/rules_lane_reference_spec.rb holds the result to the reference,
|
|
25
|
+
# type by type and span by span.
|
|
26
|
+
#
|
|
27
|
+
# 1. Word boundaries. In the reference \b is an ASCII boundary; in Ruby it
|
|
28
|
+
# follows Unicode letters, so a number directly after a Chinese or Thai
|
|
29
|
+
# word had no boundary. The pattern is compiled with (?a).
|
|
30
|
+
# 2. White space. In the reference \s includes the no-break space and the
|
|
31
|
+
# other Unicode spaces; in Ruby it is ASCII only. \s and \S are written
|
|
32
|
+
# out as that set.
|
|
33
|
+
# 3. Case. The reference lifts an inline (?i) or scoped (?i:...) to the
|
|
34
|
+
# whole pattern. So does this.
|
|
35
|
+
# 4. Case inside a character class. Ruby before 4.0 does not fold case
|
|
36
|
+
# for a non-ASCII letter in a class with other members ([øo] does not
|
|
37
|
+
# match "Ø"). The other case of each such letter is added to the class.
|
|
38
|
+
#
|
|
39
|
+
# Two types are not taken from their pattern, in the reference and here:
|
|
40
|
+
# EMAIL is found by a scan around each "@" ({email_spans}) and CREDIT_CARD
|
|
41
|
+
# by a card scanner ({CardSpans}). {reference} then settles the raw hits
|
|
42
|
+
# the way the reference does: a card wins over any other hit it touches
|
|
43
|
+
# (an API key or JWT is cut around it), and of the rest the earlier start,
|
|
44
|
+
# then the longer hit, then the earlier place in the rule set's precedence
|
|
45
|
+
# list is kept. sdk/conformance/rules-lane-vectors.json (1,170 texts) is the
|
|
46
|
+
# shared test of that; spec/pii/rules_lane_vectors_spec.rb runs it.
|
|
47
|
+
#
|
|
48
|
+
# This runs IN ADDITION to the detector the lane already had
|
|
49
|
+
# (+detector.rb+, untouched): its hits come first and keep their labels;
|
|
50
|
+
# the bundled types add what it does not report. A hit of a bundled type is
|
|
51
|
+
# labelled with that type's +cloud_token_type+ (+ID_GOV+, +CONTACT_PH+, ...),
|
|
52
|
+
# the vocabulary the engine uses.
|
|
53
|
+
#
|
|
54
|
+
# A rule run must never hang a send. Each pattern has a time limit and the
|
|
55
|
+
# whole pass has one; what could not be run in time is reported and the text
|
|
56
|
+
# goes on with what was found.
|
|
57
|
+
module BundledRules
|
|
58
|
+
# One compiled rule type.
|
|
59
|
+
Rule = Data.define(:id, :label, :regexp, :checksum, :rank)
|
|
60
|
+
|
|
61
|
+
# A hit, with the rule that produced it (for reports and tests).
|
|
62
|
+
Hit = Data.define(:rule, :start, :finish, :value)
|
|
63
|
+
|
|
64
|
+
# What one pass did.
|
|
65
|
+
# hits - Array of Hit, disjoint from nothing: the caller resolves overlaps
|
|
66
|
+
# skipped - ids of rule types that were not run to the end (time)
|
|
67
|
+
# seconds - wall time of the pass
|
|
68
|
+
Scan = Data.define(:hits, :skipped, :seconds)
|
|
69
|
+
|
|
70
|
+
# Time limit for one pattern on one text, and for all patterns together.
|
|
71
|
+
PATTERN_SECONDS = 0.25
|
|
72
|
+
PASS_SECONDS = 1.5
|
|
73
|
+
# A single hit longer than this is not a value (same bound the detector uses).
|
|
74
|
+
MAX_SPAN_CHARS = 65_536
|
|
75
|
+
|
|
76
|
+
# Above this size the rule set is not run at all: the detector's own
|
|
77
|
+
# large-input scanners handle such a text within their time limits, as
|
|
78
|
+
# before, and the run is reported as partial. (The reference stops
|
|
79
|
+
# running the patterns at 65,536 bytes; this lane goes four times as far.)
|
|
80
|
+
CEILING_BYTES = 262_144
|
|
81
|
+
CEILING_WARNING = "Detection was partial because the text is too large for every rule type to be run."
|
|
82
|
+
|
|
83
|
+
PassDeadline = Class.new(StandardError)
|
|
84
|
+
private_constant :PassDeadline
|
|
85
|
+
|
|
86
|
+
PARTIAL_WARNING = "Detection was partial because some rule types could not be run in the time allowed."
|
|
87
|
+
|
|
88
|
+
# The detector's rule: a run that takes two seconds or more is reported as
|
|
89
|
+
# reduced (Detector.wall_budget_hit?). The bundled pass gets what is left
|
|
90
|
+
# of that, within these bounds.
|
|
91
|
+
WALL_SECONDS = 2.0
|
|
92
|
+
MIN_PASS_SECONDS = 0.25
|
|
93
|
+
|
|
94
|
+
# What JavaScript's \s matches (ECMAScript WhiteSpace and LineTerminator).
|
|
95
|
+
JS_SPACE = "\\t\\n\\v\\f\\r \\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029\\u202F\\u205F\\u3000\\uFEFF"
|
|
96
|
+
|
|
97
|
+
# Rule types the reference finds with a scanner instead of their pattern.
|
|
98
|
+
SCANNER_OWNED = %w[EMAIL CREDIT_CARD].freeze
|
|
99
|
+
|
|
100
|
+
# A construct {translate} does not know how to carry over.
|
|
101
|
+
class UntranslatablePattern < StandardError; end
|
|
102
|
+
|
|
103
|
+
class << self
|
|
104
|
+
# Carry a pattern of the rule set over to Ruby: +[source, ignore_case]+.
|
|
105
|
+
# Only the four differences listed above are changed.
|
|
106
|
+
def translate(pattern)
|
|
107
|
+
ignore_case = false
|
|
108
|
+
out = +"(?a)"
|
|
109
|
+
chars = pattern.chars
|
|
110
|
+
i = 0
|
|
111
|
+
in_class = false
|
|
112
|
+
extras = nil
|
|
113
|
+
while i < chars.length
|
|
114
|
+
c = chars[i]
|
|
115
|
+
if c == "\\"
|
|
116
|
+
nxt = chars[i + 1]
|
|
117
|
+
if nxt == "s"
|
|
118
|
+
out << (in_class ? JS_SPACE : "[#{JS_SPACE}]")
|
|
119
|
+
i += 2
|
|
120
|
+
elsif nxt == "S"
|
|
121
|
+
raise UntranslatablePattern, "\\S inside a character class" if in_class
|
|
122
|
+
|
|
123
|
+
out << "[^#{JS_SPACE}]"
|
|
124
|
+
i += 2
|
|
125
|
+
elsif nxt == "u" && chars[i + 2, 4].to_a.join.match?(/\A\h{4}\z/)
|
|
126
|
+
extras&.concat(other_cases([chars[i + 2, 4].join.hex].pack("U")))
|
|
127
|
+
out << chars[i, 6].join
|
|
128
|
+
i += 6
|
|
129
|
+
else
|
|
130
|
+
out << c << nxt.to_s
|
|
131
|
+
i += 2
|
|
132
|
+
end
|
|
133
|
+
elsif in_class
|
|
134
|
+
if c == "]"
|
|
135
|
+
out << extras.uniq.join if ignore_case_ahead?(pattern) && extras
|
|
136
|
+
in_class = false
|
|
137
|
+
extras = nil
|
|
138
|
+
else
|
|
139
|
+
extras&.concat(other_cases(c))
|
|
140
|
+
end
|
|
141
|
+
# In the reference a [ inside a class is a literal; Ruby would open a nested class.
|
|
142
|
+
out << (c == "[" ? "\\[" : c)
|
|
143
|
+
i += 1
|
|
144
|
+
elsif c == "["
|
|
145
|
+
in_class = true
|
|
146
|
+
extras = []
|
|
147
|
+
out << c
|
|
148
|
+
i += 1
|
|
149
|
+
if chars[i] == "^"
|
|
150
|
+
out << "^"
|
|
151
|
+
i += 1
|
|
152
|
+
end
|
|
153
|
+
if chars[i] == "]" # a leading ] is a literal member
|
|
154
|
+
out << "\\]"
|
|
155
|
+
i += 1
|
|
156
|
+
end
|
|
157
|
+
elsif c == "(" && chars[i + 1] == "?" && (m = pattern[i..].match(/\A\(\?([a-zA-Z]*)(-[a-zA-Z]+)?([):])/)) && !(m[1].empty? && m[2].nil?)
|
|
158
|
+
raise UntranslatablePattern, "inline flags #{m[0]}" unless m[1] == "i" && m[2].nil?
|
|
159
|
+
|
|
160
|
+
ignore_case = true
|
|
161
|
+
out << "(?:" if m[3] == ":"
|
|
162
|
+
i += m[0].length
|
|
163
|
+
else
|
|
164
|
+
out << c
|
|
165
|
+
i += 1
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
[out, ignore_case]
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Run +block+ with the detector alone, as it was before the bundled
|
|
172
|
+
# types were applied. For measurements and for the specs that hold the
|
|
173
|
+
# detector's own time limits; nothing in the SDK calls it.
|
|
174
|
+
def without
|
|
175
|
+
previous = Thread.current[:cloakapi_bundled_rules_off]
|
|
176
|
+
Thread.current[:cloakapi_bundled_rules_off] = true
|
|
177
|
+
yield
|
|
178
|
+
ensure
|
|
179
|
+
Thread.current[:cloakapi_bundled_rules_off] = previous
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def off? = Thread.current[:cloakapi_bundled_rules_off] == true
|
|
183
|
+
|
|
184
|
+
def over_ceiling?(text) = text.is_a?(String) && text.bytesize > CEILING_BYTES
|
|
185
|
+
|
|
186
|
+
# The rule types that can be run: every type with a pattern. Types
|
|
187
|
+
# that need the name model (NAME, PLACE, ORGANIZATION) have none.
|
|
188
|
+
def rules
|
|
189
|
+
@rules ||= compile_all.first
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# {id => error message} for patterns Ruby's regular-expression engine
|
|
193
|
+
# does not accept. They are reported, not rewritten.
|
|
194
|
+
def compile_errors
|
|
195
|
+
compile_all unless @compile_errors
|
|
196
|
+
@compile_errors
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Run every rule type over +text+.
|
|
200
|
+
def scan(text, deadline_seconds: PASS_SECONDS)
|
|
201
|
+
started = now
|
|
202
|
+
hits = []
|
|
203
|
+
skipped = []
|
|
204
|
+
return Scan.new(hits: hits, skipped: skipped, seconds: 0.0) unless scannable?(text)
|
|
205
|
+
return Scan.new(hits: hits, skipped: rules.map(&:id), seconds: 0.0) if over_ceiling?(text)
|
|
206
|
+
|
|
207
|
+
rules.each do |rule|
|
|
208
|
+
if now - started >= deadline_seconds
|
|
209
|
+
skipped << rule.id
|
|
210
|
+
next
|
|
211
|
+
end
|
|
212
|
+
begin
|
|
213
|
+
if rule.id == "EMAIL"
|
|
214
|
+
email_spans(text).each do |start, finish|
|
|
215
|
+
hits << Hit.new(rule: rule, start: start, finish: finish, value: text[start...finish])
|
|
216
|
+
end
|
|
217
|
+
elsif rule.regexp
|
|
218
|
+
scan_rule(rule, text, hits, started + deadline_seconds)
|
|
219
|
+
end
|
|
220
|
+
rescue Regexp::TimeoutError, PassDeadline
|
|
221
|
+
skipped << rule.id
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
Scan.new(hits: hits, skipped: skipped, seconds: now - started)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# E-mail addresses as the reference finds them: for each "@", the local
|
|
228
|
+
# part is the longest run before it of ASCII letters, digits, . _ % + -
|
|
229
|
+
# and any letter or mark; the domain the longest run after it of
|
|
230
|
+
# letters, digits, marks, . and -, without trailing dots. Refused when
|
|
231
|
+
# the local part is empty, starts or ends with a dot or has two in a
|
|
232
|
+
# row, or the domain is empty, has no dot, has two dots in a row, a
|
|
233
|
+
# last label shorter than two, or a label that is empty or starts or
|
|
234
|
+
# ends with a hyphen.
|
|
235
|
+
def email_spans(text)
|
|
236
|
+
chars = text.chars
|
|
237
|
+
out = []
|
|
238
|
+
chars.each_with_index do |char, at|
|
|
239
|
+
next unless char == "@"
|
|
240
|
+
|
|
241
|
+
start = at
|
|
242
|
+
start -= 1 while start.positive? && email_atom?(chars[start - 1], true)
|
|
243
|
+
finish = at + 1
|
|
244
|
+
finish += 1 while finish < chars.length && email_atom?(chars[finish], false)
|
|
245
|
+
finish -= 1 while finish > at + 1 && chars[finish - 1] == "."
|
|
246
|
+
local = chars[start...at].join
|
|
247
|
+
domain = chars[(at + 1)...finish].join
|
|
248
|
+
next if local.empty? || local.start_with?(".") || local.end_with?(".") || local.include?("..") || domain.empty?
|
|
249
|
+
|
|
250
|
+
last_dot = domain.rindex(".")
|
|
251
|
+
next if last_dot.nil? || last_dot.zero? || domain.include?("..")
|
|
252
|
+
# The reference counts the last label in UTF-16 units.
|
|
253
|
+
next if domain[(last_dot + 1)..].each_char.sum { |c| c.ord > 0xFFFF ? 2 : 1 } < 2
|
|
254
|
+
next if domain.split(".", -1).any? { |label| label.empty? || label.start_with?("-") || label.end_with?("-") }
|
|
255
|
+
|
|
256
|
+
out << [start, finish]
|
|
257
|
+
end
|
|
258
|
+
out
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# The hits of the rule set on +text+, settled the way the reference
|
|
262
|
+
# settles them. This is what sdk/conformance/rules-lane-vectors.json
|
|
263
|
+
# states for each of its texts.
|
|
264
|
+
def reference(text, deadline_seconds: PASS_SECONDS)
|
|
265
|
+
raw = scan(text, deadline_seconds: deadline_seconds)
|
|
266
|
+
return raw unless scannable?(text) && !over_ceiling?(text)
|
|
267
|
+
|
|
268
|
+
card_rule = rules.find { |rule| rule.id == "CREDIT_CARD" }
|
|
269
|
+
cards = card_rule ? CardSpans.find(text).map { |start, finish| Hit.new(rule: card_rule, start: start, finish: finish, value: text[start...finish]) } : []
|
|
270
|
+
hits = cards.dup
|
|
271
|
+
raw.hits.each do |hit|
|
|
272
|
+
touching = cards.select { |card| card.start < hit.finish && card.finish > hit.start }
|
|
273
|
+
if touching.empty?
|
|
274
|
+
hits << hit
|
|
275
|
+
next
|
|
276
|
+
end
|
|
277
|
+
next unless %w[API_KEY AUTH_JWT].include?(hit.rule.id)
|
|
278
|
+
|
|
279
|
+
pieces = [[hit.start, hit.finish]]
|
|
280
|
+
touching.each do |card|
|
|
281
|
+
pieces = pieces.flat_map do |start, finish|
|
|
282
|
+
next [[start, finish]] if card.finish <= start || card.start >= finish
|
|
283
|
+
|
|
284
|
+
[[start, card.start], [card.finish, finish]].select { |from, to| to > from }
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
pieces.each { |start, finish| hits << Hit.new(rule: hit.rule, start: start, finish: finish, value: text[start...finish]) }
|
|
288
|
+
end
|
|
289
|
+
ordered = hits.sort_by { |hit| [hit.start, -(hit.finish - hit.start), hit.rule.rank] }
|
|
290
|
+
kept = []
|
|
291
|
+
last_end = 0
|
|
292
|
+
ordered.each do |hit|
|
|
293
|
+
next if hit.start < last_end
|
|
294
|
+
|
|
295
|
+
kept << hit
|
|
296
|
+
last_end = hit.finish
|
|
297
|
+
end
|
|
298
|
+
Scan.new(hits: kept, skipped: raw.skipped, seconds: raw.seconds)
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Hits of the bundled types that add to +existing+ (the detector's own
|
|
302
|
+
# matches), as PiiMatch values. A hit inside a span that is already
|
|
303
|
+
# reported adds nothing and is dropped, so existing labels and
|
|
304
|
+
# stand-ins stay as they were; a hit that reaches outside one is kept
|
|
305
|
+
# and the tokenizer replaces the union. Among bundled hits: earlier
|
|
306
|
+
# start, then longer, then earlier in the rule set's precedence list.
|
|
307
|
+
def additions(text, existing, deadline_seconds: PASS_SECONDS)
|
|
308
|
+
result = reference(text, deadline_seconds: deadline_seconds)
|
|
309
|
+
taken = existing.select { |m| m.start.is_a?(Integer) && m.finish.is_a?(Integer) && m.finish > m.start }
|
|
310
|
+
.map { |m| [m.start, m.finish] }.sort
|
|
311
|
+
ordered = result.hits.sort_by { |hit| [hit.start, -(hit.finish - hit.start), hit.rule.rank] }
|
|
312
|
+
added = []
|
|
313
|
+
kept = [] # bundled spans kept so far, ordered by start
|
|
314
|
+
ordered.each do |hit|
|
|
315
|
+
next if covered?(taken, hit.start, hit.finish)
|
|
316
|
+
next if kept.any? && kept.last[1] >= hit.finish && kept.last[0] <= hit.start
|
|
317
|
+
|
|
318
|
+
kept << [hit.start, hit.finish] if kept.empty? || hit.finish > kept.last[1]
|
|
319
|
+
added << PiiMatch.new(type: hit.rule.label.downcase.to_sym, start: hit.start, finish: hit.finish, value: hit.value)
|
|
320
|
+
end
|
|
321
|
+
[added, result]
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# Check-digit gate named by a rule type. An unknown name accepts (a
|
|
325
|
+
# typing error in the rule file must not start dropping real values).
|
|
326
|
+
def checksum_valid?(name, matched)
|
|
327
|
+
digits = matched.delete("^0-9")
|
|
328
|
+
alnum = matched.upcase.delete("^A-Z0-9")
|
|
329
|
+
case name
|
|
330
|
+
when "luhn" then digits.length >= 2 && luhn?(digits)
|
|
331
|
+
when "verhoeff" then !digits.empty? && validator(:valid_verhoeff?, digits)
|
|
332
|
+
when "mod97" then mod97?(matched)
|
|
333
|
+
when "nhs" then nhs?(digits)
|
|
334
|
+
when "bsn" then validator(:valid_bsn?, digits)
|
|
335
|
+
when "cpf" then digits.length == 11 && cpf?(digits)
|
|
336
|
+
when "no_org" then weighted_mod11?(digits, 9, [3, 2, 7, 6, 5, 4, 3, 2])
|
|
337
|
+
when "dk_cvr" then dk_cvr?(digits)
|
|
338
|
+
when "fi_ytunnus" then fi_ytunnus?(digits)
|
|
339
|
+
when "au_abn" then au_abn?(digits)
|
|
340
|
+
when "cn_ric" then alnum.length == 18 && validator(:valid_cn_ric?, alnum)
|
|
341
|
+
when "jp_mynumber" then validator(:valid_jp_mynumber?, digits)
|
|
342
|
+
when "be_nrn" then digits.length == 11 && validator(:valid_be_nrn?, digits)
|
|
343
|
+
when "cz_sk_rc" then digits.length == 10 && (digits.to_i % 11).zero?
|
|
344
|
+
when "mx_curp" then mx_curp?(alnum)
|
|
345
|
+
when "cl_rut" then validator(:valid_cl_rut?, alnum)
|
|
346
|
+
when "nid_bare10" then digits.length == 10 && (bg_egn?(digits) || se_pnr?(digits) || at_svnr?(digits))
|
|
347
|
+
when "nid_bare11" then digits.length == 11 && (validator(:valid_nid_bare11?, digits) || validator(:valid_be_nrn?, digits))
|
|
348
|
+
when "nid_bare12" then digits.length == 12 && (validator(:valid_jp_mynumber?, digits) || validator(:valid_verhoeff?, digits))
|
|
349
|
+
when "nid_bare13" then digits.length == 13 && validator(:valid_nid_bare13?, digits)
|
|
350
|
+
when "nid_bare16" then validator(:valid_id_nik?, digits)
|
|
351
|
+
else true
|
|
352
|
+
end
|
|
353
|
+
rescue StandardError
|
|
354
|
+
false
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
private
|
|
358
|
+
|
|
359
|
+
def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
360
|
+
|
|
361
|
+
def email_atom?(char, local)
|
|
362
|
+
if char.ascii_only?
|
|
363
|
+
return true if char.match?(/\A[A-Za-z0-9.-]\z/)
|
|
364
|
+
|
|
365
|
+
return local && ["_", "%", "+"].include?(char)
|
|
366
|
+
end
|
|
367
|
+
char.match?(/\A[\p{L}\p{M}]\z/)
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
def ignore_case_ahead?(pattern) = pattern.include?("(?i")
|
|
371
|
+
|
|
372
|
+
# The other-case forms of a non-ASCII letter (single characters only).
|
|
373
|
+
def other_cases(char)
|
|
374
|
+
return [] if char.ascii_only?
|
|
375
|
+
|
|
376
|
+
[char.upcase, char.downcase].select { |form| form != char && form.length == 1 }
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
def scannable?(text)
|
|
380
|
+
text.is_a?(String) && !text.empty? && text.valid_encoding? &&
|
|
381
|
+
(text.encoding == Encoding::UTF_8 || text.ascii_only?)
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
def compile_all
|
|
385
|
+
data = Ruleset.data
|
|
386
|
+
order = Array(data["precedence"])
|
|
387
|
+
errors = {}
|
|
388
|
+
compiled = []
|
|
389
|
+
data.fetch("types").each_with_index do |(id, entry), index|
|
|
390
|
+
source = entry["pattern"]
|
|
391
|
+
next if source.nil? || source.to_s.empty?
|
|
392
|
+
|
|
393
|
+
regexp = nil
|
|
394
|
+
unless SCANNER_OWNED.include?(id)
|
|
395
|
+
begin
|
|
396
|
+
translated, ignore_case = translate(source)
|
|
397
|
+
regexp = Regexp.new(translated, ignore_case ? Regexp::IGNORECASE : 0, timeout: PATTERN_SECONDS)
|
|
398
|
+
rescue RegexpError, UntranslatablePattern => e
|
|
399
|
+
errors[id] = e.message
|
|
400
|
+
next
|
|
401
|
+
end
|
|
402
|
+
end
|
|
403
|
+
label = entry["cloud_token_type"].to_s
|
|
404
|
+
label = id.to_s if label.empty?
|
|
405
|
+
label = label.upcase.gsub(/[^A-Z_]/, "")
|
|
406
|
+
rank = order.index(id) || (order.length + index)
|
|
407
|
+
compiled << Rule.new(id: id, label: label, regexp: regexp, checksum: entry["checksum"], rank: rank)
|
|
408
|
+
end
|
|
409
|
+
@compile_errors = errors.freeze
|
|
410
|
+
[compiled.sort_by(&:rank).freeze, errors]
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
def scan_rule(rule, text, hits, deadline)
|
|
414
|
+
position = 0
|
|
415
|
+
length = text.length
|
|
416
|
+
while position <= length && (found = rule.regexp.match(text, position))
|
|
417
|
+
raise PassDeadline if now >= deadline
|
|
418
|
+
|
|
419
|
+
start = found.begin(0)
|
|
420
|
+
finish = found.end(0)
|
|
421
|
+
if finish <= start
|
|
422
|
+
position = start + 1
|
|
423
|
+
next
|
|
424
|
+
end
|
|
425
|
+
position = finish
|
|
426
|
+
next if finish - start > MAX_SPAN_CHARS
|
|
427
|
+
|
|
428
|
+
value = found[0]
|
|
429
|
+
next if rule.checksum && !checksum_valid?(rule.checksum, value)
|
|
430
|
+
|
|
431
|
+
hits << Hit.new(rule: rule, start: start, finish: finish, value: value)
|
|
432
|
+
end
|
|
433
|
+
end
|
|
434
|
+
|
|
435
|
+
# Is [start, finish) inside one span of +taken+ (sorted by start)?
|
|
436
|
+
def covered?(taken, start, finish)
|
|
437
|
+
return false if taken.empty?
|
|
438
|
+
|
|
439
|
+
index = taken.bsearch_index { |span| span[0] > start } || taken.length
|
|
440
|
+
# Spans can nest or overlap, so look back over the few that start at or before +start+.
|
|
441
|
+
look = index - 1
|
|
442
|
+
checked = 0
|
|
443
|
+
while look >= 0 && checked < 8
|
|
444
|
+
return true if taken[look][1] >= finish
|
|
445
|
+
|
|
446
|
+
look -= 1
|
|
447
|
+
checked += 1
|
|
448
|
+
end
|
|
449
|
+
false
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# The detector's own check-digit code, reused as it is.
|
|
453
|
+
def validator(name, value)
|
|
454
|
+
(@detector ||= Detector.new).send(name, value)
|
|
455
|
+
end
|
|
456
|
+
|
|
457
|
+
def luhn?(digits)
|
|
458
|
+
sum = 0
|
|
459
|
+
digits.reverse.each_char.with_index do |char, index|
|
|
460
|
+
value = char.ord - 48
|
|
461
|
+
if index.odd?
|
|
462
|
+
value *= 2
|
|
463
|
+
value -= 9 if value > 9
|
|
464
|
+
end
|
|
465
|
+
sum += value
|
|
466
|
+
end
|
|
467
|
+
(sum % 10).zero?
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
def cpf?(digits)
|
|
471
|
+
d = digits.each_char.map { |char| char.ord - 48 }
|
|
472
|
+
calc = lambda do |take, start_weight|
|
|
473
|
+
sum = 0
|
|
474
|
+
take.times { |i| sum += d[i] * (start_weight - i) }
|
|
475
|
+
((sum * 10) % 11) % 10
|
|
476
|
+
end
|
|
477
|
+
calc.call(9, 10) == d[9] && calc.call(10, 11) == d[10]
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
def weighted_mod11?(digits, length, weights)
|
|
481
|
+
return false unless digits.length == length
|
|
482
|
+
|
|
483
|
+
d = digits.each_char.map { |char| char.ord - 48 }
|
|
484
|
+
sum = 0
|
|
485
|
+
(length - 1).times { |i| sum += d[i] * weights[i] }
|
|
486
|
+
check = 11 - (sum % 11)
|
|
487
|
+
check = 0 if check == 11
|
|
488
|
+
check != 10 && check == d[length - 1]
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
# First code point of each run of ten decimal digits the reference folds to 0-9.
|
|
492
|
+
DIGIT_BASES = [0x0660, 0x06f0, 0x07c0, 0x0966, 0x09e6, 0x0a66, 0x0ae6, 0x0b66, 0x0be6, 0x0c66, 0x0ce6, 0x0d66,
|
|
493
|
+
0x0de6, 0x0e50, 0x0ed0, 0x0f20, 0x1040, 0x1090, 0x17e0, 0x1810, 0x1946, 0x19d0, 0x1a80, 0x1a90,
|
|
494
|
+
0x1b50, 0x1bb0, 0x1c40, 0x1c50, 0xa620, 0xa8d0, 0xa900, 0xa9d0, 0xa9f0, 0xaa50, 0xabf0, 0xff10,
|
|
495
|
+
0x104a0, 0x10d30, 0x11066, 0x110f0, 0x11136, 0x111d0, 0x116c0, 0x11730, 0x1d7ce, 0x1e950].freeze
|
|
496
|
+
|
|
497
|
+
# IBAN check as the reference does it: digits of other scripts and
|
|
498
|
+
# full-width letters are read as their ASCII forms, separators dropped.
|
|
499
|
+
def mod97?(matched)
|
|
500
|
+
alnum = matched.each_char.filter_map do |char|
|
|
501
|
+
code = char.ord
|
|
502
|
+
base = DIGIT_BASES.find { |start| code >= start && code < start + 10 }
|
|
503
|
+
folded = if base then (48 + code - base).chr
|
|
504
|
+
elsif code.between?(0xff01, 0xff5e) then (code - 0xfee0).chr
|
|
505
|
+
else char
|
|
506
|
+
end
|
|
507
|
+
folded.upcase if folded.match?(/\A[A-Za-z0-9]\z/)
|
|
508
|
+
end
|
|
509
|
+
return false if alnum.length < 5
|
|
510
|
+
|
|
511
|
+
remainder = 0
|
|
512
|
+
(alnum[4..] + alnum[0, 4]).each do |char|
|
|
513
|
+
remainder = if char.match?(/[0-9]/)
|
|
514
|
+
(remainder * 10 + char.to_i) % 97
|
|
515
|
+
else
|
|
516
|
+
(remainder * 100 + char.ord - 55) % 97
|
|
517
|
+
end
|
|
518
|
+
end
|
|
519
|
+
remainder == 1
|
|
520
|
+
end
|
|
521
|
+
|
|
522
|
+
def mx_curp?(alnum)
|
|
523
|
+
return false unless alnum.length == 18
|
|
524
|
+
|
|
525
|
+
dictionary = "0123456789ABCDEFGHIJKLMN\u00D1OPQRSTUVWXYZ"
|
|
526
|
+
sum = 0
|
|
527
|
+
17.times do |i|
|
|
528
|
+
position = dictionary.index(alnum[i])
|
|
529
|
+
return false if position.nil?
|
|
530
|
+
|
|
531
|
+
sum += position * (18 - i)
|
|
532
|
+
end
|
|
533
|
+
alnum[17].match?(/[0-9]/) && alnum[17].to_i == (10 - (sum % 10)) % 10
|
|
534
|
+
end
|
|
535
|
+
|
|
536
|
+
def day_month?(day, month) = day.between?(1, 31) && month.between?(1, 12)
|
|
537
|
+
|
|
538
|
+
def bg_egn?(digits)
|
|
539
|
+
d = digits.each_char.map(&:to_i)
|
|
540
|
+
month = d[2] * 10 + d[3]
|
|
541
|
+
month -= 20 if month.between?(21, 32)
|
|
542
|
+
month -= 40 if month.between?(41, 52)
|
|
543
|
+
return false unless day_month?(d[4] * 10 + d[5], month)
|
|
544
|
+
|
|
545
|
+
check = [2, 4, 8, 5, 10, 9, 7, 3, 6].each_with_index.sum { |weight, i| d[i] * weight } % 11
|
|
546
|
+
check = 0 if check == 10
|
|
547
|
+
check == d[9]
|
|
548
|
+
end
|
|
549
|
+
|
|
550
|
+
def se_pnr?(digits)
|
|
551
|
+
d = digits.each_char.map(&:to_i)
|
|
552
|
+
day = d[4] * 10 + d[5]
|
|
553
|
+
day -= 60 if day.between?(61, 91)
|
|
554
|
+
day_month?(day, d[2] * 10 + d[3]) && luhn?(digits)
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
def at_svnr?(digits)
|
|
558
|
+
d = digits.each_char.map(&:to_i)
|
|
559
|
+
return false unless day_month?(d[4] * 10 + d[5], d[6] * 10 + d[7])
|
|
560
|
+
|
|
561
|
+
check = [0, 1, 2, 4, 5, 6, 7, 8, 9].zip([3, 7, 9, 5, 8, 4, 2, 1, 6]).sum { |index, weight| d[index] * weight } % 11
|
|
562
|
+
check != 10 && check == d[3]
|
|
563
|
+
end
|
|
564
|
+
|
|
565
|
+
def nhs?(digits) = weighted_mod11?(digits, 10, [10, 9, 8, 7, 6, 5, 4, 3, 2])
|
|
566
|
+
|
|
567
|
+
def dk_cvr?(digits)
|
|
568
|
+
return false unless digits.length == 8
|
|
569
|
+
|
|
570
|
+
weights = [2, 7, 6, 5, 4, 3, 2, 1]
|
|
571
|
+
(digits.each_char.with_index.sum { |char, i| (char.ord - 48) * weights[i] } % 11).zero?
|
|
572
|
+
end
|
|
573
|
+
|
|
574
|
+
def fi_ytunnus?(digits)
|
|
575
|
+
return false unless digits.length == 8
|
|
576
|
+
|
|
577
|
+
d = digits.each_char.map { |char| char.ord - 48 }
|
|
578
|
+
weights = [7, 9, 10, 5, 8, 4, 2]
|
|
579
|
+
remainder = 7.times.sum { |i| d[i] * weights[i] } % 11
|
|
580
|
+
return false if remainder == 1
|
|
581
|
+
|
|
582
|
+
(remainder.zero? ? 0 : 11 - remainder) == d[7]
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
def au_abn?(digits)
|
|
586
|
+
return false unless digits.length == 11
|
|
587
|
+
|
|
588
|
+
weights = [10, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19]
|
|
589
|
+
sum = digits.each_char.with_index.sum { |char, i| ((char.ord - 48) - (i.zero? ? 1 : 0)) * weights[i] }
|
|
590
|
+
# Remainder with the sign of the dividend, as the engine computes it.
|
|
591
|
+
sum.remainder(89).zero?
|
|
592
|
+
end
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
# Adds the bundled rule types to what the detector reports, without
|
|
596
|
+
# changing the detector: its own passes run first, as before.
|
|
597
|
+
module DetectorExtension
|
|
598
|
+
def detect(text)
|
|
599
|
+
return super if @bundled_rules_running || BundledRules.off?
|
|
600
|
+
|
|
601
|
+
@bundled_rules_running = true
|
|
602
|
+
begin
|
|
603
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
604
|
+
matches = super
|
|
605
|
+
hit_before = Detector.wall_budget_hit?
|
|
606
|
+
return matches if BundledRules.over_ceiling?(text)
|
|
607
|
+
|
|
608
|
+
added, result = BundledRules.additions(text, matches, deadline_seconds: bundled_allowance(started))
|
|
609
|
+
# Rule types that were not finished make this a reduced run, like a run over the wall.
|
|
610
|
+
note_bundled_wall(started, hit_before || !result.skipped.empty?)
|
|
611
|
+
added.empty? ? matches : (matches + added).sort_by(&:start)
|
|
612
|
+
ensure
|
|
613
|
+
@bundled_rules_running = false
|
|
614
|
+
end
|
|
615
|
+
end
|
|
616
|
+
|
|
617
|
+
def detect_with_status(text, **options)
|
|
618
|
+
return super if @bundled_rules_running || BundledRules.off?
|
|
619
|
+
|
|
620
|
+
@bundled_rules_running = true
|
|
621
|
+
begin
|
|
622
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
623
|
+
detection = super
|
|
624
|
+
return detection unless text.is_a?(String)
|
|
625
|
+
if BundledRules.over_ceiling?(text)
|
|
626
|
+
warning = [detection.warning, BundledRules::CEILING_WARNING].compact.reject(&:empty?).uniq.join("; ")
|
|
627
|
+
return Detector::Detection.new(matches: detection.matches, degraded: true,
|
|
628
|
+
coverage: detection.coverage, warning: warning)
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
hit_before = Detector.wall_budget_hit?
|
|
632
|
+
added, result = BundledRules.additions(text, detection.matches, deadline_seconds: bundled_allowance(started))
|
|
633
|
+
over_wall = note_bundled_wall(started, hit_before || !result.skipped.empty?)
|
|
634
|
+
return detection if added.empty? && result.skipped.empty? && !over_wall
|
|
635
|
+
|
|
636
|
+
partial = !result.skipped.empty?
|
|
637
|
+
warning = detection.warning
|
|
638
|
+
if partial
|
|
639
|
+
warning = [warning, BundledRules::PARTIAL_WARNING].compact.reject(&:empty?).uniq.join("; ")
|
|
640
|
+
end
|
|
641
|
+
Detector::Detection.new(
|
|
642
|
+
matches: (detection.matches + added).sort_by(&:start),
|
|
643
|
+
degraded: detection.degraded || partial || over_wall, coverage: detection.coverage, warning: warning
|
|
644
|
+
)
|
|
645
|
+
ensure
|
|
646
|
+
@bundled_rules_running = false
|
|
647
|
+
end
|
|
648
|
+
end
|
|
649
|
+
|
|
650
|
+
private
|
|
651
|
+
|
|
652
|
+
# What is left of the two-second wall after the detector's own passes.
|
|
653
|
+
def bundled_allowance(started)
|
|
654
|
+
left = BundledRules::WALL_SECONDS - (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
|
|
655
|
+
left.clamp(BundledRules::MIN_PASS_SECONDS, BundledRules::PASS_SECONDS)
|
|
656
|
+
end
|
|
657
|
+
|
|
658
|
+
# Keep the detector's flag true to the whole run: over two seconds in
|
|
659
|
+
# total is a reduced run, whichever part took the time.
|
|
660
|
+
def note_bundled_wall(started, hit_before)
|
|
661
|
+
over = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) >= BundledRules::WALL_SECONDS
|
|
662
|
+
Detector.note_wall_budget_hit! if over || hit_before
|
|
663
|
+
over
|
|
664
|
+
end
|
|
665
|
+
end
|
|
666
|
+
end
|
|
667
|
+
|
|
668
|
+
Detector.prepend(BundledRules::DetectorExtension)
|
|
669
|
+
end
|
|
670
|
+
end
|