cloakapi 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +160 -0
  3. data/LICENSE +202 -0
  4. data/NOTICE +18 -0
  5. data/README.md +407 -0
  6. data/THIRD-PARTY-NOTICES.md +22 -0
  7. data/TRADEMARKS.md +19 -0
  8. data/examples/fake_gateway.rb +78 -0
  9. data/examples/hello.rb +64 -0
  10. data/lib/cloakapi/client.rb +1272 -0
  11. data/lib/cloakapi/degraded.rb +77 -0
  12. data/lib/cloakapi/disclosure.rb +252 -0
  13. data/lib/cloakapi/engine.rb +702 -0
  14. data/lib/cloakapi/errors.rb +65 -0
  15. data/lib/cloakapi/pii/bundled_rules.rb +670 -0
  16. data/lib/cloakapi/pii/card_spans.rb +300 -0
  17. data/lib/cloakapi/pii/detector.rb +1925 -0
  18. data/lib/cloakapi/pii/patterns.json +2238 -0
  19. data/lib/cloakapi/pii/pii_match.rb +7 -0
  20. data/lib/cloakapi/pii/pii_type.rb +7 -0
  21. data/lib/cloakapi/pii/ruleset.rb +65 -0
  22. data/lib/cloakapi/pii/tokenizer.rb +411 -0
  23. data/lib/cloakapi/posture.rb +82 -0
  24. data/lib/cloakapi/receipts.rb +406 -0
  25. data/lib/cloakapi/resources/chat.rb +82 -0
  26. data/lib/cloakapi/resources/messages.rb +90 -0
  27. data/lib/cloakapi/resources/models.rb +25 -0
  28. data/lib/cloakapi/resources/receipts.rb +162 -0
  29. data/lib/cloakapi/resources/reconstruct.rb +120 -0
  30. data/lib/cloakapi/resources/request_strings.rb +485 -0
  31. data/lib/cloakapi/resources/reviewable.rb +163 -0
  32. data/lib/cloakapi/resources/tokens.rb +28 -0
  33. data/lib/cloakapi/routing/bundled-catalogue.json +107 -0
  34. data/lib/cloakapi/routing.rb +500 -0
  35. data/lib/cloakapi/streaming.rb +196 -0
  36. data/lib/cloakapi/surrogate_mode.rb +233 -0
  37. data/lib/cloakapi/version.rb +15 -0
  38. data/lib/cloakapi/warnings.rb +137 -0
  39. data/lib/cloakapi.rb +37 -0
  40. data/licenses/Apache-2.0.txt +202 -0
  41. data/licenses/CC-BY-4.0.txt +104 -0
  42. data/licenses/MIT-Faker.txt +21 -0
  43. data/licenses/MIT.txt +21 -0
  44. metadata +149 -0
@@ -0,0 +1,670 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "ruleset"
4
+ require_relative "pii_match"
5
+ require_relative "detector"
6
+ require_relative "card_spans"
7
+
8
+ module CloakAPI
9
+ module Pii
10
+ # Every rule type of the bundled rule set, applied by the built-in rules lane.
11
+ #
12
+ # Owner decision of 2026-11-12 (DECISIONS.md, "JA B3 A"): the fallback of
13
+ # every SDK runs all rule types that come with it. The rules themselves are
14
+ # not changed here, only applied: each pattern of
15
+ # +lib/cloakapi/pii/patterns.json+ is compiled as written, a type that names
16
+ # a check digit is accepted only when the check passes, and the file's own
17
+ # +precedence+ list settles which type names a value that several types hit.
18
+ #
19
+ # "Applied as written" has a measure: the reference rules lane, the
20
+ # data-driven TypeScript runner (packages/pii-patterns-ts), which runs each
21
+ # pattern as a JavaScript regular expression without the Unicode flag.
22
+ # Ruby's engine reads four things differently, so {translate} spells them
23
+ # out; nothing else of a pattern is touched, and
24
+ # spec/pii/rules_lane_reference_spec.rb holds the result to the reference,
25
+ # type by type and span by span.
26
+ #
27
+ # 1. Word boundaries. In the reference \b is an ASCII boundary; in Ruby it
28
+ # follows Unicode letters, so a number directly after a Chinese or Thai
29
+ # word had no boundary. The pattern is compiled with (?a).
30
+ # 2. White space. In the reference \s includes the no-break space and the
31
+ # other Unicode spaces; in Ruby it is ASCII only. \s and \S are written
32
+ # out as that set.
33
+ # 3. Case. The reference lifts an inline (?i) or scoped (?i:...) to the
34
+ # whole pattern. So does this.
35
+ # 4. Case inside a character class. Ruby before 4.0 does not fold case
36
+ # for a non-ASCII letter in a class with other members ([øo] does not
37
+ # match "Ø"). The other case of each such letter is added to the class.
38
+ #
39
+ # Two types are not taken from their pattern, in the reference and here:
40
+ # EMAIL is found by a scan around each "@" ({email_spans}) and CREDIT_CARD
41
+ # by a card scanner ({CardSpans}). {reference} then settles the raw hits
42
+ # the way the reference does: a card wins over any other hit it touches
43
+ # (an API key or JWT is cut around it), and of the rest the earlier start,
44
+ # then the longer hit, then the earlier place in the rule set's precedence
45
+ # list is kept. sdk/conformance/rules-lane-vectors.json (1,170 texts) is the
46
+ # shared test of that; spec/pii/rules_lane_vectors_spec.rb runs it.
47
+ #
48
+ # This runs IN ADDITION to the detector the lane already had
49
+ # (+detector.rb+, untouched): its hits come first and keep their labels;
50
+ # the bundled types add what it does not report. A hit of a bundled type is
51
+ # labelled with that type's +cloud_token_type+ (+ID_GOV+, +CONTACT_PH+, ...),
52
+ # the vocabulary the engine uses.
53
+ #
54
+ # A rule run must never hang a send. Each pattern has a time limit and the
55
+ # whole pass has one; what could not be run in time is reported and the text
56
+ # goes on with what was found.
57
+ module BundledRules
58
+ # One compiled rule type.
59
+ Rule = Data.define(:id, :label, :regexp, :checksum, :rank)
60
+
61
+ # A hit, with the rule that produced it (for reports and tests).
62
+ Hit = Data.define(:rule, :start, :finish, :value)
63
+
64
+ # What one pass did.
65
+ # hits - Array of Hit, disjoint from nothing: the caller resolves overlaps
66
+ # skipped - ids of rule types that were not run to the end (time)
67
+ # seconds - wall time of the pass
68
+ Scan = Data.define(:hits, :skipped, :seconds)
69
+
70
+ # Time limit for one pattern on one text, and for all patterns together.
71
+ PATTERN_SECONDS = 0.25
72
+ PASS_SECONDS = 1.5
73
+ # A single hit longer than this is not a value (same bound the detector uses).
74
+ MAX_SPAN_CHARS = 65_536
75
+
76
+ # Above this size the rule set is not run at all: the detector's own
77
+ # large-input scanners handle such a text within their time limits, as
78
+ # before, and the run is reported as partial. (The reference stops
79
+ # running the patterns at 65,536 bytes; this lane goes four times as far.)
80
+ CEILING_BYTES = 262_144
81
+ CEILING_WARNING = "Detection was partial because the text is too large for every rule type to be run."
82
+
83
+ PassDeadline = Class.new(StandardError)
84
+ private_constant :PassDeadline
85
+
86
+ PARTIAL_WARNING = "Detection was partial because some rule types could not be run in the time allowed."
87
+
88
+ # The detector's rule: a run that takes two seconds or more is reported as
89
+ # reduced (Detector.wall_budget_hit?). The bundled pass gets what is left
90
+ # of that, within these bounds.
91
+ WALL_SECONDS = 2.0
92
+ MIN_PASS_SECONDS = 0.25
93
+
94
+ # What JavaScript's \s matches (ECMAScript WhiteSpace and LineTerminator).
95
+ JS_SPACE = "\\t\\n\\v\\f\\r \\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029\\u202F\\u205F\\u3000\\uFEFF"
96
+
97
+ # Rule types the reference finds with a scanner instead of their pattern.
98
+ SCANNER_OWNED = %w[EMAIL CREDIT_CARD].freeze
99
+
100
+ # A construct {translate} does not know how to carry over.
101
+ class UntranslatablePattern < StandardError; end
102
+
103
+ class << self
104
+ # Carry a pattern of the rule set over to Ruby: +[source, ignore_case]+.
105
+ # Only the four differences listed above are changed.
106
+ def translate(pattern)
107
+ ignore_case = false
108
+ out = +"(?a)"
109
+ chars = pattern.chars
110
+ i = 0
111
+ in_class = false
112
+ extras = nil
113
+ while i < chars.length
114
+ c = chars[i]
115
+ if c == "\\"
116
+ nxt = chars[i + 1]
117
+ if nxt == "s"
118
+ out << (in_class ? JS_SPACE : "[#{JS_SPACE}]")
119
+ i += 2
120
+ elsif nxt == "S"
121
+ raise UntranslatablePattern, "\\S inside a character class" if in_class
122
+
123
+ out << "[^#{JS_SPACE}]"
124
+ i += 2
125
+ elsif nxt == "u" && chars[i + 2, 4].to_a.join.match?(/\A\h{4}\z/)
126
+ extras&.concat(other_cases([chars[i + 2, 4].join.hex].pack("U")))
127
+ out << chars[i, 6].join
128
+ i += 6
129
+ else
130
+ out << c << nxt.to_s
131
+ i += 2
132
+ end
133
+ elsif in_class
134
+ if c == "]"
135
+ out << extras.uniq.join if ignore_case_ahead?(pattern) && extras
136
+ in_class = false
137
+ extras = nil
138
+ else
139
+ extras&.concat(other_cases(c))
140
+ end
141
+ # In the reference a [ inside a class is a literal; Ruby would open a nested class.
142
+ out << (c == "[" ? "\\[" : c)
143
+ i += 1
144
+ elsif c == "["
145
+ in_class = true
146
+ extras = []
147
+ out << c
148
+ i += 1
149
+ if chars[i] == "^"
150
+ out << "^"
151
+ i += 1
152
+ end
153
+ if chars[i] == "]" # a leading ] is a literal member
154
+ out << "\\]"
155
+ i += 1
156
+ end
157
+ elsif c == "(" && chars[i + 1] == "?" && (m = pattern[i..].match(/\A\(\?([a-zA-Z]*)(-[a-zA-Z]+)?([):])/)) && !(m[1].empty? && m[2].nil?)
158
+ raise UntranslatablePattern, "inline flags #{m[0]}" unless m[1] == "i" && m[2].nil?
159
+
160
+ ignore_case = true
161
+ out << "(?:" if m[3] == ":"
162
+ i += m[0].length
163
+ else
164
+ out << c
165
+ i += 1
166
+ end
167
+ end
168
+ [out, ignore_case]
169
+ end
170
+
171
+ # Run +block+ with the detector alone, as it was before the bundled
172
+ # types were applied. For measurements and for the specs that hold the
173
+ # detector's own time limits; nothing in the SDK calls it.
174
+ def without
175
+ previous = Thread.current[:cloakapi_bundled_rules_off]
176
+ Thread.current[:cloakapi_bundled_rules_off] = true
177
+ yield
178
+ ensure
179
+ Thread.current[:cloakapi_bundled_rules_off] = previous
180
+ end
181
+
182
+ def off? = Thread.current[:cloakapi_bundled_rules_off] == true
183
+
184
+ def over_ceiling?(text) = text.is_a?(String) && text.bytesize > CEILING_BYTES
185
+
186
+ # The rule types that can be run: every type with a pattern. Types
187
+ # that need the name model (NAME, PLACE, ORGANIZATION) have none.
188
+ def rules
189
+ @rules ||= compile_all.first
190
+ end
191
+
192
+ # {id => error message} for patterns Ruby's regular-expression engine
193
+ # does not accept. They are reported, not rewritten.
194
+ def compile_errors
195
+ compile_all unless @compile_errors
196
+ @compile_errors
197
+ end
198
+
199
+ # Run every rule type over +text+.
200
+ def scan(text, deadline_seconds: PASS_SECONDS)
201
+ started = now
202
+ hits = []
203
+ skipped = []
204
+ return Scan.new(hits: hits, skipped: skipped, seconds: 0.0) unless scannable?(text)
205
+ return Scan.new(hits: hits, skipped: rules.map(&:id), seconds: 0.0) if over_ceiling?(text)
206
+
207
+ rules.each do |rule|
208
+ if now - started >= deadline_seconds
209
+ skipped << rule.id
210
+ next
211
+ end
212
+ begin
213
+ if rule.id == "EMAIL"
214
+ email_spans(text).each do |start, finish|
215
+ hits << Hit.new(rule: rule, start: start, finish: finish, value: text[start...finish])
216
+ end
217
+ elsif rule.regexp
218
+ scan_rule(rule, text, hits, started + deadline_seconds)
219
+ end
220
+ rescue Regexp::TimeoutError, PassDeadline
221
+ skipped << rule.id
222
+ end
223
+ end
224
+ Scan.new(hits: hits, skipped: skipped, seconds: now - started)
225
+ end
226
+
227
+ # E-mail addresses as the reference finds them: for each "@", the local
228
+ # part is the longest run before it of ASCII letters, digits, . _ % + -
229
+ # and any letter or mark; the domain the longest run after it of
230
+ # letters, digits, marks, . and -, without trailing dots. Refused when
231
+ # the local part is empty, starts or ends with a dot or has two in a
232
+ # row, or the domain is empty, has no dot, has two dots in a row, a
233
+ # last label shorter than two, or a label that is empty or starts or
234
+ # ends with a hyphen.
235
+ def email_spans(text)
236
+ chars = text.chars
237
+ out = []
238
+ chars.each_with_index do |char, at|
239
+ next unless char == "@"
240
+
241
+ start = at
242
+ start -= 1 while start.positive? && email_atom?(chars[start - 1], true)
243
+ finish = at + 1
244
+ finish += 1 while finish < chars.length && email_atom?(chars[finish], false)
245
+ finish -= 1 while finish > at + 1 && chars[finish - 1] == "."
246
+ local = chars[start...at].join
247
+ domain = chars[(at + 1)...finish].join
248
+ next if local.empty? || local.start_with?(".") || local.end_with?(".") || local.include?("..") || domain.empty?
249
+
250
+ last_dot = domain.rindex(".")
251
+ next if last_dot.nil? || last_dot.zero? || domain.include?("..")
252
+ # The reference counts the last label in UTF-16 units.
253
+ next if domain[(last_dot + 1)..].each_char.sum { |c| c.ord > 0xFFFF ? 2 : 1 } < 2
254
+ next if domain.split(".", -1).any? { |label| label.empty? || label.start_with?("-") || label.end_with?("-") }
255
+
256
+ out << [start, finish]
257
+ end
258
+ out
259
+ end
260
+
261
+ # The hits of the rule set on +text+, settled the way the reference
262
+ # settles them. This is what sdk/conformance/rules-lane-vectors.json
263
+ # states for each of its texts.
264
+ def reference(text, deadline_seconds: PASS_SECONDS)
265
+ raw = scan(text, deadline_seconds: deadline_seconds)
266
+ return raw unless scannable?(text) && !over_ceiling?(text)
267
+
268
+ card_rule = rules.find { |rule| rule.id == "CREDIT_CARD" }
269
+ cards = card_rule ? CardSpans.find(text).map { |start, finish| Hit.new(rule: card_rule, start: start, finish: finish, value: text[start...finish]) } : []
270
+ hits = cards.dup
271
+ raw.hits.each do |hit|
272
+ touching = cards.select { |card| card.start < hit.finish && card.finish > hit.start }
273
+ if touching.empty?
274
+ hits << hit
275
+ next
276
+ end
277
+ next unless %w[API_KEY AUTH_JWT].include?(hit.rule.id)
278
+
279
+ pieces = [[hit.start, hit.finish]]
280
+ touching.each do |card|
281
+ pieces = pieces.flat_map do |start, finish|
282
+ next [[start, finish]] if card.finish <= start || card.start >= finish
283
+
284
+ [[start, card.start], [card.finish, finish]].select { |from, to| to > from }
285
+ end
286
+ end
287
+ pieces.each { |start, finish| hits << Hit.new(rule: hit.rule, start: start, finish: finish, value: text[start...finish]) }
288
+ end
289
+ ordered = hits.sort_by { |hit| [hit.start, -(hit.finish - hit.start), hit.rule.rank] }
290
+ kept = []
291
+ last_end = 0
292
+ ordered.each do |hit|
293
+ next if hit.start < last_end
294
+
295
+ kept << hit
296
+ last_end = hit.finish
297
+ end
298
+ Scan.new(hits: kept, skipped: raw.skipped, seconds: raw.seconds)
299
+ end
300
+
301
+ # Hits of the bundled types that add to +existing+ (the detector's own
302
+ # matches), as PiiMatch values. A hit inside a span that is already
303
+ # reported adds nothing and is dropped, so existing labels and
304
+ # stand-ins stay as they were; a hit that reaches outside one is kept
305
+ # and the tokenizer replaces the union. Among bundled hits: earlier
306
+ # start, then longer, then earlier in the rule set's precedence list.
307
+ def additions(text, existing, deadline_seconds: PASS_SECONDS)
308
+ result = reference(text, deadline_seconds: deadline_seconds)
309
+ taken = existing.select { |m| m.start.is_a?(Integer) && m.finish.is_a?(Integer) && m.finish > m.start }
310
+ .map { |m| [m.start, m.finish] }.sort
311
+ ordered = result.hits.sort_by { |hit| [hit.start, -(hit.finish - hit.start), hit.rule.rank] }
312
+ added = []
313
+ kept = [] # bundled spans kept so far, ordered by start
314
+ ordered.each do |hit|
315
+ next if covered?(taken, hit.start, hit.finish)
316
+ next if kept.any? && kept.last[1] >= hit.finish && kept.last[0] <= hit.start
317
+
318
+ kept << [hit.start, hit.finish] if kept.empty? || hit.finish > kept.last[1]
319
+ added << PiiMatch.new(type: hit.rule.label.downcase.to_sym, start: hit.start, finish: hit.finish, value: hit.value)
320
+ end
321
+ [added, result]
322
+ end
323
+
324
+ # Check-digit gate named by a rule type. An unknown name accepts (a
325
+ # typing error in the rule file must not start dropping real values).
326
+ def checksum_valid?(name, matched)
327
+ digits = matched.delete("^0-9")
328
+ alnum = matched.upcase.delete("^A-Z0-9")
329
+ case name
330
+ when "luhn" then digits.length >= 2 && luhn?(digits)
331
+ when "verhoeff" then !digits.empty? && validator(:valid_verhoeff?, digits)
332
+ when "mod97" then mod97?(matched)
333
+ when "nhs" then nhs?(digits)
334
+ when "bsn" then validator(:valid_bsn?, digits)
335
+ when "cpf" then digits.length == 11 && cpf?(digits)
336
+ when "no_org" then weighted_mod11?(digits, 9, [3, 2, 7, 6, 5, 4, 3, 2])
337
+ when "dk_cvr" then dk_cvr?(digits)
338
+ when "fi_ytunnus" then fi_ytunnus?(digits)
339
+ when "au_abn" then au_abn?(digits)
340
+ when "cn_ric" then alnum.length == 18 && validator(:valid_cn_ric?, alnum)
341
+ when "jp_mynumber" then validator(:valid_jp_mynumber?, digits)
342
+ when "be_nrn" then digits.length == 11 && validator(:valid_be_nrn?, digits)
343
+ when "cz_sk_rc" then digits.length == 10 && (digits.to_i % 11).zero?
344
+ when "mx_curp" then mx_curp?(alnum)
345
+ when "cl_rut" then validator(:valid_cl_rut?, alnum)
346
+ when "nid_bare10" then digits.length == 10 && (bg_egn?(digits) || se_pnr?(digits) || at_svnr?(digits))
347
+ when "nid_bare11" then digits.length == 11 && (validator(:valid_nid_bare11?, digits) || validator(:valid_be_nrn?, digits))
348
+ when "nid_bare12" then digits.length == 12 && (validator(:valid_jp_mynumber?, digits) || validator(:valid_verhoeff?, digits))
349
+ when "nid_bare13" then digits.length == 13 && validator(:valid_nid_bare13?, digits)
350
+ when "nid_bare16" then validator(:valid_id_nik?, digits)
351
+ else true
352
+ end
353
+ rescue StandardError
354
+ false
355
+ end
356
+
357
+ private
358
+
359
+ def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
360
+
361
+ def email_atom?(char, local)
362
+ if char.ascii_only?
363
+ return true if char.match?(/\A[A-Za-z0-9.-]\z/)
364
+
365
+ return local && ["_", "%", "+"].include?(char)
366
+ end
367
+ char.match?(/\A[\p{L}\p{M}]\z/)
368
+ end
369
+
370
+ def ignore_case_ahead?(pattern) = pattern.include?("(?i")
371
+
372
+ # The other-case forms of a non-ASCII letter (single characters only).
373
+ def other_cases(char)
374
+ return [] if char.ascii_only?
375
+
376
+ [char.upcase, char.downcase].select { |form| form != char && form.length == 1 }
377
+ end
378
+
379
+ def scannable?(text)
380
+ text.is_a?(String) && !text.empty? && text.valid_encoding? &&
381
+ (text.encoding == Encoding::UTF_8 || text.ascii_only?)
382
+ end
383
+
384
+ def compile_all
385
+ data = Ruleset.data
386
+ order = Array(data["precedence"])
387
+ errors = {}
388
+ compiled = []
389
+ data.fetch("types").each_with_index do |(id, entry), index|
390
+ source = entry["pattern"]
391
+ next if source.nil? || source.to_s.empty?
392
+
393
+ regexp = nil
394
+ unless SCANNER_OWNED.include?(id)
395
+ begin
396
+ translated, ignore_case = translate(source)
397
+ regexp = Regexp.new(translated, ignore_case ? Regexp::IGNORECASE : 0, timeout: PATTERN_SECONDS)
398
+ rescue RegexpError, UntranslatablePattern => e
399
+ errors[id] = e.message
400
+ next
401
+ end
402
+ end
403
+ label = entry["cloud_token_type"].to_s
404
+ label = id.to_s if label.empty?
405
+ label = label.upcase.gsub(/[^A-Z_]/, "")
406
+ rank = order.index(id) || (order.length + index)
407
+ compiled << Rule.new(id: id, label: label, regexp: regexp, checksum: entry["checksum"], rank: rank)
408
+ end
409
+ @compile_errors = errors.freeze
410
+ [compiled.sort_by(&:rank).freeze, errors]
411
+ end
412
+
413
+ def scan_rule(rule, text, hits, deadline)
414
+ position = 0
415
+ length = text.length
416
+ while position <= length && (found = rule.regexp.match(text, position))
417
+ raise PassDeadline if now >= deadline
418
+
419
+ start = found.begin(0)
420
+ finish = found.end(0)
421
+ if finish <= start
422
+ position = start + 1
423
+ next
424
+ end
425
+ position = finish
426
+ next if finish - start > MAX_SPAN_CHARS
427
+
428
+ value = found[0]
429
+ next if rule.checksum && !checksum_valid?(rule.checksum, value)
430
+
431
+ hits << Hit.new(rule: rule, start: start, finish: finish, value: value)
432
+ end
433
+ end
434
+
435
+ # Is [start, finish) inside one span of +taken+ (sorted by start)?
436
+ def covered?(taken, start, finish)
437
+ return false if taken.empty?
438
+
439
+ index = taken.bsearch_index { |span| span[0] > start } || taken.length
440
+ # Spans can nest or overlap, so look back over the few that start at or before +start+.
441
+ look = index - 1
442
+ checked = 0
443
+ while look >= 0 && checked < 8
444
+ return true if taken[look][1] >= finish
445
+
446
+ look -= 1
447
+ checked += 1
448
+ end
449
+ false
450
+ end
451
+
452
+ # The detector's own check-digit code, reused as it is.
453
+ def validator(name, value)
454
+ (@detector ||= Detector.new).send(name, value)
455
+ end
456
+
457
+ def luhn?(digits)
458
+ sum = 0
459
+ digits.reverse.each_char.with_index do |char, index|
460
+ value = char.ord - 48
461
+ if index.odd?
462
+ value *= 2
463
+ value -= 9 if value > 9
464
+ end
465
+ sum += value
466
+ end
467
+ (sum % 10).zero?
468
+ end
469
+
470
+ def cpf?(digits)
471
+ d = digits.each_char.map { |char| char.ord - 48 }
472
+ calc = lambda do |take, start_weight|
473
+ sum = 0
474
+ take.times { |i| sum += d[i] * (start_weight - i) }
475
+ ((sum * 10) % 11) % 10
476
+ end
477
+ calc.call(9, 10) == d[9] && calc.call(10, 11) == d[10]
478
+ end
479
+
480
+ def weighted_mod11?(digits, length, weights)
481
+ return false unless digits.length == length
482
+
483
+ d = digits.each_char.map { |char| char.ord - 48 }
484
+ sum = 0
485
+ (length - 1).times { |i| sum += d[i] * weights[i] }
486
+ check = 11 - (sum % 11)
487
+ check = 0 if check == 11
488
+ check != 10 && check == d[length - 1]
489
+ end
490
+
491
+ # First code point of each run of ten decimal digits the reference folds to 0-9.
492
+ DIGIT_BASES = [0x0660, 0x06f0, 0x07c0, 0x0966, 0x09e6, 0x0a66, 0x0ae6, 0x0b66, 0x0be6, 0x0c66, 0x0ce6, 0x0d66,
493
+ 0x0de6, 0x0e50, 0x0ed0, 0x0f20, 0x1040, 0x1090, 0x17e0, 0x1810, 0x1946, 0x19d0, 0x1a80, 0x1a90,
494
+ 0x1b50, 0x1bb0, 0x1c40, 0x1c50, 0xa620, 0xa8d0, 0xa900, 0xa9d0, 0xa9f0, 0xaa50, 0xabf0, 0xff10,
495
+ 0x104a0, 0x10d30, 0x11066, 0x110f0, 0x11136, 0x111d0, 0x116c0, 0x11730, 0x1d7ce, 0x1e950].freeze
496
+
497
+ # IBAN check as the reference does it: digits of other scripts and
498
+ # full-width letters are read as their ASCII forms, separators dropped.
499
+ def mod97?(matched)
500
+ alnum = matched.each_char.filter_map do |char|
501
+ code = char.ord
502
+ base = DIGIT_BASES.find { |start| code >= start && code < start + 10 }
503
+ folded = if base then (48 + code - base).chr
504
+ elsif code.between?(0xff01, 0xff5e) then (code - 0xfee0).chr
505
+ else char
506
+ end
507
+ folded.upcase if folded.match?(/\A[A-Za-z0-9]\z/)
508
+ end
509
+ return false if alnum.length < 5
510
+
511
+ remainder = 0
512
+ (alnum[4..] + alnum[0, 4]).each do |char|
513
+ remainder = if char.match?(/[0-9]/)
514
+ (remainder * 10 + char.to_i) % 97
515
+ else
516
+ (remainder * 100 + char.ord - 55) % 97
517
+ end
518
+ end
519
+ remainder == 1
520
+ end
521
+
522
+ def mx_curp?(alnum)
523
+ return false unless alnum.length == 18
524
+
525
+ dictionary = "0123456789ABCDEFGHIJKLMN\u00D1OPQRSTUVWXYZ"
526
+ sum = 0
527
+ 17.times do |i|
528
+ position = dictionary.index(alnum[i])
529
+ return false if position.nil?
530
+
531
+ sum += position * (18 - i)
532
+ end
533
+ alnum[17].match?(/[0-9]/) && alnum[17].to_i == (10 - (sum % 10)) % 10
534
+ end
535
+
536
+ def day_month?(day, month) = day.between?(1, 31) && month.between?(1, 12)
537
+
538
+ def bg_egn?(digits)
539
+ d = digits.each_char.map(&:to_i)
540
+ month = d[2] * 10 + d[3]
541
+ month -= 20 if month.between?(21, 32)
542
+ month -= 40 if month.between?(41, 52)
543
+ return false unless day_month?(d[4] * 10 + d[5], month)
544
+
545
+ check = [2, 4, 8, 5, 10, 9, 7, 3, 6].each_with_index.sum { |weight, i| d[i] * weight } % 11
546
+ check = 0 if check == 10
547
+ check == d[9]
548
+ end
549
+
550
+ def se_pnr?(digits)
551
+ d = digits.each_char.map(&:to_i)
552
+ day = d[4] * 10 + d[5]
553
+ day -= 60 if day.between?(61, 91)
554
+ day_month?(day, d[2] * 10 + d[3]) && luhn?(digits)
555
+ end
556
+
557
+ def at_svnr?(digits)
558
+ d = digits.each_char.map(&:to_i)
559
+ return false unless day_month?(d[4] * 10 + d[5], d[6] * 10 + d[7])
560
+
561
+ check = [0, 1, 2, 4, 5, 6, 7, 8, 9].zip([3, 7, 9, 5, 8, 4, 2, 1, 6]).sum { |index, weight| d[index] * weight } % 11
562
+ check != 10 && check == d[3]
563
+ end
564
+
565
+ def nhs?(digits) = weighted_mod11?(digits, 10, [10, 9, 8, 7, 6, 5, 4, 3, 2])
566
+
567
+ def dk_cvr?(digits)
568
+ return false unless digits.length == 8
569
+
570
+ weights = [2, 7, 6, 5, 4, 3, 2, 1]
571
+ (digits.each_char.with_index.sum { |char, i| (char.ord - 48) * weights[i] } % 11).zero?
572
+ end
573
+
574
+ def fi_ytunnus?(digits)
575
+ return false unless digits.length == 8
576
+
577
+ d = digits.each_char.map { |char| char.ord - 48 }
578
+ weights = [7, 9, 10, 5, 8, 4, 2]
579
+ remainder = 7.times.sum { |i| d[i] * weights[i] } % 11
580
+ return false if remainder == 1
581
+
582
+ (remainder.zero? ? 0 : 11 - remainder) == d[7]
583
+ end
584
+
585
+ def au_abn?(digits)
586
+ return false unless digits.length == 11
587
+
588
+ weights = [10, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19]
589
+ sum = digits.each_char.with_index.sum { |char, i| ((char.ord - 48) - (i.zero? ? 1 : 0)) * weights[i] }
590
+ # Remainder with the sign of the dividend, as the engine computes it.
591
+ sum.remainder(89).zero?
592
+ end
593
+ end
594
+
595
+ # Adds the bundled rule types to what the detector reports, without
596
+ # changing the detector: its own passes run first, as before.
597
+ module DetectorExtension
598
+ def detect(text)
599
+ return super if @bundled_rules_running || BundledRules.off?
600
+
601
+ @bundled_rules_running = true
602
+ begin
603
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
604
+ matches = super
605
+ hit_before = Detector.wall_budget_hit?
606
+ return matches if BundledRules.over_ceiling?(text)
607
+
608
+ added, result = BundledRules.additions(text, matches, deadline_seconds: bundled_allowance(started))
609
+ # Rule types that were not finished make this a reduced run, like a run over the wall.
610
+ note_bundled_wall(started, hit_before || !result.skipped.empty?)
611
+ added.empty? ? matches : (matches + added).sort_by(&:start)
612
+ ensure
613
+ @bundled_rules_running = false
614
+ end
615
+ end
616
+
617
+ def detect_with_status(text, **options)
618
+ return super if @bundled_rules_running || BundledRules.off?
619
+
620
+ @bundled_rules_running = true
621
+ begin
622
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
623
+ detection = super
624
+ return detection unless text.is_a?(String)
625
+ if BundledRules.over_ceiling?(text)
626
+ warning = [detection.warning, BundledRules::CEILING_WARNING].compact.reject(&:empty?).uniq.join("; ")
627
+ return Detector::Detection.new(matches: detection.matches, degraded: true,
628
+ coverage: detection.coverage, warning: warning)
629
+ end
630
+
631
+ hit_before = Detector.wall_budget_hit?
632
+ added, result = BundledRules.additions(text, detection.matches, deadline_seconds: bundled_allowance(started))
633
+ over_wall = note_bundled_wall(started, hit_before || !result.skipped.empty?)
634
+ return detection if added.empty? && result.skipped.empty? && !over_wall
635
+
636
+ partial = !result.skipped.empty?
637
+ warning = detection.warning
638
+ if partial
639
+ warning = [warning, BundledRules::PARTIAL_WARNING].compact.reject(&:empty?).uniq.join("; ")
640
+ end
641
+ Detector::Detection.new(
642
+ matches: (detection.matches + added).sort_by(&:start),
643
+ degraded: detection.degraded || partial || over_wall, coverage: detection.coverage, warning: warning
644
+ )
645
+ ensure
646
+ @bundled_rules_running = false
647
+ end
648
+ end
649
+
650
+ private
651
+
652
+ # What is left of the two-second wall after the detector's own passes.
653
+ def bundled_allowance(started)
654
+ left = BundledRules::WALL_SECONDS - (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
655
+ left.clamp(BundledRules::MIN_PASS_SECONDS, BundledRules::PASS_SECONDS)
656
+ end
657
+
658
+ # Keep the detector's flag true to the whole run: over two seconds in
659
+ # total is a reduced run, whichever part took the time.
660
+ def note_bundled_wall(started, hit_before)
661
+ over = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) >= BundledRules::WALL_SECONDS
662
+ Detector.note_wall_budget_hit! if over || hit_before
663
+ over
664
+ end
665
+ end
666
+ end
667
+
668
+ Detector.prepend(BundledRules::DetectorExtension)
669
+ end
670
+ end