disarm 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +104 -0
- data/ext/disarm/Cargo.toml +36 -0
- data/ext/disarm/extconf.rb +7 -0
- data/ext/disarm/src/lib.rs +959 -0
- data/lib/disarm/version.rb +6 -0
- data/lib/disarm.rb +618 -0
- metadata +144 -0
data/lib/disarm.rb
ADDED
|
@@ -0,0 +1,618 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "disarm/version"
|
|
4
|
+
|
|
5
|
+
# Load the native extension. Precompiled platform gems ship a per-minor-version
|
|
6
|
+
# subdir (e.g. lib/disarm/3.3/disarm.so); a source gem compiles to
|
|
7
|
+
# lib/disarm/disarm.so. Try the versioned path first, then fall back.
|
|
8
|
+
begin
|
|
9
|
+
ruby_minor = RUBY_VERSION[/\d+\.\d+/]
|
|
10
|
+
require_relative "disarm/#{ruby_minor}/disarm"
|
|
11
|
+
rescue LoadError => e
|
|
12
|
+
# Only fall back to the unversioned (source-gem) path when the versioned file
|
|
13
|
+
# is genuinely absent. A real load failure of an *existing* ext (e.g. a missing
|
|
14
|
+
# dependent shared library or an undefined symbol) must propagate, not be masked
|
|
15
|
+
# by the fallback.
|
|
16
|
+
raise unless e.message.include?("cannot load such file")
|
|
17
|
+
|
|
18
|
+
require_relative "disarm/disarm"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The native extension (ext/disarm) defines the raw `_`-prefixed shims and the
|
|
22
|
+
# already-idiomatic no-option methods (strip_accents, fold_case,
|
|
23
|
+
# suspicious_hostname?). This file adds the idiomatic Ruby surface on top (#357):
|
|
24
|
+
# keyword arguments with the core's defaults, symbol tokens (:latin, :default, …),
|
|
25
|
+
# a single transliterate(text, scheme:) entrypoint, and a Disarm::Error hierarchy.
|
|
26
|
+
# Each method is still a thin wrapper over the pure-Rust `disarm` core.
|
|
27
|
+
module Disarm
|
|
28
|
+
# Base class for every error disarm raises, so consumers can `rescue
|
|
29
|
+
# Disarm::Error`. The native shim raises Ruby's built-in ArgumentError /
|
|
30
|
+
# RuntimeError; the wrappers below translate those into this hierarchy.
|
|
31
|
+
class Error < StandardError; end
|
|
32
|
+
|
|
33
|
+
# Raised for an invalid argument — an unknown scheme/target token, a
|
|
34
|
+
# malformed option, etc. (the core's `ErrorKind::InvalidArgument`).
|
|
35
|
+
class InvalidArgument < Error; end
|
|
36
|
+
|
|
37
|
+
class << self
|
|
38
|
+
# Transliterate Unicode text to ASCII. `scheme:` selects the standard:
|
|
39
|
+
# :default (the general-purpose scheme), :strict_iso9, or :gost7034. `lang:`
|
|
40
|
+
# applies a language profile on top of the scheme (e.g. "uk" → Київ → "Kyiv",
|
|
41
|
+
# "de" → ü → "ue"); nil means no profile. Both accept a String or Symbol.
|
|
42
|
+
def transliterate(text, scheme: :default, lang: nil)
|
|
43
|
+
scheme = scheme.to_s
|
|
44
|
+
lang = lang&.to_s
|
|
45
|
+
translate_errors do
|
|
46
|
+
# The bare default with no profile keeps the core's borrow-on-no-op fast
|
|
47
|
+
# path; any scheme or lang takes the option-carrying builder path.
|
|
48
|
+
if lang.nil? && scheme == "default"
|
|
49
|
+
_transliterate(text)
|
|
50
|
+
else
|
|
51
|
+
_transliterate_opts(text, scheme, lang)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Fold cross-script confusables toward `target:` (:latin or :cyrillic).
|
|
57
|
+
#
|
|
58
|
+
# `digit_policy:` selects how non-Latin DIGITS fold (#561).
|
|
59
|
+
#
|
|
60
|
+
# `:numeric` (default) sends them to the ASCII digit — `०` becomes `0` — which is
|
|
61
|
+
# right for prose, where a Devanagari zero really is a zero.
|
|
62
|
+
#
|
|
63
|
+
# `:tr39` uses upstream's targets, which send most of them to a Latin letter
|
|
64
|
+
# (`०` → `o`; three of the 45 rows fold to `.` or to the two characters `rn`
|
|
65
|
+
# instead). That is what an identifier *skeleton* wants, since its only job is to
|
|
66
|
+
# make two confusable identifiers collide. The two differ on 45 rows and agree
|
|
67
|
+
# everywhere else. Scoped to `target: :latin` — the override rows are generated from
|
|
68
|
+
# the Latin table and carry TR39's Latin-script targets, so with `target: :cyrillic`
|
|
69
|
+
# it is a no-op.
|
|
70
|
+
#
|
|
71
|
+
# `:preserve` leaves the digit alone (#648). The other two both yield a mixed-script
|
|
72
|
+
# numeral — `२०२४` becomes `२0२४` or `२o२४` — so neither keeps the script. Unlike
|
|
73
|
+
# `:tr39` it applies under every target script.
|
|
74
|
+
def normalize_confusables(text, target: :latin, digit_policy: :numeric)
|
|
75
|
+
translate_errors { _normalize_confusables(text, target.to_s, digit_policy.to_s) }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Whether `text` contains a character confusable with `target:` (:latin or
|
|
79
|
+
# :cyrillic).
|
|
80
|
+
def confusable?(text, target: :latin)
|
|
81
|
+
translate_errors { _confusable?(text, target.to_s) }
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Generate a URL-safe slug. Mirrors the core's `SlugConfig` defaults; every
|
|
85
|
+
# option past `text` is keyword-only. (`regex_pattern`/`replacements` are not
|
|
86
|
+
# surfaced yet — see ext/disarm/src/lib.rs.)
|
|
87
|
+
def slugify(
|
|
88
|
+
text,
|
|
89
|
+
separator: "-",
|
|
90
|
+
lowercase: true,
|
|
91
|
+
max_length: 0,
|
|
92
|
+
word_boundary: false,
|
|
93
|
+
save_order: false,
|
|
94
|
+
stopwords: [],
|
|
95
|
+
allow_unicode: false,
|
|
96
|
+
lang: nil,
|
|
97
|
+
entities: true,
|
|
98
|
+
decimal: true,
|
|
99
|
+
hexadecimal: true,
|
|
100
|
+
safe_chars: ""
|
|
101
|
+
)
|
|
102
|
+
translate_errors do
|
|
103
|
+
# `Array(stopwords)` tolerates the common `stopwords: nil` (and a bare
|
|
104
|
+
# String) instead of raising NoMethodError on `.map`.
|
|
105
|
+
_slugify(
|
|
106
|
+
text, separator.to_s, lowercase, max_length, word_boundary, save_order,
|
|
107
|
+
Array(stopwords).map(&:to_s), allow_unicode, lang&.to_s, entities, decimal,
|
|
108
|
+
hexadecimal, safe_chars.to_s
|
|
109
|
+
)
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Replace emoji with their plain names (e.g. "👍" → "thumbs up").
|
|
114
|
+
# `strip_modifiers:` drops skin-tone / variation modifiers before naming.
|
|
115
|
+
def demojize(text, strip_modifiers: false)
|
|
116
|
+
translate_errors { _demojize(text, strip_modifiers) }
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Canonicalize, but raise rather than silently normalize a structural difference
|
|
120
|
+
# away — the half of the pair that lets a caller reject input instead of comparing
|
|
121
|
+
# a value the sender never wrote.
|
|
122
|
+
def canonicalize_strict(text)
|
|
123
|
+
translate_errors { _canonicalize_strict(text) }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Strip the non-interchange and invisible classes while KEEPING the script.
|
|
127
|
+
#
|
|
128
|
+
# Unlike `canonicalize` it folds no confusables, so non-Latin text survives as
|
|
129
|
+
# itself. It cannot be rebuilt from the seven universal `strip_*` methods, and the
|
|
130
|
+
# difference runs both ways: this preserves the Private Use Area (icon fonts) and
|
|
131
|
+
# keeps the VS15/VS16 presentation selectors after a base, which the naive chain
|
|
132
|
+
# deletes, and it collapses TAB/LF to a space, which the primitives leave alone.
|
|
133
|
+
def strip_format(text)
|
|
134
|
+
_strip_format(text)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Remove obfuscation (zero-width, bidi, combining-mark abuse) while keeping
|
|
138
|
+
# legible content.
|
|
139
|
+
def strip_obfuscation(text)
|
|
140
|
+
translate_errors { _strip_obfuscation(text) }
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Canonicalize text for security-sensitive comparison: strip obfuscation,
|
|
144
|
+
# control characters, and other spoofing vectors. The name describes the
|
|
145
|
+
# mechanism (Unicode canonicalization for matching), not a safety guarantee —
|
|
146
|
+
# this is not an output sanitizer; encode at the sink.
|
|
147
|
+
#
|
|
148
|
+
# Two steps introduce ASCII, not one (#719): the leading NFKC, and the confusable
|
|
149
|
+
# fold, which reaches characters NFKC leaves alone. U+2236 RATIO becomes ":", U+2044
|
|
150
|
+
# FRACTION SLASH becomes "/", U+2216 SET MINUS becomes "\\". A string that carried no
|
|
151
|
+
# delimiter can leave here carrying one. #inspect_anomalies reports it as :confusable
|
|
152
|
+
# WHEN the word also carries an ASCII letter, which is the gate that keeps ordinary
|
|
153
|
+
# non-Latin text from firing; a delimiter-only string is not reported.
|
|
154
|
+
def canonicalize(text)
|
|
155
|
+
translate_errors { _canonicalize(text) }
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# @deprecated Renamed to {#canonicalize} in 0.11 (the +_clean+ name
|
|
159
|
+
# overpromised safety); removed in 1.0.
|
|
160
|
+
def security_clean(text)
|
|
161
|
+
warn("[disarm] security_clean is deprecated; use canonicalize (removed in 1.0)", category: :deprecated)
|
|
162
|
+
canonicalize(text)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Case/accent/script-insensitive search lookup key. `lang:` applies a
|
|
166
|
+
# language profile for transliteration (e.g. "ru", "uk"); nil means none.
|
|
167
|
+
# Raises Disarm::InvalidArgument on an unknown lang.
|
|
168
|
+
def search_key(text, lang: nil)
|
|
169
|
+
translate_errors { _search_key(text, lang&.to_s) }
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Collation sort key (like #search_key, but keeps base accented characters
|
|
173
|
+
# for correct ordering). `lang:` applies a language profile; nil means none.
|
|
174
|
+
# Raises Disarm::InvalidArgument on an unknown lang.
|
|
175
|
+
def sort_key(text, lang: nil)
|
|
176
|
+
translate_errors { _sort_key(text, lang&.to_s) }
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# Library catalog deduplication key (search_key plus confusable folding).
|
|
180
|
+
# `lang:` applies a language profile; `strict_iso9:` selects the ISO 9:1995
|
|
181
|
+
# Cyrillic scheme. Raises Disarm::InvalidArgument on an unknown lang.
|
|
182
|
+
def catalog_key(text, lang: nil, strict_iso9: false)
|
|
183
|
+
translate_errors { _catalog_key(text, lang&.to_s, strict_iso9) }
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Strip diacritics ("café" → "cafe").
|
|
187
|
+
def strip_accents(text)
|
|
188
|
+
translate_errors { _strip_accents(text) }
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Unicode case-fold ("HELLO" → "hello").
|
|
192
|
+
def fold_case(text)
|
|
193
|
+
translate_errors { _fold_case(text) }
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Whether `text` is a stable identity key under case folding — whether
|
|
197
|
+
# `fold_case` and `String#downcase` agree on it (#619). A false result means
|
|
198
|
+
# some other string folds to the same value, so a table keyed on this one can
|
|
199
|
+
# collide: "groß.txt" and "gross.txt" are the pair node-tar collided on
|
|
200
|
+
# (CVE-2026-23950). It is a fact about the string, not an accusation.
|
|
201
|
+
def case_fold_stable?(text)
|
|
202
|
+
translate_errors { _is_case_fold_stable?(text) }
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Which of `values` are the same name under `key` (#620).
|
|
206
|
+
#
|
|
207
|
+
# Every other disarm detector is a single-string predicate, and a collision is
|
|
208
|
+
# not a property of a single string — "groß.txt" is an ordinary German filename,
|
|
209
|
+
# and "аdmin" is only a problem next to "admin". This is the set-shaped question
|
|
210
|
+
# node-tar's PathReservations guard failed to ask (CVE-2026-23950).
|
|
211
|
+
#
|
|
212
|
+
# `key:` is one of "fold_case", "search_key", "catalog_key", "canonicalize",
|
|
213
|
+
# "canonicalize_strict", "normalize_confusables". There is no default: a stronger
|
|
214
|
+
# key finds more collisions, including ones nobody attacked ("search_key" collides
|
|
215
|
+
# "Muller" with "Müller"), so the choice is the policy.
|
|
216
|
+
#
|
|
217
|
+
# A group is reported only when two or more DISTINCT inputs share a key; the same
|
|
218
|
+
# name twice is the same name twice. `lang:` reaches "search_key" and
|
|
219
|
+
# "catalog_key" and is ignored by the rest.
|
|
220
|
+
#
|
|
221
|
+
# Returns an array of hashes with `:key`, `:values` (distinct, first-appearance
|
|
222
|
+
# order) and `:indices` (every position, ascending — not parallel to `:values`).
|
|
223
|
+
def find_key_collisions(values, key:, lang: nil)
|
|
224
|
+
translate_errors { _find_key_collisions(values, key.to_s, lang&.to_s) }
|
|
225
|
+
.map { |k, vals, idx| { key: k, values: vals, indices: idx } }
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Whether the hostname looks like a mixed-script / confusable / bidi-reorder
|
|
229
|
+
# IDN spoof. Flags a mixed-script label, a Latin confusable, or a
|
|
230
|
+
# bidi-direction conflict (see #bidi_conflict?, the "BiDi Swap" precondition),
|
|
231
|
+
# or a UAX #9 bidi control character (#603 — the RLO spoof, which the direction
|
|
232
|
+
# conflict is blind to).
|
|
233
|
+
# A false result asserts nothing was *found*, not that the host is safe.
|
|
234
|
+
def suspicious_hostname?(host)
|
|
235
|
+
translate_errors { _suspicious_hostname?(host) }
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# The full hostname homoglyph analysis (#549) behind #suspicious_hostname?,
|
|
239
|
+
# as a Hash. `:suspicious` is a maximally conservative screen (an any-character
|
|
240
|
+
# confusable test flags essentially every non-Latin host), not a precise verdict;
|
|
241
|
+
# branch on the granular signals plus your own policy. Keys: :suspicious,
|
|
242
|
+
# :scripts, :mixed_script, :has_confusables, :bidi_conflict, :bidi_control,
|
|
243
|
+
# :has_invisible, :compat_fold, :cross_label_script,
|
|
244
|
+
# :label_scripts, :whole_script_confusable, :label_whole_script_confusable,
|
|
245
|
+
# :canonical. `:whole_script_confusable` is a graded signal, NOT folded into
|
|
246
|
+
# `:suspicious` (see #545).
|
|
247
|
+
def analyze_hostname(host, contractions: false)
|
|
248
|
+
suspicious, scripts, mixed_script, has_confusables, bidi_conflict,
|
|
249
|
+
bidi_control, has_invisible, compat_fold, cross_label_script, label_scripts,
|
|
250
|
+
whole_script_confusable, (label_whole_script_confusable, canonical) =
|
|
251
|
+
translate_errors { _analyze_hostname(host, contractions) }
|
|
252
|
+
{
|
|
253
|
+
suspicious:,
|
|
254
|
+
scripts:,
|
|
255
|
+
mixed_script:,
|
|
256
|
+
has_confusables:,
|
|
257
|
+
bidi_conflict:,
|
|
258
|
+
bidi_control:,
|
|
259
|
+
has_invisible:,
|
|
260
|
+
compat_fold:,
|
|
261
|
+
cross_label_script:,
|
|
262
|
+
label_scripts:,
|
|
263
|
+
whole_script_confusable:,
|
|
264
|
+
label_whole_script_confusable:,
|
|
265
|
+
canonical:,
|
|
266
|
+
}
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Apply a Unicode normalization form. `form:` is :nfc (default), :nfd,
|
|
270
|
+
# :nfkc, or :nfkd (a Symbol or String; case-insensitive).
|
|
271
|
+
def normalize(text, form: :nfc)
|
|
272
|
+
translate_errors { _normalize(text, form.to_s.upcase) }
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Whether `text` is already in normalization `form:` (default :nfc).
|
|
276
|
+
def normalized?(text, form: :nfc)
|
|
277
|
+
translate_errors { _normalized?(text, form.to_s.upcase) }
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# Fold every run of Unicode whitespace to a single ASCII space and trim
|
|
281
|
+
# leading/trailing whitespace (#433). Folds whitespace ONLY — the line
|
|
282
|
+
# controls (TAB/LF/VT/FF/CR), the information separators (U+001C–U+001F),
|
|
283
|
+
# NEL, the Zs/Zl/Zp spaces, and the blank-rendering set (Braille blank,
|
|
284
|
+
# Hangul fillers) each fold to a single space. It does NOT delete control or
|
|
285
|
+
# zero-width characters — use `strip_control_chars` / `strip_zero_width_chars`
|
|
286
|
+
# for that. Folding the line controls (not deleting) means "a\rb" → "a b".
|
|
287
|
+
def collapse_whitespace(text)
|
|
288
|
+
translate_errors { _collapse_whitespace(text) }
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Remove C0/C1 control characters (except tab and newline).
|
|
292
|
+
def strip_control_chars(text)
|
|
293
|
+
translate_errors { _strip_control_chars(text) }
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Remove zero-width characters (ZWSP, ZWNJ, ZWJ, word joiner).
|
|
297
|
+
def strip_zero_width_chars(text)
|
|
298
|
+
translate_errors { _strip_zero_width_chars(text) }
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Remove Unicode bidirectional control characters (a homoglyph/spoof vector).
|
|
302
|
+
def strip_bidi(text)
|
|
303
|
+
translate_errors { _strip_bidi(text) }
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# Strip the Unicode Tags block (U+E0000-U+E007F) - the "ASCII smuggling"
|
|
307
|
+
# channel - preserving well-formed emoji subdivision flag sequences (#413).
|
|
308
|
+
def strip_tags(text)
|
|
309
|
+
translate_errors { _strip_tags(text) }
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# Strip every variation selector (VS1-VS256) - the arbitrary-byte smuggling
|
|
313
|
+
# channel (#413).
|
|
314
|
+
def strip_variation_selectors(text)
|
|
315
|
+
translate_errors { _strip_variation_selectors(text) }
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
# Strip every Unicode noncharacter (U+FDD0-U+FDEF and U+xFFFE/U+xFFFF) (#413).
|
|
319
|
+
def strip_noncharacters(text)
|
|
320
|
+
translate_errors { _strip_noncharacters(text) }
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# Strip every Private Use Area code point (BMP and planes 15/16) (#413).
|
|
324
|
+
def strip_pua(text)
|
|
325
|
+
translate_errors { _strip_pua(text) }
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
# Strip "zalgo" combining-mark stacking, keeping at most `max_marks:` (2)
|
|
329
|
+
# combining marks per base character.
|
|
330
|
+
def strip_zalgo(text, max_marks: 2)
|
|
331
|
+
translate_errors { _strip_zalgo(text, max_marks) }
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# Whether `text` looks like zalgo: any base character carries more than
|
|
335
|
+
# `threshold:` (3) combining marks.
|
|
336
|
+
def zalgo?(text, threshold: 3)
|
|
337
|
+
translate_errors { _zalgo?(text, threshold) }
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# Number of grapheme clusters (user-perceived characters). Counts an emoji
|
|
341
|
+
# or flag as one, unlike `String#length` (code points).
|
|
342
|
+
def grapheme_len(text)
|
|
343
|
+
translate_errors { _grapheme_len(text) }
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# Split `text` into an array of grapheme-cluster strings.
|
|
347
|
+
def grapheme_split(text)
|
|
348
|
+
translate_errors { _grapheme_split(text) }
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# Truncate `text` to at most `max_graphemes` grapheme clusters, never cutting
|
|
352
|
+
# through the middle of a cluster.
|
|
353
|
+
def grapheme_truncate(text, max_graphemes)
|
|
354
|
+
translate_errors { _grapheme_truncate(text, max_graphemes) }
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# Display width (terminal columns) of a single grapheme `cluster` by East
|
|
358
|
+
# Asian Width. Pass `ambiguous_wide: true` to treat ambiguous-width
|
|
359
|
+
# characters as 2 columns.
|
|
360
|
+
def grapheme_width(cluster, ambiguous_wide: false)
|
|
361
|
+
translate_errors { _grapheme_width(cluster, ambiguous_wide) }
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
# Total display width (terminal columns) of `text`.
|
|
365
|
+
def terminal_width(text, ambiguous_wide: false)
|
|
366
|
+
translate_errors { _terminal_width(text, ambiguous_wide) }
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# Turn arbitrary text into a safe filename. `platform:` is :universal
|
|
370
|
+
# (default), :windows, or :posix; `preserve_extension:` keeps the final
|
|
371
|
+
# extension when truncating to `max_length:`. Raises Disarm::InvalidArgument
|
|
372
|
+
# on an unknown platform.
|
|
373
|
+
#
|
|
374
|
+
# A safe *filename*, not a safe URL path segment. "%" is legal in a filename, so one
|
|
375
|
+
# the caller typed is kept — sanitize_filename("..%2Fetc") returns "%2Fetc" — and a
|
|
376
|
+
# consumer that percent-decodes the result must validate AFTER decoding. What this
|
|
377
|
+
# will not do is manufacture one: "%" never appears in the output unless it appeared
|
|
378
|
+
# in the input (#721).
|
|
379
|
+
def sanitize_filename(text, separator: "_", max_length: 255, platform: :universal,
|
|
380
|
+
lang: nil, preserve_extension: true)
|
|
381
|
+
translate_errors do
|
|
382
|
+
_sanitize_filename(text, separator.to_s, max_length, platform.to_s,
|
|
383
|
+
lang&.to_s, preserve_extension)
|
|
384
|
+
end
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
# Reverse-transliterate Latin back to a native script. `lang:` is :el (Greek),
|
|
388
|
+
# :ru (Russian), or :uk (Ukrainian) — a Symbol or String.
|
|
389
|
+
def reverse_transliterate(text, lang:)
|
|
390
|
+
translate_errors { _reverse_transliterate(text, lang.to_s) }
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
# Every character in `text` with no romanization, as an array of
|
|
394
|
+
# `{ char:, offset: }` hashes (byte offset), in order of appearance.
|
|
395
|
+
# `scheme:`/`lang:` mirror #transliterate.
|
|
396
|
+
def find_untranslatable(text, scheme: :default, lang: nil)
|
|
397
|
+
translate_errors do
|
|
398
|
+
_find_untranslatable(text, scheme.to_s, lang&.to_s)
|
|
399
|
+
.map { |ch, offset| { char: ch, offset: offset } }
|
|
400
|
+
end
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
# Every upstream confusable source the bundled table does not fold, as a sorted
|
|
404
|
+
# Array of single-character Strings (#563). An Array rather than a Set, matching
|
|
405
|
+
# `list_scripts` and avoiding a `require "set"` in the gem entrypoint.
|
|
406
|
+
#
|
|
407
|
+
# Read as exposure, not as a score — this is where an adaptive attacker goes once
|
|
408
|
+
# the mapped sources stop working. It includes five ASCII characters ("%", "0",
|
|
409
|
+
# "1", "I", "m"): TR39 is a skeleton transform, and disarm deliberately does not
|
|
410
|
+
# apply those rows because folding a legitimate "m" to "rn" corrupts prose.
|
|
411
|
+
def unmapped_confusables(target: :latin)
|
|
412
|
+
translate_errors { _unmapped_confusables(target.to_s) }
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
# Confusable sources in `text` the bundled table does not fold, as an Array of
|
|
416
|
+
# `{ char:, offset: }` hashes in order of appearance — the confusables analogue of
|
|
417
|
+
# `find_untranslatable`, with the same byte-offset convention (#563).
|
|
418
|
+
def find_unmapped_confusables(text, target: :latin)
|
|
419
|
+
translate_errors do
|
|
420
|
+
_find_unmapped_confusables(text, target.to_s)
|
|
421
|
+
.map { |ch, offset| { char: ch, offset: offset } }
|
|
422
|
+
end
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
# ML/NLP normalization: NFKC → emoji→text → transliterate → strip accents →
|
|
426
|
+
# [case fold] → strip control → strip zero-width → collapse whitespace.
|
|
427
|
+
#
|
|
428
|
+
# `fold_case:` defaults to true. Pass false in front of a CASED model — folding is
|
|
429
|
+
# destructive, cannot be undone downstream, and an uncased evaluation harness cannot
|
|
430
|
+
# measure what it cost. It restores case, not diacritics: accents are still stripped.
|
|
431
|
+
#
|
|
432
|
+
# Folds no confusables, so it is not a homoglyph defence at any setting; compose it
|
|
433
|
+
# after `normalize_confusables` when a model needs both.
|
|
434
|
+
def ml_normalize(text, lang: nil, emoji_style: "cldr", fold_case: true)
|
|
435
|
+
translate_errors { _ml_normalize(text, lang&.to_s, emoji_style.to_s, fold_case) }
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# The Unicode scripts present in `text`, in first-appearance order
|
|
439
|
+
# (Common/Inherited excluded), as stable UCD identifiers (e.g. "Latin").
|
|
440
|
+
def detect_scripts(text)
|
|
441
|
+
translate_errors { _detect_scripts(text) }
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
# Whether `text` mixes characters from more than one script.
|
|
445
|
+
def mixed_script?(text)
|
|
446
|
+
translate_errors { _is_mixed_script?(text) }
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
# Whether `text` mixes strong left-to-right and strong right-to-left
|
|
450
|
+
# characters — the precondition for Bidi display-reordering (UAX #9) and the
|
|
451
|
+
# structural signal behind "BiDi Swap"-style spoofs. Fires on the real
|
|
452
|
+
# letters (no U+202x override). A false result is not a safety guarantee.
|
|
453
|
+
def bidi_conflict?(text)
|
|
454
|
+
translate_errors { _has_bidi_conflict?(text) }
|
|
455
|
+
end
|
|
456
|
+
|
|
457
|
+
# Whether `text` carries any of the twelve UAX #9 explicit formatting characters,
|
|
458
|
+
# with no context taken into account. The counterpart to `bidi_conflict?`, which
|
|
459
|
+
# reads strong-direction letters and is blind to these; the two are disjoint. The
|
|
460
|
+
# anomaly detector's `bidi` kind reports nine of the twelve, holding back LRM, RLM
|
|
461
|
+
# and ALM because a lone directional mark is ordinary in right-to-left text.
|
|
462
|
+
def bidi_control?(text)
|
|
463
|
+
_has_bidi_control?(text)
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
# Explain how `lang: "auto"` detection resolves `text`: a hash with
|
|
467
|
+
# `:script`, `:chosen_lang` (both nil if undetected), `:reason`, and
|
|
468
|
+
# `:discriminators_hit`.
|
|
469
|
+
def inspect_auto_lang(text)
|
|
470
|
+
script, chosen_lang, reason, discriminators = translate_errors { _inspect_auto_lang(text) }
|
|
471
|
+
{ script: script, chosen_lang: chosen_lang, reason: reason,
|
|
472
|
+
discriminators_hit: discriminators }
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
# Curated metadata for one language `code` (e.g. "de"), as a hash with symbol
|
|
476
|
+
# keys: `:name`, `:script`, `:region`, and `:context` ("none"/"partial"/"full").
|
|
477
|
+
# Raises Disarm::InvalidArgument on an unknown code.
|
|
478
|
+
def lang_info(code)
|
|
479
|
+
translate_errors { _lang_info(code.to_s) }
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
# Curated metadata for one script `name` (e.g. "Coptic"), as a hash with symbol
|
|
483
|
+
# keys: `:name`, `:default_lang` (nil when none), `:example`, and
|
|
484
|
+
# `:context_aware`. Raises Disarm::InvalidArgument on an unknown script.
|
|
485
|
+
def script_info(name)
|
|
486
|
+
translate_errors { _script_info(name.to_s) }
|
|
487
|
+
end
|
|
488
|
+
|
|
489
|
+
# The Unicode `confusables.txt` release the bundled confusable tables were folded
|
|
490
|
+
# from, e.g. "17.0.0". Not a Unicode version for the library as a whole — the
|
|
491
|
+
# case-folding and width tables track different releases (see docs/provenance.md).
|
|
492
|
+
# The UCD release disarm's normalizer implements. Not a library-wide Unicode
|
|
493
|
+
# version — the bundled tables track different releases. This is the one integrators
|
|
494
|
+
# ask about: it decides whether disarm's normalization agrees with Ruby's.
|
|
495
|
+
def unicode_version
|
|
496
|
+
_unicode_version
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
# Whether a key stored under an earlier release still compares equal. A monotonic
|
|
500
|
+
# counter, not a version: two artifacts reporting the same value produce the same key
|
|
501
|
+
# for the same input. Meaningless in isolation, by design.
|
|
502
|
+
def key_schema_version
|
|
503
|
+
_key_schema_version
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
def confusables_version
|
|
507
|
+
translate_errors { _confusables_version }
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
# Every script disarm knows, as stable UCD script identifiers (includes
|
|
511
|
+
# "Common"/"Inherited"), sorted by name.
|
|
512
|
+
def list_scripts
|
|
513
|
+
translate_errors { _list_scripts }
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
# The language codes with context-aware transliteration support, sorted by code.
|
|
517
|
+
def list_context_langs
|
|
518
|
+
translate_errors { _list_context_langs }
|
|
519
|
+
end
|
|
520
|
+
|
|
521
|
+
# Whether any whitespace token carries out-of-place characters that disguise a
|
|
522
|
+
# real word — a cross-script homoglyph, leet, segmentation, a zero-width / bidi
|
|
523
|
+
# control, or zalgo. Reports a technical fact and leaves the malicious-or-not
|
|
524
|
+
# judgement to the caller. `lexicon` is a common-word collection (Array or Set)
|
|
525
|
+
# used only by the leet and segmentation branches; it defaults to an empty list
|
|
526
|
+
# when those branches are not needed. A bare String is rejected — pass an Array
|
|
527
|
+
# or any object responding to `:each`.
|
|
528
|
+
#
|
|
529
|
+
# For repeated calls over the same word list, build a Disarm::Lexicon once and
|
|
530
|
+
# pass it here: the native HashSet is then reused rather than rebuilt per call
|
|
531
|
+
# (HAI-SDLC 6.1).
|
|
532
|
+
def has_anomalies?(text, lexicon = [])
|
|
533
|
+
translate_errors do
|
|
534
|
+
if lexicon.is_a?(Disarm::Lexicon)
|
|
535
|
+
_has_anomalies_lex(text, lexicon)
|
|
536
|
+
else
|
|
537
|
+
_has_anomalies?(text, coerce_lexicon(lexicon))
|
|
538
|
+
end
|
|
539
|
+
end
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
# Full anomaly analysis: a hash with `:anomalous`, `:kinds` (in first-appearance
|
|
543
|
+
# order), `:findings` (each `{ kind:, token:, start:, end:, detail:, reason: }`,
|
|
544
|
+
# with byte offsets), and `:reason` (the first finding's reason, or nil).
|
|
545
|
+
# `lexicon` defaults to an empty list; a bare String is rejected. Pass a
|
|
546
|
+
# pre-built Disarm::Lexicon to reuse the native HashSet across calls (6.1).
|
|
547
|
+
def inspect_anomalies(text, lexicon = [])
|
|
548
|
+
anomalous, kinds, findings, reason =
|
|
549
|
+
translate_errors do
|
|
550
|
+
if lexicon.is_a?(Disarm::Lexicon)
|
|
551
|
+
_inspect_anomalies_lex(text, lexicon)
|
|
552
|
+
else
|
|
553
|
+
_inspect_anomalies(text, coerce_lexicon(lexicon))
|
|
554
|
+
end
|
|
555
|
+
end
|
|
556
|
+
{
|
|
557
|
+
anomalous: anomalous,
|
|
558
|
+
kinds: kinds,
|
|
559
|
+
findings: findings.map do |kind, token, start, finish, detail, fr|
|
|
560
|
+
{ kind: kind, token: token, start: start, end: finish, detail: detail, reason: fr }
|
|
561
|
+
end,
|
|
562
|
+
reason: reason,
|
|
563
|
+
}
|
|
564
|
+
end
|
|
565
|
+
|
|
566
|
+
# Build a reusable Disarm::Pipeline for a named policy `profile` (e.g.
|
|
567
|
+
# "search_index", "normalize_web_input"). The profile's steps are validated
|
|
568
|
+
# and assembled once at construction, so the returned handle can be reused
|
|
569
|
+
# across many `#process` calls without re-resolving the profile each time —
|
|
570
|
+
# the same reuse pattern as Disarm::Lexicon. Raises Disarm::InvalidArgument
|
|
571
|
+
# on an unknown profile name.
|
|
572
|
+
#
|
|
573
|
+
# pipe = Disarm.get_pipeline("search_index")
|
|
574
|
+
# pipe.process("Café") # => "cafe"
|
|
575
|
+
# pipe.process("Köln") # reuse the same handle
|
|
576
|
+
#
|
|
577
|
+
# Disarm::Pipeline#process is the Rust-defined instance method on the handle.
|
|
578
|
+
def get_pipeline(profile)
|
|
579
|
+
translate_errors { _get_pipeline(profile.to_s) }
|
|
580
|
+
end
|
|
581
|
+
|
|
582
|
+
private
|
|
583
|
+
|
|
584
|
+
# Coerce a lexicon argument to an Array of Strings for the native layer.
|
|
585
|
+
# Fast-path: an Array already containing only Strings is passed through as-is.
|
|
586
|
+
# Any other Enumerable (Set, etc.) is mapped to String. A bare String is rejected
|
|
587
|
+
# with ArgumentError — callers must wrap it in an Array: ["word"].
|
|
588
|
+
def coerce_lexicon(lexicon)
|
|
589
|
+
# An explicit nil is treated as an empty lexicon (parity with the `= []`
|
|
590
|
+
# default and the other bindings' null handling), not an error.
|
|
591
|
+
return [] if lexicon.nil?
|
|
592
|
+
|
|
593
|
+
raise ::ArgumentError, "lexicon must be an Array or Enumerable, not a String" \
|
|
594
|
+
if lexicon.is_a?(::String)
|
|
595
|
+
|
|
596
|
+
return lexicon if lexicon.is_a?(::Array) && lexicon.all?(::String)
|
|
597
|
+
|
|
598
|
+
lexicon.map(&:to_s)
|
|
599
|
+
end
|
|
600
|
+
|
|
601
|
+
# Run a native call, re-raising its built-in exception as the matching
|
|
602
|
+
# Disarm::Error subclass so callers can `rescue Disarm::Error` across the
|
|
603
|
+
# whole surface. The original backtrace is preserved (passed as the third
|
|
604
|
+
# `raise` argument) so the failing native call site stays visible. A bad
|
|
605
|
+
# argument from the native layer can arrive as ArgumentError (an invalid
|
|
606
|
+
# scheme/target), TypeError (a non-String argument), or RangeError (e.g. a
|
|
607
|
+
# negative max_length) — all map to Disarm::InvalidArgument.
|
|
608
|
+
def translate_errors
|
|
609
|
+
yield
|
|
610
|
+
rescue Error
|
|
611
|
+
raise # already in our hierarchy — don't re-wrap
|
|
612
|
+
rescue ::ArgumentError, ::TypeError, ::RangeError => e
|
|
613
|
+
raise InvalidArgument, e.message, e.backtrace
|
|
614
|
+
rescue ::RuntimeError => e
|
|
615
|
+
raise Error, e.message, e.backtrace
|
|
616
|
+
end
|
|
617
|
+
end
|
|
618
|
+
end
|