disarm 0.15.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d75406993aaa2ac76a20429c567601447bcffa79708398444b83dc8b445ebf01
-  data.tar.gz: cfea208f2da86fd373ed82f4e25a9166277ffd1806c22614551418ddb20acb05
+  metadata.gz: 8109d11076864c4647b18efd378ad51e561dc3302ec339f1da1f895253139be6
+  data.tar.gz: a42fa1ad22324a6e07bf502e92231a36c628367f720b440ccefe0c3ea4fd82e0
 SHA512:
-  metadata.gz: 68a46c759f40421b154e1879500edecf0ced53931224e317b1f3dacfb5a03750ebe132bfe7b1efd7f4be21d384f4478bfc104d826e27675c85b61c79723b40a1
-  data.tar.gz: 93d71a7e1d5aac2f0a33e26a72e6658ad842a2408b4e744f70ed73bdc1d141b567b2b575ec241c8e738e2c54dc39d4f49309118b353f7985084226d089de7ca3
+  metadata.gz: 764e4f27dc2a2774fdb6df0f4b0cb016f122281dbb44ee8558ce50148427c6a7c23b7b6261004a6a3e64ed931dc0da613e8ebd98fa8b36e58d10b24030a99837
+  data.tar.gz: 608a56409c3bdc8ab76f8073880ddd1cd66ca60067063775624fae2306ec83d2591a2011c56f2c329b00f01a09f2db9b16e05331b4b072dbd1977ebec1777369

data/README.md

@@ -19,10 +19,10 @@ gem "disarm"
 gem install disarm
 ```
 
-Requires Ruby >= 3.1. Precompiled platform gems ship for Ruby 3.1 through 4.0
+Requires Ruby >= 3.3. Precompiled platform gems ship for Ruby 3.3 through 4.0
 (Linux x86_64/aarch64, macOS x86_64/arm64, Windows). On a supported Ruby with no
 matching platform gem, the source gem installs and compiles locally, which needs a
-Rust toolchain. Below 3.1 the gem does not install at all.
+Rust toolchain. Below 3.3 the gem does not install at all.
 
 ## Usage
 

data/ext/disarm/Cargo.toml

@@ -13,7 +13,7 @@
 name = "disarm"
 version = "0.0.0"
 edition = "2021"
-rust-version = "1.81"
+rust-version = "1.88"
 license = "MIT"
 description = "Ruby bindings for disarm — Unicode confusable/text-security building blocks"
 publish = false
@@ -30,7 +30,7 @@ crate-type = ["cdylib"]
 # dep — not a path — so the gem is self-contained and the rb-sys-dock cross-gem
 # build (which mounts only this gem dir) can fetch it. Imported as `disarm_core`
 # because the package name `disarm` would otherwise clash with this crate.
-disarm_core = { package = "disarm", version = "0.15", default-features = false }
+disarm_core = { package = "disarm", version = "0.17", default-features = false }
 # magnus: ergonomic, safe Ruby<->Rust bindings over rb-sys. 0.8 supports Ruby >= 3.0
 # (the gem's required_ruby_version); rb-sys handles the platform glue.
 magnus = "0.8"

data/ext/disarm/src/lib.rs

@@ -34,7 +34,8 @@
 use std::collections::HashSet;
 
 use disarm_core::api;
-use magnus::{function, method, prelude::*, Error, RHash, Ruby};
+use magnus::encoding::EncodingCapable;
+use magnus::{function, kwargs, method, prelude::*, Error, RHash, RString, Ruby};
 
 /// Map a `disarm` error onto the closest standard Ruby exception:
 /// `InvalidArgument` → `ArgumentError`, everything else → `RuntimeError`. The
@@ -147,9 +148,30 @@ fn wtf8_to_utf8(bytes: &[u8]) -> String {
     out
 }
 
-/// A text argument decoded at the boundary with the WTF-8 → UTF-8 contract (#472).
-/// Used in place of `String` for every text parameter; `Deref<Target = str>` lets the
-/// existing `&text` call sites reach the core unchanged.
+/// A text argument decoded at the boundary by the encoding the String declares (#472,
+/// and R1 of `formal/bindings`). Used in place of `String` for every text parameter;
+/// `Deref<Target = str>` lets the existing `&text` call sites reach the core unchanged.
+///
+/// The contract, per `String#encoding`:
+///
+/// - **UTF-8**: the bytes as they are, with the WTF-8 scrub above for malformed input: a
+///   well-formed surrogate pair recombines, each lone surrogate or undecodable byte is
+///   one `U+FFFD`.
+/// - **US-ASCII**: the bytes as they are; a byte above `0x7F` is not US-ASCII and is one
+///   `U+FFFD`.
+/// - **ASCII-8BIT (BINARY)**: bytes with no declared encoding, read as UTF-8 under the
+///   same scrub as a UTF-8 String. `File.binread` and socket reads return BINARY for
+///   what is usually UTF-8, and the C ABI reads its bytes the same way (#1020).
+/// - **Any other encoding** (ISO-8859-1, Windows-1251, Shift_JIS, UTF-16LE, ...):
+///   transcoded to UTF-8 by Ruby's own `String#encode`, each invalid or unmappable
+///   sequence becoming one `U+FFFD`. An encoding Ruby cannot convert from at all (a
+///   dummy such as UTF-7) raises `Encoding::ConverterNotFoundError`, which the Ruby
+///   layer reports as `Disarm::InvalidArgument`.
+///
+/// Until R1 every String's bytes were read as UTF-8 whatever it declared, so an
+/// ISO-8859-1 `"caf\xE9"` transliterated to `"caf[?]"` and a Windows-1251 word to a row
+/// of `[?]`, while the plain `String` parameters of the same functions, which magnus
+/// converts, read the same bytes correctly.
 struct Wtf8Text(String);
 
 impl std::ops::Deref for Wtf8Text {
@@ -161,13 +183,43 @@ impl std::ops::Deref for Wtf8Text {
 
 impl magnus::TryConvert for Wtf8Text {
     fn try_convert(val: magnus::Value) -> Result<Self, Error> {
-        let s = magnus::RString::try_convert(val)?;
+        let s = RString::try_convert(val)?;
+        // GVL invariant: argument conversion runs inside a Ruby method callback.
+        #[allow(clippy::expect_used)]
+        let ruby = Ruby::get().expect("argument conversion runs while holding the Ruby GVL");
+        let encoding = s.enc_get();
+        let usascii = encoding == ruby.usascii_encindex();
+        let s =
+            if encoding == ruby.utf8_encindex() || usascii || encoding == ruby.ascii8bit_encindex()
+            {
+                s
+            } else {
+                let options = kwargs!(
+                    &ruby,
+                    "invalid" => ruby.to_symbol("replace"),
+                    "undef" => ruby.to_symbol("replace"),
+                    "replace" => "\u{FFFD}"
+                );
+                s.funcall::<_, _, RString>("encode", (ruby.utf8_encoding(), options))?
+            };
         // SAFETY: the bytes are copied out immediately; no Ruby API runs between
         // `as_slice` and `to_vec`, so the string cannot be moved or collected.
         let bytes = unsafe { s.as_slice().to_vec() };
         let decoded = match std::str::from_utf8(&bytes) {
-            Ok(valid) => valid.to_owned(),
-            Err(_) => wtf8_to_utf8(&bytes),
+            Ok(valid) if !usascii || valid.is_ascii() => valid.to_owned(),
+            // A US-ASCII String carrying a byte above 0x7F: each such byte is invalid in
+            // the encoding the String declares, and is one U+FFFD.
+            _ if usascii => bytes
+                .iter()
+                .map(|&b| {
+                    if b.is_ascii() {
+                        char::from(b)
+                    } else {
+                        '\u{FFFD}'
+                    }
+                })
+                .collect(),
+            _ => wtf8_to_utf8(&bytes),
         };
         Ok(Wtf8Text(decoded))
     }
@@ -191,6 +243,16 @@ fn transliterate_opts(
     scheme: String,
     lang: Option<String>,
 ) -> Result<String, Error> {
+    // `try_run` rejects an unknown `lang` (B2) and applies registered replacements.
+    transliterate_builder(&scheme, lang)?
+        .try_run(&text)
+        .map(std::borrow::Cow::into_owned)
+        .map_err(|e| map_err(&e))
+}
+
+/// The core's `Transliterate` builder for a scheme token and an optional `lang`; the
+/// `lang` is checked when the builder runs.
+fn transliterate_builder(scheme: &str, lang: Option<String>) -> Result<api::Transliterate, Error> {
     let mut builder = api::Transliterate::new();
     if scheme != "default" {
         let scheme: api::Scheme = scheme.parse().map_err(|e| map_err(&e))?;
@@ -199,7 +261,7 @@ fn transliterate_opts(
     if let Some(lang) = lang {
         builder = builder.lang(lang);
     }
-    Ok(builder.run(&text).into_owned())
+    Ok(builder)
 }
 
 // ── Confusables (TR39) ────────────────────────────────────────────────────────
@@ -215,6 +277,16 @@ fn normalize_confusables(
     Ok(api::normalize_confusables_with(&text, target, digit_policy).into_owned())
 }
 
+/// The `digit_policy` token every key builder takes (#896): parsed once, at the boundary.
+fn parse_policy(digit_policy: &str) -> Result<api::DigitPolicy, Error> {
+    digit_policy.parse().map_err(|e| map_err(&e))
+}
+
+/// `Disarm._is_canonical?(text, preset)` — already its own canonical form (#730).
+fn is_canonical(text: Wtf8Text, preset: String) -> Result<bool, Error> {
+    api::is_canonical(&text, &preset).map_err(|e| map_err(&e))
+}
+
 /// `Disarm._confusable?(text, "latin" | "cyrillic" | "arabic" | "hebrew")`.
 fn is_confusable(text: Wtf8Text, target: String) -> Result<bool, Error> {
     let target: api::TargetScript = target.parse().map_err(|e| map_err(&e))?;
@@ -300,7 +372,7 @@ fn slugify(
     decimal: bool,
     hexadecimal: bool,
     safe_chars: String,
-) -> String {
+) -> Result<String, Error> {
     let mut config = api::SlugConfig::default()
         .with_separator(separator)
         .with_lowercase(lowercase)
@@ -318,10 +390,17 @@ fn slugify(
     config.entities = entities;
     config.decimal = decimal;
     config.hexadecimal = hexadecimal;
-    api::slugify(&text, &config)
+    // `try_slugify` rejects an unknown `lang` rather than falling back (B2).
+    api::try_slugify(&text, &config).map_err(|e| map_err(&e))
+}
+
+/// `Disarm._replace_emoji(text, replacement)` — every emoji replaced, verbatim (#972).
+fn replace_emoji(text: Wtf8Text, replacement: String) -> String {
+    api::replace_emoji(&text, &replacement)
 }
 
-/// `Disarm._demojize(text, strip_modifiers)`.
+/// `Disarm._demojize(text, strip_modifiers)`. An emoji CLDR cannot name becomes `[?]`,
+/// as in every binding.
 fn demojize(text: Wtf8Text, strip_modifiers: bool) -> String {
     api::demojize(&text, strip_modifiers)
 }
@@ -340,8 +419,8 @@ fn demojize(text: Wtf8Text, strip_modifiers: bool) -> String {
 ///
 /// Unlike `canonicalize` it does NOT fold confusables, so non-Latin text keeps its script
 /// — the point of the preset.
-fn canonicalize_strict(text: Wtf8Text) -> Result<String, Error> {
-    api::canonicalize_strict(&text)
+fn canonicalize_strict(text: Wtf8Text, digit_policy: String) -> Result<String, Error> {
+    api::canonicalize_strict_with(&text, parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
@@ -350,42 +429,77 @@ fn strip_format(text: Wtf8Text) -> String {
     api::strip_format(&text).into_owned()
 }
 
-fn strip_obfuscation(text: Wtf8Text) -> Result<String, Error> {
-    api::strip_obfuscation(&text)
+fn strip_obfuscation(text: Wtf8Text, digit_policy: String) -> Result<String, Error> {
+    api::strip_obfuscation_with(&text, parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
-fn canonicalize(text: Wtf8Text) -> Result<String, Error> {
-    api::canonicalize(&text)
+fn canonicalize(text: Wtf8Text, digit_policy: String) -> Result<String, Error> {
+    api::canonicalize_with(&text, parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
 /// `Disarm._search_key(text, lang)` — case/accent/script-insensitive lookup key.
 /// `lang` is `nil` (no profile) or a code like `"ru"`. Fails on an unknown `lang`.
-fn search_key(text: Wtf8Text, lang: Option<String>) -> Result<String, Error> {
-    api::search_key(&text, lang.as_deref())
+fn search_key(text: Wtf8Text, lang: Option<String>, digit_policy: String) -> Result<String, Error> {
+    api::search_key_with(&text, lang.as_deref(), parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
 /// `Disarm._sort_key(text, lang)` — collation sort key (preserves base accented
 /// characters for correct ordering). Fails on an unknown `lang`.
-fn sort_key(text: Wtf8Text, lang: Option<String>) -> Result<String, Error> {
-    api::sort_key(&text, lang.as_deref())
+fn sort_key(text: Wtf8Text, lang: Option<String>, digit_policy: String) -> Result<String, Error> {
+    api::sort_key_with(&text, lang.as_deref(), parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
 /// `Disarm._catalog_key(text, lang, strict_iso9)` — catalog deduplication key.
 /// `strict_iso9` selects the ISO 9:1995 Cyrillic scheme. Fails on an unknown `lang`.
-fn catalog_key(text: Wtf8Text, lang: Option<String>, strict_iso9: bool) -> Result<String, Error> {
-    api::catalog_key(&text, lang.as_deref(), strict_iso9)
+fn catalog_key(
+    text: Wtf8Text,
+    lang: Option<String>,
+    strict_iso9: bool,
+    digit_policy: String,
+) -> Result<String, Error> {
+    api::catalog_key_with(
+        &text,
+        lang.as_deref(),
+        strict_iso9,
+        parse_policy(&digit_policy)?,
+    )
+    .map(std::borrow::Cow::into_owned)
+    .map_err(|e| map_err(&e))
+}
+
+/// `Disarm._skeleton_key(text, digit_policy)` — the TR39 identifier skeleton plus the two
+/// prototype classes disarm's table keeps apart (#650). A spoof key: never for display.
+fn skeleton_key(text: Wtf8Text, digit_policy: String) -> Result<String, Error> {
+    api::skeleton_key(&text, parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
+/// `Disarm._edit_distance(a, b)` — Levenshtein distance in characters (#894).
+fn edit_distance(a: Wtf8Text, b: Wtf8Text) -> usize {
+    api::edit_distance(&a, &b)
+}
+
+/// `Disarm._nearest_match(value, candidates, max_distance)` — the closest candidate and
+/// its distance as a `(value, distance)` tuple, or `nil` beyond `max_distance` (#894).
+/// The Ruby layer maps the tuple to a `{ value:, distance: }` hash.
+fn nearest_match(
+    value: Wtf8Text,
+    candidates: Vec<String>,
+    max_distance: usize,
+) -> Option<(String, usize)> {
+    api::nearest_match(&value, candidates.iter().map(String::as_str), max_distance)
+        .map(|m| (m.value, m.distance))
+}
+
 /// `Disarm._suspicious_hostname?(host)` — flags mixed-script / confusable IDN
 /// spoofs. A false result asserts nothing was *found*, not that the host is safe.
 fn suspicious_hostname(host: Wtf8Text) -> bool {
@@ -583,16 +697,9 @@ fn find_untranslatable(
     scheme: String,
     lang: Option<String>,
 ) -> Result<Vec<(String, usize)>, Error> {
-    let mut builder = api::Transliterate::new();
-    if scheme != "default" {
-        let scheme: api::Scheme = scheme.parse().map_err(|e| map_err(&e))?;
-        builder = builder.scheme(scheme);
-    }
-    if let Some(lang) = lang {
-        builder = builder.lang(lang);
-    }
-    Ok(builder
-        .find_untranslatable(&text)
+    Ok(transliterate_builder(&scheme, lang)?
+        .try_find_untranslatable(&text)
+        .map_err(|e| map_err(&e))?
         .into_iter()
         .map(|u| (u.ch.to_string(), u.offset))
         .collect())
@@ -670,6 +777,22 @@ fn lang_info(code: String) -> Result<RHash, Error> {
     Ok(hash)
 }
 
+/// `Disarm._confusable_coverage(script)` — TR39 sources whose prototype is in `script`
+/// and how many the bundled tables fold (#963), as a Ruby Hash with symbol keys
+/// (`{ script:, sources:, folded: }`). The denominator `unmapped_confusables` does not
+/// have. Fails (ArgumentError → Disarm::InvalidArgument) on an unknown script.
+fn confusable_coverage(script: String) -> Result<RHash, Error> {
+    let row = api::confusable_coverage(&script).map_err(|e| map_err(&e))?;
+    // GVL invariant: a Ruby method callback always holds the GVL. Justified.
+    #[allow(clippy::expect_used)]
+    let ruby = Ruby::get().expect("a Ruby method callback always holds the GVL");
+    let hash = ruby.hash_new();
+    hash.aset(ruby.to_symbol("script"), row.script)?;
+    hash.aset(ruby.to_symbol("sources"), row.sources)?;
+    hash.aset(ruby.to_symbol("folded"), row.folded)?;
+    Ok(hash)
+}
+
 /// `Disarm._script_info(name)` — curated metadata for one script as a Ruby Hash
 /// with symbol keys (`{ name:, default_lang:, example:, context_aware: }`);
 /// `default_lang` is `nil` when the core has none. Fails (ArgumentError →
@@ -823,6 +946,26 @@ fn pipeline_process(rb_self: &Pipeline, text: Wtf8Text) -> Result<String, Error>
     rb_self.inner.process(&text).map_err(|e| map_err(&e))
 }
 
+/// `Disarm::Pipeline#purpose` — what the named profile is for, or `nil` (#860).
+fn pipeline_purpose(rb_self: &Pipeline) -> Option<String> {
+    rb_self.inner.purpose().map(str::to_owned)
+}
+
+/// `Disarm::Pipeline#_with_digit_policy(policy)` — a copy of this pipeline whose confusable
+/// passes fold under `policy` (#646). Fails when the profile has no confusables step and the
+/// policy is not the default: a setting that would never run is refused rather than kept.
+/// The Ruby layer wraps it as `#with_digit_policy` so the error arrives as
+/// `Disarm::InvalidArgument`, the way every module-level call does.
+fn pipeline_with_digit_policy(rb_self: &Pipeline, digit_policy: String) -> Result<Pipeline, Error> {
+    Ok(Pipeline {
+        inner: rb_self
+            .inner
+            .clone()
+            .with_digit_policy(parse_policy(&digit_policy)?)
+            .map_err(|e| map_err(&e))?,
+    })
+}
+
 /// `Disarm._get_pipeline(profile)` — build a reusable `Disarm::Pipeline` for a
 /// named policy profile. Fails (Disarm::InvalidArgument) on an unknown profile.
 fn get_pipeline(profile: String) -> Result<Pipeline, Error> {
@@ -837,6 +980,12 @@ fn get_pipeline(profile: String) -> Result<Pipeline, Error> {
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = ruby.define_module("Disarm")?;
 
+    // The zalgo defaults, read from the core so the Ruby layer never restates them: it
+    // kept a literal cap of 2 after #788 raised the core's to 3 (`formal/bindings`, B1).
+    // Defined before `lib/disarm.rb`'s method definitions read them as defaults.
+    module.const_set("DEFAULT_ZALGO_THRESHOLD", api::DEFAULT_ZALGO_THRESHOLD)?;
+    module.const_set("DEFAULT_ZALGO_MAX_MARKS", api::DEFAULT_ZALGO_MAX_MARKS)?;
+
     // Raw, `_`-prefixed shims wrapped by the idiomatic Ruby layer (#357).
     module.define_singleton_method("_transliterate", function!(transliterate, 1))?;
     module.define_singleton_method("_transliterate_opts", function!(transliterate_opts, 3))?;
@@ -847,25 +996,27 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     module.define_singleton_method("_confusable?", function!(is_confusable, 2))?;
     module.define_singleton_method("_slugify", function!(slugify, 13))?;
     module.define_singleton_method("_demojize", function!(demojize, 2))?;
-    module.define_singleton_method("_canonicalize_strict", function!(canonicalize_strict, 1))?;
+    module.define_singleton_method("_replace_emoji", function!(replace_emoji, 2))?;
+    module.define_singleton_method("_canonicalize_strict", function!(canonicalize_strict, 2))?;
     module.define_singleton_method("_strip_format", function!(strip_format, 1))?;
-    module.define_singleton_method("_strip_obfuscation", function!(strip_obfuscation, 1))?;
-    module.define_singleton_method("_canonicalize", function!(canonicalize, 1))?;
+    module.define_singleton_method("_strip_obfuscation", function!(strip_obfuscation, 2))?;
+    module.define_singleton_method("_canonicalize", function!(canonicalize, 2))?;
+    module.define_singleton_method("_is_canonical?", function!(is_canonical, 2))?;
 
     // Key-derivation presets (#404 Group A parity backfill).
-    module.define_singleton_method("_search_key", function!(search_key, 2))?;
-    module.define_singleton_method("_sort_key", function!(sort_key, 2))?;
-    module.define_singleton_method("_catalog_key", function!(catalog_key, 3))?;
+    module.define_singleton_method("_search_key", function!(search_key, 3))?;
+    module.define_singleton_method("_sort_key", function!(sort_key, 3))?;
+    module.define_singleton_method("_catalog_key", function!(catalog_key, 4))?;
+    module.define_singleton_method("_skeleton_key", function!(skeleton_key, 2))?;
+    module.define_singleton_method("_edit_distance", function!(edit_distance, 2))?;
+    module.define_singleton_method("_nearest_match", function!(nearest_match, 3))?;
 
     // No options / no symbols, but still wrapped by the Ruby layer so a wrong-type
     // argument surfaces as Disarm::InvalidArgument rather than a raw TypeError —
     // keeping `rescue Disarm::Error` exhaustive across the whole public surface.
     module.define_singleton_method("_strip_accents", function!(strip_accents, 1))?;
     module.define_singleton_method("_fold_case", function!(fold_case, 1))?;
-    module.define_singleton_method(
-        "_is_case_fold_stable?",
-        function!(is_case_fold_stable, 1),
-    )?;
+    module.define_singleton_method("_is_case_fold_stable?", function!(is_case_fold_stable, 1))?;
     module.define_singleton_method("_find_key_collisions", function!(find_key_collisions, 3))?;
     module.define_singleton_method("_suspicious_hostname?", function!(suspicious_hostname, 1))?;
     module.define_singleton_method("_analyze_hostname", function!(analyze_hostname, 2))?;
@@ -904,10 +1055,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
         function!(reverse_transliterate, 2),
     )?;
     module.define_singleton_method("_find_untranslatable", function!(find_untranslatable, 3))?;
-    module.define_singleton_method(
-        "_unmapped_confusables",
-        function!(unmapped_confusables, 1),
-    )?;
+    module.define_singleton_method("_unmapped_confusables", function!(unmapped_confusables, 1))?;
     module.define_singleton_method(
         "_find_unmapped_confusables",
         function!(find_unmapped_confusables, 2),
@@ -922,15 +1070,10 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     // Metadata introspection (#404 phase 3 parity backfill).
     module.define_singleton_method("_lang_info", function!(lang_info, 1))?;
     module.define_singleton_method("_script_info", function!(script_info, 1))?;
-    module.define_singleton_method(
-        "_confusables_version",
-        function!(confusables_version, 0),
-    )?;
+    module.define_singleton_method("_confusable_coverage", function!(confusable_coverage, 1))?;
+    module.define_singleton_method("_confusables_version", function!(confusables_version, 0))?;
     module.define_singleton_method("_unicode_version", function!(unicode_version, 0))?;
-    module.define_singleton_method(
-        "_key_schema_version",
-        function!(key_schema_version, 0),
-    )?;
+    module.define_singleton_method("_key_schema_version", function!(key_schema_version, 0))?;
     module.define_singleton_method("_list_scripts", function!(list_scripts, 0))?;
     module.define_singleton_method("_list_context_langs", function!(list_context_langs, 0))?;
 
@@ -954,6 +1097,8 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     // instance method on the wrapped handle. Mirrors `Disarm::Lexicon` above.
     let pipeline = module.define_class("Pipeline", ruby.class_object())?;
     pipeline.define_method("process", method!(pipeline_process, 1))?;
+    pipeline.define_method("purpose", method!(pipeline_purpose, 0))?;
+    pipeline.define_method("_with_digit_policy", method!(pipeline_with_digit_policy, 1))?;
     module.define_singleton_method("_get_pipeline", function!(get_pipeline, 1))?;
     Ok(())
 }

data/lib/disarm/version.rb

@@ -2,5 +2,5 @@
 
 module Disarm
   # Kept in lockstep with the Rust crate / Python package version.
-  VERSION = "0.15.0"
+  VERSION = "0.17.0"
 end

data/lib/disarm.rb

@@ -24,6 +24,13 @@ end
 # keyword arguments with the core's defaults, symbol tokens (:latin, :default, …),
 # a single transliterate(text, scheme:) entrypoint, and a Disarm::Error hierarchy.
 # Each method is still a thin wrapper over the pure-Rust `disarm` core.
+#
+# Text arguments are read by the encoding the String declares. UTF-8 and US-ASCII are
+# read as they are, and ASCII-8BIT (BINARY) is read as UTF-8, the way the C ABI reads
+# its bytes; any other encoding (ISO-8859-1, Windows-1251, UTF-16LE, ...) is transcoded
+# to UTF-8 first. A malformed or unmappable sequence becomes one U+FFFD, and so does a
+# lone surrogate, the contract every binding shares (#469). An encoding Ruby cannot
+# convert from at all raises Disarm::InvalidArgument.
 module Disarm
   # Base class for every error disarm raises, so consumers can `rescue
   # Disarm::Error`. The native shim raises Ruby's built-in ArgumentError /
@@ -38,7 +45,8 @@ module Disarm
     # Transliterate Unicode text to ASCII. `scheme:` selects the standard:
     # :default (the general-purpose scheme), :strict_iso9, or :gost7034. `lang:`
     # applies a language profile on top of the scheme (e.g. "uk" → Київ → "Kyiv",
-    # "de" → ü → "ue"); nil means no profile. Both accept a String or Symbol.
+    # "de" → ü → "ue"); nil means no profile. Both accept a String or Symbol. Raises
+    # Disarm::InvalidArgument on an unknown lang, as every binding does.
     def transliterate(text, scheme: :default, lang: nil)
       scheme = scheme.to_s
       lang = lang&.to_s
@@ -53,7 +61,8 @@ module Disarm
       end
     end
 
-    # Fold cross-script confusables toward `target:` (:latin or :cyrillic).
+    # Fold cross-script confusables toward `target:` (:latin, :cyrillic, :arabic or
+    # :hebrew).
     #
     # `digit_policy:` selects how non-Latin DIGITS fold (#561).
     #
@@ -61,9 +70,9 @@ module Disarm
     # right for prose, where a Devanagari zero really is a zero.
     #
     # `:tr39` uses upstream's targets, which send most of them to a Latin letter
-    # (`०` → `o`; three of the 45 rows fold to `.` or to the two characters `rn`
+    # (`०` → `o`; three of the 47 rows fold to `.` or to the two characters `rn`
     # instead). That is what an identifier *skeleton* wants, since its only job is to
-    # make two confusable identifiers collide. The two differ on 45 rows and agree
+    # make two confusable identifiers collide. The two differ on 47 rows and agree
     # everywhere else. Scoped to `target: :latin` — the override rows are generated from
     # the Latin table and carry TR39's Latin-script targets, so with `target: :cyrillic`
     # it is a no-op.
@@ -75,15 +84,16 @@ module Disarm
       translate_errors { _normalize_confusables(text, target.to_s, digit_policy.to_s) }
     end
 
-    # Whether `text` contains a character confusable with `target:` (:latin or
-    # :cyrillic).
+    # Whether `text` contains a character confusable with `target:` (:latin,
+    # :cyrillic, :arabic or :hebrew).
     def confusable?(text, target: :latin)
       translate_errors { _confusable?(text, target.to_s) }
     end
 
     # Generate a URL-safe slug. Mirrors the core's `SlugConfig` defaults; every
     # option past `text` is keyword-only. (`regex_pattern`/`replacements` are not
-    # surfaced yet — see ext/disarm/src/lib.rs.)
+    # surfaced yet — see ext/disarm/src/lib.rs.) Raises Disarm::InvalidArgument on an
+    # unknown `lang:`.
     def slugify(
       text,
       separator: "-",
@@ -111,16 +121,34 @@ module Disarm
     end
 
     # Replace emoji with their plain names (e.g. "👍" → "thumbs up").
-    # `strip_modifiers:` drops skin-tone / variation modifiers before naming.
+    # `strip_modifiers:` drops skin-tone / variation modifiers before naming. An emoji
+    # CLDR cannot name (a regional indicator or a Plane 14 tag character standing
+    # alone) becomes "[?]", the sentinel #transliterate writes, as in every binding.
     def demojize(text, strip_modifiers: false)
       translate_errors { _demojize(text, strip_modifiers) }
     end
 
+    # Replace every emoji with +replacement+, verbatim.
+    #
+    # The counterpart to +demojize+, and a different question of a different table.
+    # +demojize+ asks what CLDR calls a character, so its domain is the name table, which
+    # is wider than the emoji: <tt>demojize("x\u2122y")</tt> is "x trade mark y". This
+    # asks whether the UCD calls it an emoji — Emoji_Presentation=Yes, an Emoji=Yes base
+    # carrying U+FE0F (not an Extended_Pictographic one, so a black star with a selector
+    # stays), and the ZWJ, modifier, keycap and flag sequences on those. Nothing
+    # else moves.
+    #
+    # +replacement+ is inserted exactly as given: "" closes an intra-word split and " "
+    # keeps two words apart, and no rule serves both.
+    def replace_emoji(text, replacement = "")
+      translate_errors { _replace_emoji(text, replacement) }
+    end
+
     # Canonicalize, but raise rather than silently normalize a structural difference
     # away — the half of the pair that lets a caller reject input instead of comparing
     # a value the sender never wrote.
-    def canonicalize_strict(text)
-      translate_errors { _canonicalize_strict(text) }
+    def canonicalize_strict(text, digit_policy: :numeric)
+      translate_errors { _canonicalize_strict(text, digit_policy.to_s) }
     end
 
     # Strip the non-interchange and invisible classes while KEEPING the script.
@@ -131,13 +159,13 @@ module Disarm
     # keeps the VS15/VS16 presentation selectors after a base, which the naive chain
     # deletes, and it collapses TAB/LF to a space, which the primitives leave alone.
     def strip_format(text)
-      _strip_format(text)
+      translate_errors { _strip_format(text) }
     end
 
     # Remove obfuscation (zero-width, bidi, combining-mark abuse) while keeping
     # legible content.
-    def strip_obfuscation(text)
-      translate_errors { _strip_obfuscation(text) }
+    def strip_obfuscation(text, digit_policy: :numeric)
+      translate_errors { _strip_obfuscation(text, digit_policy.to_s) }
     end
 
     # Canonicalize text for security-sensitive comparison: strip obfuscation,
@@ -151,8 +179,8 @@ module Disarm
     # delimiter can leave here carrying one. #inspect_anomalies reports it as :confusable
     # WHEN the word also carries an ASCII letter, which is the gate that keeps ordinary
     # non-Latin text from firing; a delimiter-only string is not reported.
-    def canonicalize(text)
-      translate_errors { _canonicalize(text) }
+    def canonicalize(text, digit_policy: :numeric)
+      translate_errors { _canonicalize(text, digit_policy.to_s) }
     end
 
     # @deprecated Renamed to {#canonicalize} in 0.11 (the +_clean+ name
@@ -165,22 +193,65 @@ module Disarm
     # Case/accent/script-insensitive search lookup key. `lang:` applies a
     # language profile for transliteration (e.g. "ru", "uk"); nil means none.
     # Raises Disarm::InvalidArgument on an unknown lang.
-    def search_key(text, lang: nil)
-      translate_errors { _search_key(text, lang&.to_s) }
+    def search_key(text, lang: nil, digit_policy: :numeric)
+      translate_errors { _search_key(text, lang&.to_s, digit_policy.to_s) }
     end
 
     # Collation sort key (like #search_key, but keeps base accented characters
     # for correct ordering). `lang:` applies a language profile; nil means none.
     # Raises Disarm::InvalidArgument on an unknown lang.
-    def sort_key(text, lang: nil)
-      translate_errors { _sort_key(text, lang&.to_s) }
+    def sort_key(text, lang: nil, digit_policy: :numeric)
+      translate_errors { _sort_key(text, lang&.to_s, digit_policy.to_s) }
     end
 
     # Library catalog deduplication key (search_key plus confusable folding).
     # `lang:` applies a language profile; `strict_iso9:` selects the ISO 9:1995
     # Cyrillic scheme. Raises Disarm::InvalidArgument on an unknown lang.
-    def catalog_key(text, lang: nil, strict_iso9: false)
-      translate_errors { _catalog_key(text, lang&.to_s, strict_iso9) }
+    def catalog_key(text, lang: nil, strict_iso9: false, digit_policy: :numeric)
+      translate_errors { _catalog_key(text, lang&.to_s, strict_iso9, digit_policy.to_s) }
+    end
+
+    # The TR39 identifier skeleton plus the two prototype classes disarm keeps apart
+    # (#650). A spoof key: its only job is to make confusable identifiers collide, and its
+    # output is never for display. `digit_policy:` is `:numeric` (the letter half only),
+    # `:tr39` (adds `1 ≡ l` and `0 ≡ O`) or `:preserve` (a non-Latin numeral keeps its
+    # script).
+    def skeleton_key(text, digit_policy: :numeric)
+      translate_errors { _skeleton_key(text, digit_policy.to_s) }
+    end
+
+    # Levenshtein edit distance between `a` and `b`, in characters (#894). The one class
+    # of registry spoofing the confusable tables deliberately do not model — `paypa1`,
+    # `adm1n`. Canonicalize both sides first when composed and decomposed spellings
+    # should compare equal.
+    def edit_distance(left, right)
+      translate_errors { _edit_distance(left, right) }
+    end
+
+    # The candidate closest to `value`, as `{ value:, distance: }`, or nil beyond
+    # `max_distance:` (#894). Reports; it does not decide. An exact match is reported with
+    # distance 0, and ties go to the first candidate at the lowest distance.
+    def nearest_match(value, candidates, max_distance: 1)
+      hit = translate_errors do
+        # A non-collection is a wrong-typed argument, not a NoMethodError (N2).
+        raise ::TypeError, "candidates must be an Array or Enumerable" unless candidates.respond_to?(:map)
+
+        _nearest_match(value, candidates.map(&:to_s), max_distance)
+      end
+      hit && { value: hit[0], distance: hit[1] }
+    end
+
+    # Whether `text` is already its own canonical form under `preset:` (#730).
+    #
+    # The verification-path counterpart to the presets: text in, normalized text out is
+    # the generation path, and this is the question a caller asks about bytes that arrive
+    # already bound. `has_anomalies?` is not this predicate — 142,760 assigned code points are
+    # reported clean by the detector and are not their own canonical form.
+    #
+    # `preset:` is any name in the preset registry or any profile. Raises
+    # Disarm::InvalidArgument on an unknown one.
+    def canonical?(text, preset: "canonicalize")
+      translate_errors { _is_canonical?(text, preset.to_s) }
     end
 
     # Strip diacritics ("café" → "cafe").
@@ -325,15 +396,18 @@ module Disarm
       translate_errors { _strip_pua(text) }
     end
 
-    # Strip "zalgo" combining-mark stacking, keeping at most `max_marks:` (2)
-    # combining marks per base character.
-    def strip_zalgo(text, max_marks: 2)
+    # Strip "zalgo" combining-mark stacking, keeping at most `max_marks:` marks of each
+    # combining class on one base character. The default is the core's,
+    # DEFAULT_ZALGO_MAX_MARKS (3), equal to #zalgo?'s threshold (#788), so this never
+    # strips from text #zalgo? declines to flag.
+    def strip_zalgo(text, max_marks: DEFAULT_ZALGO_MAX_MARKS)
       translate_errors { _strip_zalgo(text, max_marks) }
     end
 
     # Whether `text` looks like zalgo: any base character carries more than
-    # `threshold:` (3) combining marks.
-    def zalgo?(text, threshold: 3)
+    # `threshold:` marks of one combining class. The default is the core's,
+    # DEFAULT_ZALGO_THRESHOLD (3).
+    def zalgo?(text, threshold: DEFAULT_ZALGO_THRESHOLD)
       translate_errors { _zalgo?(text, threshold) }
     end
 
@@ -369,13 +443,15 @@ module Disarm
     # Turn arbitrary text into a safe filename. `platform:` is :universal
     # (default), :windows, or :posix; `preserve_extension:` keeps the final
     # extension when truncating to `max_length:`. Raises Disarm::InvalidArgument
-    # on an unknown platform.
+    # on an unknown platform, or on a `separator:` that is not printable, non-space
+    # ASCII, contains a character illegal on the platform, or contains a path
+    # separator ("/" or "\\"). An empty separator is allowed.
     #
     # A safe *filename*, not a safe URL path segment. "%" is legal in a filename, so one
     # the caller typed is kept — sanitize_filename("..%2Fetc") returns "%2Fetc" — and a
     # consumer that percent-decodes the result must validate AFTER decoding. What this
-    # will not do is manufacture one: "%" never appears in the output unless it appeared
-    # in the input (#721).
+    # will not do is manufacture one: every "%" in the output is one the input
+    # contained, or part of the separator (#721).
     def sanitize_filename(text, separator: "_", max_length: 255, platform: :universal,
                           lang: nil, preserve_extension: true)
       translate_errors do
@@ -422,8 +498,9 @@ module Disarm
       end
     end
 
-    # ML/NLP normalization: NFKC → emoji→text → transliterate → strip accents →
-    # [case fold] → strip control → strip zero-width → collapse whitespace.
+    # ML/NLP normalization: resolve deletions → NFKC → emoji→text → transliterate →
+    # strip accents → emoji→text → [case fold] → strip control → strip zero-width →
+    # collapse whitespace → NFC.
     #
     # `fold_case:` defaults to true. Pass false in front of a CASED model — folding is
     # destructive, cannot be undone downstream, and an uncased evaluation harness cannot
@@ -460,7 +537,7 @@ module Disarm
     # anomaly detector's `bidi` kind reports nine of the twelve, holding back LRM, RLM
     # and ALM because a lone directional mark is ordinary in right-to-left text.
     def bidi_control?(text)
-      _has_bidi_control?(text)
+      translate_errors { _has_bidi_control?(text) }
     end
 
     # Explain how `lang: "auto"` detection resolves `text`: a hash with
@@ -486,6 +563,22 @@ module Disarm
       translate_errors { _script_info(name.to_s) }
     end
 
+    # TR39 confusable sources whose prototype is in +script+, and how many of those
+    # disarm's bundled tables fold. Returns a Hash with +:script+, +:sources+ and
+    # +:folded+ keys.
+    #
+    # The denominator +unmapped_confusables+ does not have: that measures one bundled
+    # table against the whole 6,565-source population, so a script disarm ships no table
+    # for reports a number determined by that absence rather than by its coverage.
+    #
+    # +:folded+ counts sources any bundled table reaches, not sources folded *toward*
+    # this script — Greek is 71 of 159, because the Latin table folds Greek letters that
+    # look Latin. A script disarm knows that TR39 never uses as a prototype returns 0 of
+    # 0. Raises Disarm::InvalidArgument on an unknown script.
+    def confusable_coverage(script)
+      translate_errors { _confusable_coverage(script.to_s) }
+    end
+
     # The Unicode `confusables.txt` release the bundled confusable tables were folded
     # from, e.g. "17.0.0". Not a Unicode version for the library as a whole — the
     # case-folding and width tables track different releases (see docs/provenance.md).
@@ -574,7 +667,9 @@ module Disarm
     #   pipe.process("Café") # => "cafe"
     #   pipe.process("Köln") # reuse the same handle
     #
-    # Disarm::Pipeline#process is the Rust-defined instance method on the handle.
+    # Disarm::Pipeline#process is the Rust-defined instance method on the handle, and
+    # Disarm::Pipeline#with_digit_policy(policy) returns a copy whose confusable passes
+    # fold under `policy` (#646); a profile with no confusables step refuses one.
     def get_pipeline(profile)
       translate_errors { _get_pipeline(profile.to_s) }
     end
@@ -603,16 +698,30 @@ module Disarm
     # whole surface. The original backtrace is preserved (passed as the third
     # `raise` argument) so the failing native call site stays visible. A bad
     # argument from the native layer can arrive as ArgumentError (an invalid
-    # scheme/target), TypeError (a non-String argument), or RangeError (e.g. a
-    # negative max_length) — all map to Disarm::InvalidArgument.
+    # scheme/target), TypeError (a non-String argument), RangeError (e.g. a
+    # negative max_length) or EncodingError (a String in an encoding Ruby cannot
+    # transcode to UTF-8) — all map to Disarm::InvalidArgument.
     def translate_errors
       yield
     rescue Error
       raise # already in our hierarchy — don't re-wrap
-    rescue ::ArgumentError, ::TypeError, ::RangeError => e
+    rescue ::ArgumentError, ::TypeError, ::RangeError, ::EncodingError => e
       raise InvalidArgument, e.message, e.backtrace
     rescue ::RuntimeError => e
       raise Error, e.message, e.backtrace
     end
   end
+
+  # The reusable handle `Disarm.get_pipeline` returns. `#process` is Rust-defined; this
+  # reopens the class for the one method that can fail, so its error arrives as
+  # `Disarm::InvalidArgument` the way every module-level call's does.
+  class Pipeline
+    # A copy of this pipeline whose confusable passes fold under `policy` (#646):
+    # `:numeric`, `:tr39` or `:preserve`. Raises Disarm::InvalidArgument when the profile
+    # has no confusables step and the policy is not the default — a setting that would
+    # never run is refused rather than kept.
+    def with_digit_policy(policy)
+      Disarm.send(:translate_errors) { _with_digit_policy(policy.to_s) }
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disarm
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.17.0
 platform: ruby
 authors:
 - Richard Quinn
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-02 00:00:00.000000000 Z
+date: 2026-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -130,7 +130,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="