disarm 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d75406993aaa2ac76a20429c567601447bcffa79708398444b83dc8b445ebf01
-  data.tar.gz: cfea208f2da86fd373ed82f4e25a9166277ffd1806c22614551418ddb20acb05
+  metadata.gz: 0b0ec44307d78ca53394c2b8aa21522f3e2e9a486c83bd536abdb8afd2b0ea7e
+  data.tar.gz: e4e5edead5532bc809a8b2c3e4389b02177246d0d5c783bf5b24b641e32bb576
 SHA512:
-  metadata.gz: 68a46c759f40421b154e1879500edecf0ced53931224e317b1f3dacfb5a03750ebe132bfe7b1efd7f4be21d384f4478bfc104d826e27675c85b61c79723b40a1
-  data.tar.gz: 93d71a7e1d5aac2f0a33e26a72e6658ad842a2408b4e744f70ed73bdc1d141b567b2b575ec241c8e738e2c54dc39d4f49309118b353f7985084226d089de7ca3
+  metadata.gz: 65a1b31c791a89e407a697af6d89b8f65e772803705e0cb12d3a8b458d4bae093c1dfc558dfa9749db10857dcafe521b221d28979e22f1a37d66326cd2d86fe6
+  data.tar.gz: a26f6cdfca9f10d678d5b8b56d243be22ac91a1d582d92a5fb6aee7d0f28b8abbe154da24e2bad1e924c1c5121f6c26baf937cba8f5caebc06cc954810d13458

data/ext/disarm/Cargo.toml

@@ -30,7 +30,7 @@ crate-type = ["cdylib"]
 # dep — not a path — so the gem is self-contained and the rb-sys-dock cross-gem
 # build (which mounts only this gem dir) can fetch it. Imported as `disarm_core`
 # because the package name `disarm` would otherwise clash with this crate.
-disarm_core = { package = "disarm", version = "0.15", default-features = false }
+disarm_core = { package = "disarm", version = "0.16", default-features = false }
 # magnus: ergonomic, safe Ruby<->Rust bindings over rb-sys. 0.8 supports Ruby >= 3.0
 # (the gem's required_ruby_version); rb-sys handles the platform glue.
 magnus = "0.8"

data/ext/disarm/src/lib.rs

@@ -215,6 +215,16 @@ fn normalize_confusables(
     Ok(api::normalize_confusables_with(&text, target, digit_policy).into_owned())
 }
 
+/// The `digit_policy` token every key builder takes (#896): parsed once, at the boundary.
+fn parse_policy(digit_policy: &str) -> Result<api::DigitPolicy, Error> {
+    digit_policy.parse().map_err(|e| map_err(&e))
+}
+
+/// `Disarm._is_canonical?(text, preset)` — already its own canonical form (#730).
+fn is_canonical(text: Wtf8Text, preset: String) -> Result<bool, Error> {
+    api::is_canonical(&text, &preset).map_err(|e| map_err(&e))
+}
+
 /// `Disarm._confusable?(text, "latin" | "cyrillic" | "arabic" | "hebrew")`.
 fn is_confusable(text: Wtf8Text, target: String) -> Result<bool, Error> {
     let target: api::TargetScript = target.parse().map_err(|e| map_err(&e))?;
@@ -321,6 +331,11 @@ fn slugify(
     api::slugify(&text, &config)
 }
 
+/// `Disarm._replace_emoji(text, replacement)` — every emoji replaced, verbatim (#972).
+fn replace_emoji(text: Wtf8Text, replacement: String) -> String {
+    api::replace_emoji(&text, &replacement)
+}
+
 /// `Disarm._demojize(text, strip_modifiers)`.
 fn demojize(text: Wtf8Text, strip_modifiers: bool) -> String {
     api::demojize(&text, strip_modifiers)
@@ -340,8 +355,8 @@ fn demojize(text: Wtf8Text, strip_modifiers: bool) -> String {
 ///
 /// Unlike `canonicalize` it does NOT fold confusables, so non-Latin text keeps its script
 /// — the point of the preset.
-fn canonicalize_strict(text: Wtf8Text) -> Result<String, Error> {
-    api::canonicalize_strict(&text)
+fn canonicalize_strict(text: Wtf8Text, digit_policy: String) -> Result<String, Error> {
+    api::canonicalize_strict_with(&text, parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
@@ -350,42 +365,72 @@ fn strip_format(text: Wtf8Text) -> String {
     api::strip_format(&text).into_owned()
 }
 
-fn strip_obfuscation(text: Wtf8Text) -> Result<String, Error> {
-    api::strip_obfuscation(&text)
+fn strip_obfuscation(text: Wtf8Text, digit_policy: String) -> Result<String, Error> {
+    api::strip_obfuscation_with(&text, parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
-fn canonicalize(text: Wtf8Text) -> Result<String, Error> {
-    api::canonicalize(&text)
+fn canonicalize(text: Wtf8Text, digit_policy: String) -> Result<String, Error> {
+    api::canonicalize_with(&text, parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
 /// `Disarm._search_key(text, lang)` — case/accent/script-insensitive lookup key.
 /// `lang` is `nil` (no profile) or a code like `"ru"`. Fails on an unknown `lang`.
-fn search_key(text: Wtf8Text, lang: Option<String>) -> Result<String, Error> {
-    api::search_key(&text, lang.as_deref())
+fn search_key(text: Wtf8Text, lang: Option<String>, digit_policy: String) -> Result<String, Error> {
+    api::search_key_with(&text, lang.as_deref(), parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
 /// `Disarm._sort_key(text, lang)` — collation sort key (preserves base accented
 /// characters for correct ordering). Fails on an unknown `lang`.
-fn sort_key(text: Wtf8Text, lang: Option<String>) -> Result<String, Error> {
-    api::sort_key(&text, lang.as_deref())
+fn sort_key(text: Wtf8Text, lang: Option<String>, digit_policy: String) -> Result<String, Error> {
+    api::sort_key_with(&text, lang.as_deref(), parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
 /// `Disarm._catalog_key(text, lang, strict_iso9)` — catalog deduplication key.
 /// `strict_iso9` selects the ISO 9:1995 Cyrillic scheme. Fails on an unknown `lang`.
-fn catalog_key(text: Wtf8Text, lang: Option<String>, strict_iso9: bool) -> Result<String, Error> {
-    api::catalog_key(&text, lang.as_deref(), strict_iso9)
+fn catalog_key(
+    text: Wtf8Text,
+    lang: Option<String>,
+    strict_iso9: bool,
+    digit_policy: String,
+) -> Result<String, Error> {
+    api::catalog_key_with(&text, lang.as_deref(), strict_iso9, parse_policy(&digit_policy)?)
+        .map(std::borrow::Cow::into_owned)
+        .map_err(|e| map_err(&e))
+}
+
+/// `Disarm._skeleton_key(text, digit_policy)` — the TR39 identifier skeleton plus the two
+/// prototype classes disarm's table keeps apart (#650). A spoof key: never for display.
+fn skeleton_key(text: Wtf8Text, digit_policy: String) -> Result<String, Error> {
+    api::skeleton_key(&text, parse_policy(&digit_policy)?)
         .map(std::borrow::Cow::into_owned)
         .map_err(|e| map_err(&e))
 }
 
+/// `Disarm._edit_distance(a, b)` — Levenshtein distance in characters (#894).
+fn edit_distance(a: Wtf8Text, b: Wtf8Text) -> usize {
+    api::edit_distance(&a, &b)
+}
+
+/// `Disarm._nearest_match(value, candidates, max_distance)` — the closest candidate and
+/// its distance as a `(value, distance)` tuple, or `nil` beyond `max_distance` (#894).
+/// The Ruby layer maps the tuple to a `{ value:, distance: }` hash.
+fn nearest_match(
+    value: Wtf8Text,
+    candidates: Vec<String>,
+    max_distance: usize,
+) -> Option<(String, usize)> {
+    api::nearest_match(&value, candidates.iter().map(String::as_str), max_distance)
+        .map(|m| (m.value, m.distance))
+}
+
 /// `Disarm._suspicious_hostname?(host)` — flags mixed-script / confusable IDN
 /// spoofs. A false result asserts nothing was *found*, not that the host is safe.
 fn suspicious_hostname(host: Wtf8Text) -> bool {
@@ -670,6 +715,22 @@ fn lang_info(code: String) -> Result<RHash, Error> {
     Ok(hash)
 }
 
+/// `Disarm._confusable_coverage(script)` — TR39 sources whose prototype is in `script`
+/// and how many the bundled tables fold (#963), as a Ruby Hash with symbol keys
+/// (`{ script:, sources:, folded: }`). The denominator `unmapped_confusables` does not
+/// have. Fails (ArgumentError → Disarm::InvalidArgument) on an unknown script.
+fn confusable_coverage(script: String) -> Result<RHash, Error> {
+    let row = api::confusable_coverage(&script).map_err(|e| map_err(&e))?;
+    // GVL invariant: a Ruby method callback always holds the GVL. Justified.
+    #[allow(clippy::expect_used)]
+    let ruby = Ruby::get().expect("a Ruby method callback always holds the GVL");
+    let hash = ruby.hash_new();
+    hash.aset(ruby.to_symbol("script"), row.script)?;
+    hash.aset(ruby.to_symbol("sources"), row.sources)?;
+    hash.aset(ruby.to_symbol("folded"), row.folded)?;
+    Ok(hash)
+}
+
 /// `Disarm._script_info(name)` — curated metadata for one script as a Ruby Hash
 /// with symbol keys (`{ name:, default_lang:, example:, context_aware: }`);
 /// `default_lang` is `nil` when the core has none. Fails (ArgumentError →
@@ -823,6 +884,26 @@ fn pipeline_process(rb_self: &Pipeline, text: Wtf8Text) -> Result<String, Error>
     rb_self.inner.process(&text).map_err(|e| map_err(&e))
 }
 
+/// `Disarm::Pipeline#purpose` — what the named profile is for, or `nil` (#860).
+fn pipeline_purpose(rb_self: &Pipeline) -> Option<String> {
+    rb_self.inner.purpose().map(str::to_owned)
+}
+
+/// `Disarm::Pipeline#_with_digit_policy(policy)` — a copy of this pipeline whose confusable
+/// passes fold under `policy` (#646). Fails when the profile has no confusables step and the
+/// policy is not the default: a setting that would never run is refused rather than kept.
+/// The Ruby layer wraps it as `#with_digit_policy` so the error arrives as
+/// `Disarm::InvalidArgument`, the way every module-level call does.
+fn pipeline_with_digit_policy(rb_self: &Pipeline, digit_policy: String) -> Result<Pipeline, Error> {
+    Ok(Pipeline {
+        inner: rb_self
+            .inner
+            .clone()
+            .with_digit_policy(parse_policy(&digit_policy)?)
+            .map_err(|e| map_err(&e))?,
+    })
+}
+
 /// `Disarm._get_pipeline(profile)` — build a reusable `Disarm::Pipeline` for a
 /// named policy profile. Fails (Disarm::InvalidArgument) on an unknown profile.
 fn get_pipeline(profile: String) -> Result<Pipeline, Error> {
@@ -847,15 +928,20 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     module.define_singleton_method("_confusable?", function!(is_confusable, 2))?;
     module.define_singleton_method("_slugify", function!(slugify, 13))?;
     module.define_singleton_method("_demojize", function!(demojize, 2))?;
-    module.define_singleton_method("_canonicalize_strict", function!(canonicalize_strict, 1))?;
+    module.define_singleton_method("_replace_emoji", function!(replace_emoji, 2))?;
+    module.define_singleton_method("_canonicalize_strict", function!(canonicalize_strict, 2))?;
     module.define_singleton_method("_strip_format", function!(strip_format, 1))?;
-    module.define_singleton_method("_strip_obfuscation", function!(strip_obfuscation, 1))?;
-    module.define_singleton_method("_canonicalize", function!(canonicalize, 1))?;
+    module.define_singleton_method("_strip_obfuscation", function!(strip_obfuscation, 2))?;
+    module.define_singleton_method("_canonicalize", function!(canonicalize, 2))?;
+    module.define_singleton_method("_is_canonical?", function!(is_canonical, 2))?;
 
     // Key-derivation presets (#404 Group A parity backfill).
-    module.define_singleton_method("_search_key", function!(search_key, 2))?;
-    module.define_singleton_method("_sort_key", function!(sort_key, 2))?;
-    module.define_singleton_method("_catalog_key", function!(catalog_key, 3))?;
+    module.define_singleton_method("_search_key", function!(search_key, 3))?;
+    module.define_singleton_method("_sort_key", function!(sort_key, 3))?;
+    module.define_singleton_method("_catalog_key", function!(catalog_key, 4))?;
+    module.define_singleton_method("_skeleton_key", function!(skeleton_key, 2))?;
+    module.define_singleton_method("_edit_distance", function!(edit_distance, 2))?;
+    module.define_singleton_method("_nearest_match", function!(nearest_match, 3))?;
 
     // No options / no symbols, but still wrapped by the Ruby layer so a wrong-type
     // argument surfaces as Disarm::InvalidArgument rather than a raw TypeError —
@@ -922,6 +1008,10 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     // Metadata introspection (#404 phase 3 parity backfill).
     module.define_singleton_method("_lang_info", function!(lang_info, 1))?;
     module.define_singleton_method("_script_info", function!(script_info, 1))?;
+    module.define_singleton_method(
+        "_confusable_coverage",
+        function!(confusable_coverage, 1),
+    )?;
     module.define_singleton_method(
         "_confusables_version",
         function!(confusables_version, 0),
@@ -954,6 +1044,8 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     // instance method on the wrapped handle. Mirrors `Disarm::Lexicon` above.
     let pipeline = module.define_class("Pipeline", ruby.class_object())?;
     pipeline.define_method("process", method!(pipeline_process, 1))?;
+    pipeline.define_method("purpose", method!(pipeline_purpose, 0))?;
+    pipeline.define_method("_with_digit_policy", method!(pipeline_with_digit_policy, 1))?;
     module.define_singleton_method("_get_pipeline", function!(get_pipeline, 1))?;
     Ok(())
 }

data/lib/disarm/version.rb

@@ -2,5 +2,5 @@
 
 module Disarm
   # Kept in lockstep with the Rust crate / Python package version.
-  VERSION = "0.15.0"
+  VERSION = "0.16.0"
 end

data/lib/disarm.rb

@@ -116,11 +116,26 @@ module Disarm
       translate_errors { _demojize(text, strip_modifiers) }
     end
 
+    # Replace every emoji with +replacement+, verbatim.
+    #
+    # The counterpart to +demojize+, and a different question of a different table.
+    # +demojize+ asks what CLDR calls a character, so its domain is the name table, which
+    # is wider than the emoji: <tt>demojize("x\u2122y")</tt> is "x trade mark y". This
+    # asks whether the UCD calls it an emoji — Emoji_Presentation=Yes, an Emoji=Yes base
+    # carrying U+FE0F, and the ZWJ, modifier, keycap and flag sequences on those. Nothing
+    # else moves.
+    #
+    # +replacement+ is inserted exactly as given: "" closes an intra-word split and " "
+    # keeps two words apart, and no rule serves both.
+    def replace_emoji(text, replacement = "")
+      translate_errors { _replace_emoji(text, replacement) }
+    end
+
     # Canonicalize, but raise rather than silently normalize a structural difference
     # away — the half of the pair that lets a caller reject input instead of comparing
     # a value the sender never wrote.
-    def canonicalize_strict(text)
-      translate_errors { _canonicalize_strict(text) }
+    def canonicalize_strict(text, digit_policy: :numeric)
+      translate_errors { _canonicalize_strict(text, digit_policy.to_s) }
     end
 
     # Strip the non-interchange and invisible classes while KEEPING the script.
@@ -136,8 +151,8 @@ module Disarm
 
     # Remove obfuscation (zero-width, bidi, combining-mark abuse) while keeping
     # legible content.
-    def strip_obfuscation(text)
-      translate_errors { _strip_obfuscation(text) }
+    def strip_obfuscation(text, digit_policy: :numeric)
+      translate_errors { _strip_obfuscation(text, digit_policy.to_s) }
     end
 
     # Canonicalize text for security-sensitive comparison: strip obfuscation,
@@ -151,8 +166,8 @@ module Disarm
     # delimiter can leave here carrying one. #inspect_anomalies reports it as :confusable
     # WHEN the word also carries an ASCII letter, which is the gate that keeps ordinary
     # non-Latin text from firing; a delimiter-only string is not reported.
-    def canonicalize(text)
-      translate_errors { _canonicalize(text) }
+    def canonicalize(text, digit_policy: :numeric)
+      translate_errors { _canonicalize(text, digit_policy.to_s) }
     end
 
     # @deprecated Renamed to {#canonicalize} in 0.11 (the +_clean+ name
@@ -165,22 +180,60 @@ module Disarm
     # Case/accent/script-insensitive search lookup key. `lang:` applies a
     # language profile for transliteration (e.g. "ru", "uk"); nil means none.
     # Raises Disarm::InvalidArgument on an unknown lang.
-    def search_key(text, lang: nil)
-      translate_errors { _search_key(text, lang&.to_s) }
+    def search_key(text, lang: nil, digit_policy: :numeric)
+      translate_errors { _search_key(text, lang&.to_s, digit_policy.to_s) }
     end
 
     # Collation sort key (like #search_key, but keeps base accented characters
     # for correct ordering). `lang:` applies a language profile; nil means none.
     # Raises Disarm::InvalidArgument on an unknown lang.
-    def sort_key(text, lang: nil)
-      translate_errors { _sort_key(text, lang&.to_s) }
+    def sort_key(text, lang: nil, digit_policy: :numeric)
+      translate_errors { _sort_key(text, lang&.to_s, digit_policy.to_s) }
     end
 
     # Library catalog deduplication key (search_key plus confusable folding).
     # `lang:` applies a language profile; `strict_iso9:` selects the ISO 9:1995
     # Cyrillic scheme. Raises Disarm::InvalidArgument on an unknown lang.
-    def catalog_key(text, lang: nil, strict_iso9: false)
-      translate_errors { _catalog_key(text, lang&.to_s, strict_iso9) }
+    def catalog_key(text, lang: nil, strict_iso9: false, digit_policy: :numeric)
+      translate_errors { _catalog_key(text, lang&.to_s, strict_iso9, digit_policy.to_s) }
+    end
+
+    # The TR39 identifier skeleton plus the two prototype classes disarm keeps apart
+    # (#650). A spoof key: its only job is to make confusable identifiers collide, and its
+    # output is never for display. `digit_policy:` is `:numeric` (the letter half only),
+    # `:tr39` (adds `1 ≡ l` and `0 ≡ O`) or `:preserve` (a non-Latin numeral keeps its
+    # script).
+    def skeleton_key(text, digit_policy: :numeric)
+      translate_errors { _skeleton_key(text, digit_policy.to_s) }
+    end
+
+    # Levenshtein edit distance between `a` and `b`, in characters (#894). The one class
+    # of registry spoofing the confusable tables deliberately do not model — `paypa1`,
+    # `adm1n`. Canonicalize both sides first when composed and decomposed spellings
+    # should compare equal.
+    def edit_distance(left, right)
+      translate_errors { _edit_distance(left, right) }
+    end
+
+    # The candidate closest to `value`, as `{ value:, distance: }`, or nil beyond
+    # `max_distance:` (#894). Reports; it does not decide. An exact match is reported with
+    # distance 0, and ties go to the first candidate at the lowest distance.
+    def nearest_match(value, candidates, max_distance: 1)
+      hit = translate_errors { _nearest_match(value, candidates.map(&:to_s), max_distance) }
+      hit && { value: hit[0], distance: hit[1] }
+    end
+
+    # Whether `text` is already its own canonical form under `preset:` (#730).
+    #
+    # The verification-path counterpart to the presets: text in, normalized text out is
+    # the generation path, and this is the question a caller asks about bytes that arrive
+    # already bound. `has_anomalies?` is not this predicate — 142,760 assigned code points are
+    # reported clean by the detector and are not their own canonical form.
+    #
+    # `preset:` is any name in the preset registry or any profile. Raises
+    # Disarm::InvalidArgument on an unknown one.
+    def canonical?(text, preset: "canonicalize")
+      translate_errors { _is_canonical?(text, preset.to_s) }
     end
 
     # Strip diacritics ("café" → "cafe").
@@ -486,6 +539,22 @@ module Disarm
       translate_errors { _script_info(name.to_s) }
     end
 
+    # TR39 confusable sources whose prototype is in +script+, and how many of those
+    # disarm's bundled tables fold. Returns a Hash with +:script+, +:sources+ and
+    # +:folded+ keys.
+    #
+    # The denominator +unmapped_confusables+ does not have: that measures one bundled
+    # table against the whole 6,565-source population, so a script disarm ships no table
+    # for reports a number determined by that absence rather than by its coverage.
+    #
+    # +:folded+ counts sources any bundled table reaches, not sources folded *toward*
+    # this script — Greek is 71 of 159, because the Latin table folds Greek letters that
+    # look Latin. A script disarm knows that TR39 never uses as a prototype returns 0 of
+    # 0. Raises Disarm::InvalidArgument on an unknown script.
+    def confusable_coverage(script)
+      translate_errors { _confusable_coverage(script.to_s) }
+    end
+
     # The Unicode `confusables.txt` release the bundled confusable tables were folded
     # from, e.g. "17.0.0". Not a Unicode version for the library as a whole — the
     # case-folding and width tables track different releases (see docs/provenance.md).
@@ -574,7 +643,9 @@ module Disarm
     #   pipe.process("Café") # => "cafe"
     #   pipe.process("Köln") # reuse the same handle
     #
-    # Disarm::Pipeline#process is the Rust-defined instance method on the handle.
+    # Disarm::Pipeline#process is the Rust-defined instance method on the handle, and
+    # Disarm::Pipeline#with_digit_policy(policy) returns a copy whose confusable passes
+    # fold under `policy` (#646); a profile with no confusables step refuses one.
     def get_pipeline(profile)
       translate_errors { _get_pipeline(profile.to_s) }
     end
@@ -615,4 +686,17 @@ module Disarm
       raise Error, e.message, e.backtrace
     end
   end
+
+  # The reusable handle `Disarm.get_pipeline` returns. `#process` is Rust-defined; this
+  # reopens the class for the one method that can fail, so its error arrives as
+  # `Disarm::InvalidArgument` the way every module-level call's does.
+  class Pipeline
+    # A copy of this pipeline whose confusable passes fold under `policy` (#646):
+    # `:numeric`, `:tr39` or `:preserve`. Raises Disarm::InvalidArgument when the profile
+    # has no confusables step and the policy is not the default — a setting that would
+    # never run is refused rather than kept.
+    def with_digit_policy(policy)
+      Disarm.send(:translate_errors) { _with_digit_policy(policy.to_s) }
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disarm
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Richard Quinn
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-02 00:00:00.000000000 Z
+date: 2026-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys