disarm 0.15.0-aarch64-linux → 0.17.0-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: affa6ae8c002469f386f0cc010093af423746f17b2b23f33cb8e5baeef7d460a
-  data.tar.gz: 8c04e95811cc9608f473a44c85815c2e37332c5c71d489963a236b83e1a405e4
+  metadata.gz: 0f8710ef6fd0f70a94b8cef89b07ba9e3ab600baf666dcfeb78eafb931c1a846
+  data.tar.gz: 20388fac65395ab997fc2ab9e8bdc00685aa12202fea170886be626fdea96e27
 SHA512:
-  metadata.gz: 2102a383ea426986f32d6bbc2c1aa52970226e9e7ed135b5d6f0bd34102c060b5076dd12071985e695ccf3638499930ee4d2ad8b414f0fbc5d4871de38d82bbd
-  data.tar.gz: 70976043387c5e0d0cf23c5ced76e0bb58af9cae186f03937963ff08f98e225f90443e4077f93d4cb56e7a27d29812ca37bea3bc4a21fb9ac2f97ca2ec849cab
+  metadata.gz: 996597cb752b703dd8e83f7fe476fc3d0622b93911335eca50674a6ea2afffcdab4d048a1f97d48b83f7ee269f7b69a20185ff447cf78d44b109b5667744f462
+  data.tar.gz: f8b0788a1900b1cee4840ac387d6917aff961ca17e75f2d9b28902a000ea1fcfc97eb3c7a7eaa97b87b769dfe27e2bcb42481b876d2f5da115c6457415b45ccc

data/README.md

@@ -19,10 +19,10 @@ gem "disarm"
 gem install disarm
 ```
 
-Requires Ruby >= 3.1. Precompiled platform gems ship for Ruby 3.1 through 4.0
+Requires Ruby >= 3.3. Precompiled platform gems ship for Ruby 3.3 through 4.0
 (Linux x86_64/aarch64, macOS x86_64/arm64, Windows). On a supported Ruby with no
 matching platform gem, the source gem installs and compiles locally, which needs a
-Rust toolchain. Below 3.1 the gem does not install at all.
+Rust toolchain. Below 3.3 the gem does not install at all.
 
 ## Usage
 

data/lib/disarm/version.rb

@@ -2,5 +2,5 @@
 
 module Disarm
   # Kept in lockstep with the Rust crate / Python package version.
-  VERSION = "0.15.0"
+  VERSION = "0.17.0"
 end

data/lib/disarm.rb

@@ -24,6 +24,13 @@ end
 # keyword arguments with the core's defaults, symbol tokens (:latin, :default, …),
 # a single transliterate(text, scheme:) entrypoint, and a Disarm::Error hierarchy.
 # Each method is still a thin wrapper over the pure-Rust `disarm` core.
+#
+# Text arguments are read by the encoding the String declares. UTF-8 and US-ASCII are
+# read as they are, and ASCII-8BIT (BINARY) is read as UTF-8, the way the C ABI reads
+# its bytes; any other encoding (ISO-8859-1, Windows-1251, UTF-16LE, ...) is transcoded
+# to UTF-8 first. A malformed or unmappable sequence becomes one U+FFFD, and so does a
+# lone surrogate, the contract every binding shares (#469). An encoding Ruby cannot
+# convert from at all raises Disarm::InvalidArgument.
 module Disarm
   # Base class for every error disarm raises, so consumers can `rescue
   # Disarm::Error`. The native shim raises Ruby's built-in ArgumentError /
@@ -38,7 +45,8 @@ module Disarm
     # Transliterate Unicode text to ASCII. `scheme:` selects the standard:
     # :default (the general-purpose scheme), :strict_iso9, or :gost7034. `lang:`
     # applies a language profile on top of the scheme (e.g. "uk" → Київ → "Kyiv",
-    # "de" → ü → "ue"); nil means no profile. Both accept a String or Symbol.
+    # "de" → ü → "ue"); nil means no profile. Both accept a String or Symbol. Raises
+    # Disarm::InvalidArgument on an unknown lang, as every binding does.
     def transliterate(text, scheme: :default, lang: nil)
       scheme = scheme.to_s
       lang = lang&.to_s
@@ -53,7 +61,8 @@ module Disarm
       end
     end
 
-    # Fold cross-script confusables toward `target:` (:latin or :cyrillic).
+    # Fold cross-script confusables toward `target:` (:latin, :cyrillic, :arabic or
+    # :hebrew).
     #
     # `digit_policy:` selects how non-Latin DIGITS fold (#561).
     #
@@ -61,9 +70,9 @@ module Disarm
     # right for prose, where a Devanagari zero really is a zero.
     #
     # `:tr39` uses upstream's targets, which send most of them to a Latin letter
-    # (`०` → `o`; three of the 45 rows fold to `.` or to the two characters `rn`
+    # (`०` → `o`; three of the 47 rows fold to `.` or to the two characters `rn`
     # instead). That is what an identifier *skeleton* wants, since its only job is to
-    # make two confusable identifiers collide. The two differ on 45 rows and agree
+    # make two confusable identifiers collide. The two differ on 47 rows and agree
     # everywhere else. Scoped to `target: :latin` — the override rows are generated from
     # the Latin table and carry TR39's Latin-script targets, so with `target: :cyrillic`
     # it is a no-op.
@@ -75,15 +84,16 @@ module Disarm
       translate_errors { _normalize_confusables(text, target.to_s, digit_policy.to_s) }
     end
 
-    # Whether `text` contains a character confusable with `target:` (:latin or
-    # :cyrillic).
+    # Whether `text` contains a character confusable with `target:` (:latin,
+    # :cyrillic, :arabic or :hebrew).
     def confusable?(text, target: :latin)
       translate_errors { _confusable?(text, target.to_s) }
     end
 
     # Generate a URL-safe slug. Mirrors the core's `SlugConfig` defaults; every
     # option past `text` is keyword-only. (`regex_pattern`/`replacements` are not
-    # surfaced yet — see ext/disarm/src/lib.rs.)
+    # surfaced yet — see ext/disarm/src/lib.rs.) Raises Disarm::InvalidArgument on an
+    # unknown `lang:`.
     def slugify(
       text,
       separator: "-",
@@ -111,16 +121,34 @@ module Disarm
     end
 
     # Replace emoji with their plain names (e.g. "👍" → "thumbs up").
-    # `strip_modifiers:` drops skin-tone / variation modifiers before naming.
+    # `strip_modifiers:` drops skin-tone / variation modifiers before naming. An emoji
+    # CLDR cannot name (a regional indicator or a Plane 14 tag character standing
+    # alone) becomes "[?]", the sentinel #transliterate writes, as in every binding.
     def demojize(text, strip_modifiers: false)
       translate_errors { _demojize(text, strip_modifiers) }
     end
 
+    # Replace every emoji with +replacement+, verbatim.
+    #
+    # The counterpart to +demojize+, and a different question of a different table.
+    # +demojize+ asks what CLDR calls a character, so its domain is the name table, which
+    # is wider than the emoji: <tt>demojize("x\u2122y")</tt> is "x trade mark y". This
+    # asks whether the UCD calls it an emoji — Emoji_Presentation=Yes, an Emoji=Yes base
+    # carrying U+FE0F (not an Extended_Pictographic one, so a black star with a selector
+    # stays), and the ZWJ, modifier, keycap and flag sequences on those. Nothing
+    # else moves.
+    #
+    # +replacement+ is inserted exactly as given: "" closes an intra-word split and " "
+    # keeps two words apart, and no rule serves both.
+    def replace_emoji(text, replacement = "")
+      translate_errors { _replace_emoji(text, replacement) }
+    end
+
     # Canonicalize, but raise rather than silently normalize a structural difference
     # away — the half of the pair that lets a caller reject input instead of comparing
     # a value the sender never wrote.
-    def canonicalize_strict(text)
-      translate_errors { _canonicalize_strict(text) }
+    def canonicalize_strict(text, digit_policy: :numeric)
+      translate_errors { _canonicalize_strict(text, digit_policy.to_s) }
     end
 
     # Strip the non-interchange and invisible classes while KEEPING the script.
@@ -131,13 +159,13 @@ module Disarm
     # keeps the VS15/VS16 presentation selectors after a base, which the naive chain
     # deletes, and it collapses TAB/LF to a space, which the primitives leave alone.
     def strip_format(text)
-      _strip_format(text)
+      translate_errors { _strip_format(text) }
     end
 
     # Remove obfuscation (zero-width, bidi, combining-mark abuse) while keeping
     # legible content.
-    def strip_obfuscation(text)
-      translate_errors { _strip_obfuscation(text) }
+    def strip_obfuscation(text, digit_policy: :numeric)
+      translate_errors { _strip_obfuscation(text, digit_policy.to_s) }
     end
 
     # Canonicalize text for security-sensitive comparison: strip obfuscation,
@@ -151,8 +179,8 @@ module Disarm
     # delimiter can leave here carrying one. #inspect_anomalies reports it as :confusable
     # WHEN the word also carries an ASCII letter, which is the gate that keeps ordinary
     # non-Latin text from firing; a delimiter-only string is not reported.
-    def canonicalize(text)
-      translate_errors { _canonicalize(text) }
+    def canonicalize(text, digit_policy: :numeric)
+      translate_errors { _canonicalize(text, digit_policy.to_s) }
     end
 
     # @deprecated Renamed to {#canonicalize} in 0.11 (the +_clean+ name
@@ -165,22 +193,65 @@ module Disarm
     # Case/accent/script-insensitive search lookup key. `lang:` applies a
     # language profile for transliteration (e.g. "ru", "uk"); nil means none.
     # Raises Disarm::InvalidArgument on an unknown lang.
-    def search_key(text, lang: nil)
-      translate_errors { _search_key(text, lang&.to_s) }
+    def search_key(text, lang: nil, digit_policy: :numeric)
+      translate_errors { _search_key(text, lang&.to_s, digit_policy.to_s) }
     end
 
     # Collation sort key (like #search_key, but keeps base accented characters
     # for correct ordering). `lang:` applies a language profile; nil means none.
     # Raises Disarm::InvalidArgument on an unknown lang.
-    def sort_key(text, lang: nil)
-      translate_errors { _sort_key(text, lang&.to_s) }
+    def sort_key(text, lang: nil, digit_policy: :numeric)
+      translate_errors { _sort_key(text, lang&.to_s, digit_policy.to_s) }
     end
 
     # Library catalog deduplication key (search_key plus confusable folding).
     # `lang:` applies a language profile; `strict_iso9:` selects the ISO 9:1995
     # Cyrillic scheme. Raises Disarm::InvalidArgument on an unknown lang.
-    def catalog_key(text, lang: nil, strict_iso9: false)
-      translate_errors { _catalog_key(text, lang&.to_s, strict_iso9) }
+    def catalog_key(text, lang: nil, strict_iso9: false, digit_policy: :numeric)
+      translate_errors { _catalog_key(text, lang&.to_s, strict_iso9, digit_policy.to_s) }
+    end
+
+    # The TR39 identifier skeleton plus the two prototype classes disarm keeps apart
+    # (#650). A spoof key: its only job is to make confusable identifiers collide, and its
+    # output is never for display. `digit_policy:` is `:numeric` (the letter half only),
+    # `:tr39` (adds `1 ≡ l` and `0 ≡ O`) or `:preserve` (a non-Latin numeral keeps its
+    # script).
+    def skeleton_key(text, digit_policy: :numeric)
+      translate_errors { _skeleton_key(text, digit_policy.to_s) }
+    end
+
+    # Levenshtein edit distance between `a` and `b`, in characters (#894). The one class
+    # of registry spoofing the confusable tables deliberately do not model — `paypa1`,
+    # `adm1n`. Canonicalize both sides first when composed and decomposed spellings
+    # should compare equal.
+    def edit_distance(left, right)
+      translate_errors { _edit_distance(left, right) }
+    end
+
+    # The candidate closest to `value`, as `{ value:, distance: }`, or nil beyond
+    # `max_distance:` (#894). Reports; it does not decide. An exact match is reported with
+    # distance 0, and ties go to the first candidate at the lowest distance.
+    def nearest_match(value, candidates, max_distance: 1)
+      hit = translate_errors do
+        # A non-collection is a wrong-typed argument, not a NoMethodError (N2).
+        raise ::TypeError, "candidates must be an Array or Enumerable" unless candidates.respond_to?(:map)
+
+        _nearest_match(value, candidates.map(&:to_s), max_distance)
+      end
+      hit && { value: hit[0], distance: hit[1] }
+    end
+
+    # Whether `text` is already its own canonical form under `preset:` (#730).
+    #
+    # The verification-path counterpart to the presets: text in, normalized text out is
+    # the generation path, and this is the question a caller asks about bytes that arrive
+    # already bound. `has_anomalies?` is not this predicate — 142,760 assigned code points are
+    # reported clean by the detector and are not their own canonical form.
+    #
+    # `preset:` is any name in the preset registry or any profile. Raises
+    # Disarm::InvalidArgument on an unknown one.
+    def canonical?(text, preset: "canonicalize")
+      translate_errors { _is_canonical?(text, preset.to_s) }
     end
 
     # Strip diacritics ("café" → "cafe").
@@ -325,15 +396,18 @@ module Disarm
       translate_errors { _strip_pua(text) }
     end
 
-    # Strip "zalgo" combining-mark stacking, keeping at most `max_marks:` (2)
-    # combining marks per base character.
-    def strip_zalgo(text, max_marks: 2)
+    # Strip "zalgo" combining-mark stacking, keeping at most `max_marks:` marks of each
+    # combining class on one base character. The default is the core's,
+    # DEFAULT_ZALGO_MAX_MARKS (3), equal to #zalgo?'s threshold (#788), so this never
+    # strips from text #zalgo? declines to flag.
+    def strip_zalgo(text, max_marks: DEFAULT_ZALGO_MAX_MARKS)
       translate_errors { _strip_zalgo(text, max_marks) }
     end
 
     # Whether `text` looks like zalgo: any base character carries more than
-    # `threshold:` (3) combining marks.
-    def zalgo?(text, threshold: 3)
+    # `threshold:` marks of one combining class. The default is the core's,
+    # DEFAULT_ZALGO_THRESHOLD (3).
+    def zalgo?(text, threshold: DEFAULT_ZALGO_THRESHOLD)
       translate_errors { _zalgo?(text, threshold) }
     end
 
@@ -369,13 +443,15 @@ module Disarm
     # Turn arbitrary text into a safe filename. `platform:` is :universal
     # (default), :windows, or :posix; `preserve_extension:` keeps the final
     # extension when truncating to `max_length:`. Raises Disarm::InvalidArgument
-    # on an unknown platform.
+    # on an unknown platform, or on a `separator:` that is not printable, non-space
+    # ASCII, contains a character illegal on the platform, or contains a path
+    # separator ("/" or "\\"). An empty separator is allowed.
     #
     # A safe *filename*, not a safe URL path segment. "%" is legal in a filename, so one
     # the caller typed is kept — sanitize_filename("..%2Fetc") returns "%2Fetc" — and a
     # consumer that percent-decodes the result must validate AFTER decoding. What this
-    # will not do is manufacture one: "%" never appears in the output unless it appeared
-    # in the input (#721).
+    # will not do is manufacture one: every "%" in the output is one the input
+    # contained, or part of the separator (#721).
     def sanitize_filename(text, separator: "_", max_length: 255, platform: :universal,
                           lang: nil, preserve_extension: true)
       translate_errors do
@@ -422,8 +498,9 @@ module Disarm
       end
     end
 
-    # ML/NLP normalization: NFKC → emoji→text → transliterate → strip accents →
-    # [case fold] → strip control → strip zero-width → collapse whitespace.
+    # ML/NLP normalization: resolve deletions → NFKC → emoji→text → transliterate →
+    # strip accents → emoji→text → [case fold] → strip control → strip zero-width →
+    # collapse whitespace → NFC.
     #
     # `fold_case:` defaults to true. Pass false in front of a CASED model — folding is
     # destructive, cannot be undone downstream, and an uncased evaluation harness cannot
@@ -460,7 +537,7 @@ module Disarm
     # anomaly detector's `bidi` kind reports nine of the twelve, holding back LRM, RLM
     # and ALM because a lone directional mark is ordinary in right-to-left text.
     def bidi_control?(text)
-      _has_bidi_control?(text)
+      translate_errors { _has_bidi_control?(text) }
     end
 
     # Explain how `lang: "auto"` detection resolves `text`: a hash with
@@ -486,6 +563,22 @@ module Disarm
       translate_errors { _script_info(name.to_s) }
     end
 
+    # TR39 confusable sources whose prototype is in +script+, and how many of those
+    # disarm's bundled tables fold. Returns a Hash with +:script+, +:sources+ and
+    # +:folded+ keys.
+    #
+    # The denominator +unmapped_confusables+ does not have: that measures one bundled
+    # table against the whole 6,565-source population, so a script disarm ships no table
+    # for reports a number determined by that absence rather than by its coverage.
+    #
+    # +:folded+ counts sources any bundled table reaches, not sources folded *toward*
+    # this script — Greek is 71 of 159, because the Latin table folds Greek letters that
+    # look Latin. A script disarm knows that TR39 never uses as a prototype returns 0 of
+    # 0. Raises Disarm::InvalidArgument on an unknown script.
+    def confusable_coverage(script)
+      translate_errors { _confusable_coverage(script.to_s) }
+    end
+
     # The Unicode `confusables.txt` release the bundled confusable tables were folded
     # from, e.g. "17.0.0". Not a Unicode version for the library as a whole — the
     # case-folding and width tables track different releases (see docs/provenance.md).
@@ -574,7 +667,9 @@ module Disarm
     #   pipe.process("Café") # => "cafe"
     #   pipe.process("Köln") # reuse the same handle
     #
-    # Disarm::Pipeline#process is the Rust-defined instance method on the handle.
+    # Disarm::Pipeline#process is the Rust-defined instance method on the handle, and
+    # Disarm::Pipeline#with_digit_policy(policy) returns a copy whose confusable passes
+    # fold under `policy` (#646); a profile with no confusables step refuses one.
     def get_pipeline(profile)
       translate_errors { _get_pipeline(profile.to_s) }
     end
@@ -603,16 +698,30 @@ module Disarm
     # whole surface. The original backtrace is preserved (passed as the third
     # `raise` argument) so the failing native call site stays visible. A bad
     # argument from the native layer can arrive as ArgumentError (an invalid
-    # scheme/target), TypeError (a non-String argument), or RangeError (e.g. a
-    # negative max_length) — all map to Disarm::InvalidArgument.
+    # scheme/target), TypeError (a non-String argument), RangeError (e.g. a
+    # negative max_length) or EncodingError (a String in an encoding Ruby cannot
+    # transcode to UTF-8) — all map to Disarm::InvalidArgument.
     def translate_errors
       yield
     rescue Error
       raise # already in our hierarchy — don't re-wrap
-    rescue ::ArgumentError, ::TypeError, ::RangeError => e
+    rescue ::ArgumentError, ::TypeError, ::RangeError, ::EncodingError => e
       raise InvalidArgument, e.message, e.backtrace
     rescue ::RuntimeError => e
       raise Error, e.message, e.backtrace
     end
   end
+
+  # The reusable handle `Disarm.get_pipeline` returns. `#process` is Rust-defined; this
+  # reopens the class for the one method that can fail, so its error arrives as
+  # `Disarm::InvalidArgument` the way every module-level call's does.
+  class Pipeline
+    # A copy of this pipeline whose confusable passes fold under `policy` (#646):
+    # `:numeric`, `:tr39` or `:preserve`. Raises Disarm::InvalidArgument when the profile
+    # has no confusables step and the policy is not the default — a setting that would
+    # never run is refused rather than kept.
+    def with_digit_policy(policy)
+      Disarm.send(:translate_errors) { _with_digit_policy(policy.to_s) }
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disarm
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.17.0
 platform: aarch64-linux
 authors:
 - Richard Quinn
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-09-01 00:00:00.000000000 Z
+date: 2026-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -95,8 +95,6 @@ files:
 - LICENSE
 - README.md
 - lib/disarm.rb
-- lib/disarm/3.1/disarm.so
-- lib/disarm/3.2/disarm.so
 - lib/disarm/3.3/disarm.so
 - lib/disarm/3.4/disarm.so
 - lib/disarm/4.0/disarm.so
@@ -117,7 +115,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.3'
   - - "<"
     - !ruby/object:Gem::Version
       version: 4.1.dev