unicode-confusable 1.11.0 → 1.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11b387c2c116918a68b3d27168aa46d664b7805060030130dae1c8daa31c8582
-  data.tar.gz: 6e52671b52b5f7f826929df05d815f195743a502deb7fc427f97ba900e7a1d73
+  metadata.gz: decb589505c410acb8b064c0aa0b412f4178bac748a7b9eb0fb7a7d6d3b00da7
+  data.tar.gz: 5e60c81b398e29a1914cb356476b47c4c3b95312970d1c22064e7083c133c3b3
 SHA512:
-  metadata.gz: e31bae98039b1b83dde599b4f7442a8afc83cc21f1a7085da012af5acf1447faa659f2f9b27a0644b23d1975446c74e4358046c9884b7b3403c90ea0a762ace1
-  data.tar.gz: dc86a08f2b475448292e7a669ac929577f5299c33d2292d5c6004e82ee19fda19db7dfecd1fd739cf383e8dabc1b0d767483fde89e7b88c9062d5ac58286102f
+  metadata.gz: fc871a5e4a4291c95b0218e61f9a7125f872f5e27b94e1f63b5f33f36e6188bf90f147fcc82e24ea4d3c1153ba2cc364284124049f15a25914d6a520da33428c
+  data.tar.gz: de60105cb97d2255cf9dd930a767e019a7e5e0d1e6deb4d2201bc4e726c203cdf20ebf29417ae3c19d552bb93d0a9a90802a181513136e07701aec05c89fb885

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## CHANGELOG
 
+### 1.12.0
+
+- Remove default ignorable codepoints, which is now part of the skeleton algorithm
+- Fix the confusable list for ";" (wrongly contained null bytes)
+
 ### 1.11.0
 
 - Unicode 16.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    unicode-confusable (1.11.0)
+    unicode-confusable (1.12.0)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -1,14 +1,14 @@
 # Unicode::Confusable [![[version]](https://badge.fury.io/rb/unicode-confusable.svg)](https://badge.fury.io/rb/unicode-confusable)  [![[ci]](https://github.com/janlelis/unicode-confusable/workflows/Test/badge.svg)](https://github.com/janlelis/unicode-confusable/actions?query=workflow%3ATest)
 
-Compares two strings if they are visually confusable as described in [Unicode® Technical Standard #39](https://www.unicode.org/reports/tr39/#Confusable_Detection): Both strings get transformed into a skeleton format before comparing them. The skeleton is generated by normalizing the string ([NFD](http://unicode.org/reports/tr15/#Norm_Forms)), replacing [confusable characters](https://unicode.org/Public/security/12.1.0/confusables.txt), and normalizing the string again.
+Compares two strings if they are visually confusable as described in [Unicode® Technical Standard #39](https://www.unicode.org/reports/tr39/#Confusable_Detection): Both strings get transformed into a skeleton format before comparing them. The skeleton is generated by normalizing the string ([NFD](http://unicode.org/reports/tr15/#Norm_Forms)), removing ignorable characters, replacing [confusable characters](https://unicode.org/Public/security/16.0.0/confusables.txt), and normalizing the string again.
 
 Unicode version: **16.0.0** (September 2024)
 
 \* The Unicode normalization [depends on your Ruby version](https://idiosyncratic-ruby.com/73-unicode-version-mapping.html)
 
-Supported Rubies: **3.3**, **3.2**, **3.1**, **3.0**
+Please note: The TR39 standard now includes detection of confusables based on bidi formatting (i.e. right-to-left text). This is currently not supported by this gen.
 
-Old Rubies which might still work: **2.7**, **2.6**, **2.5**, **2.4**, **2.3**, **2.2**
+Supported Rubies: **3.x** (might stil work: **2.x**)
 
 ## Usage
 
@@ -49,6 +49,10 @@ Unicode::Confusable.list("o")
 # => ["⒪", "ꜵ", "℅", "ᴔ", "ꭁ", "ꭂ", "ﷲ", "№", "ం", "ಂ", "ം", "ං", "०", "੦", "૦", "௦", "౦", "೦", "൦", "๐", "໐", "၀", "٥", "۵", "ｏ", "ℴ", "𝐨", "𝑜", "𝒐", "𝓸", "𝔬", "𝕠", "𝖔", "𝗈", "𝗼", "𝘰", "𝙤", "𝚘", "ᴏ", "ᴑ", "ꬽ", "ο", "𝛐", "𝜊", "𝝄", "𝝾", "𝞸", "σ", "𝛔", "𝜎", "𝝈", "𝞂", "𝞼", "ⲟ", "о", "ჿ", "օ", "ס", "ه", "𞸤", "𞹤", "𞺄", "ﻫ", "ﻬ", "ﻪ", "ﻩ", "ھ", "ﮬ", "ﮭ", "ﮫ", "ﮪ", "ہ", "ﮨ", "ﮩ", "ﮧ", "ﮦ", "ە", "ഠ", "ဝ", "𐓪", "𑣈", "𑣗", "𐐬", "ۿ", "ø", "ꬾ", "ɵ", "ꝋ", "ө", "ѳ", "ꮎ", "ꮻ", "ꭴ", "ﳙ", "ơ", "œ", "ɶ", "∞", "ꝏ", "ꚙ", "ﳗ", "ﱑ", "ﳘ", "ﱒ", "ﶓ", "ﶔ", "ﱓ", "ﱔ", "ൟ", "တ", "ꭣ", "ﲠ", "ﳢ", "ﲥ", "ﳤ", "ﷻ", "ﴱ", "ﳨ", "ﴲ", "ﳪ", "ﷺ", "ﷷ", "ﳍ", "ﳖ", "ﳯ", "ﳞ", "ﳱ", "ﳦ", "ﲛ", "ﳠ", "ﯭ", "ﯬ"]
 ```
 
+## No Bidi-Confusable Check
+
+Testing for bidirectional confusables is currently not supported.
+
 ## No Advanced Detection
 
 TR 39 also describes mechanisms for a more exact recognition of confusables, also within the same string:
@@ -57,7 +61,7 @@ TR 39 also describes mechanisms for a more exact recognition of confusables, als
 - Mixed-script confusable
 - Whole-script confusable
 
-This is currently **not** supported by this gem.
+This is currently not supported by this gem.
 
 See [unicode-x](https://github.com/janlelis/unicode-x) for more Unicode related micro libraries.
 

data/lib/unicode/confusable/constants.rb

@@ -2,7 +2,7 @@
 
 module Unicode
   module Confusable
-    VERSION = "1.11.0"
+    VERSION = "1.12.0"
     UNICODE_VERSION = "16.0.0"
     DATA_DIRECTORY = File.expand_path(File.dirname(__FILE__) + "/../../../data/").freeze
     INDEX_FILENAME = (DATA_DIRECTORY + "/confusable.marshal.gz").freeze

data/lib/unicode/confusable/ignorable.rb

@@ -0,0 +1,9 @@
+require_relative 'index' unless defined? ::Unicode::Confusable::INDEX
+
+module Unicode
+  module Confusable
+  	IGNORABLE = INDEX[:IGNORABLE].reduce([]){|acc, cur|
+  	  acc + [*(cur.is_a?(Array) ? Range.new(*cur) : cur)]
+  	}.freeze
+  end
+end

data/lib/unicode/confusable.rb

@@ -4,6 +4,8 @@ require 'unicode_normalize/normalize'
 
 module Unicode
   module Confusable
+    autoload :IGNORABLE, File.expand_path('confusable/ignorable', __dir__)
+
     def self.confusable?(string1, string2)
       skeleton(string1) == skeleton(string2)
     end
@@ -12,8 +14,10 @@ module Unicode
       require_relative 'confusable/index' unless defined? ::Unicode::Confusable::INDEX
       UnicodeNormalize.normalize(
         UnicodeNormalize.normalize(string, :nfd).each_codepoint.map{ |codepoint|
-          INDEX[codepoint] || codepoint
-        }.flatten.pack("U*"), :nfd
+          unless IGNORABLE.include?(codepoint)
+            INDEX[:CONFUSABLE][codepoint] || codepoint
+          end
+        }.flatten.compact.pack("U*"), :nfd
       )
     end
 
@@ -21,9 +25,9 @@ module Unicode
       require_relative 'confusable/index' unless defined? ::Unicode::Confusable::INDEX
       codepoint = char.codepoints.first or raise ArgumentError, "no data given to Unicode::Confusable.list"
       if partial_mapping_allowed
-        INDEX.select{ |k,v| v == codepoint || v.is_a?(Array) && v.include?(codepoint) }.keys.map{ |codepoint| [codepoint].pack("U*") }
+        INDEX[:CONFUSABLE].select{ |k,v| v == codepoint || v.is_a?(Array) && v.include?(codepoint) }.keys.map{ |codepoint| [codepoint].pack("U*") }
       else
-        INDEX.select{ |k,v| v == codepoint }.keys.map{ |codepoint| [codepoint].pack("U") }
+        INDEX[:CONFUSABLE].select{ |k,v| v == codepoint }.keys.map{ |codepoint| [codepoint].pack("U") }
       end
     end
   end

data/spec/unicode_confusable_spec.rb

@@ -2,16 +2,28 @@ require_relative "../lib/unicode/confusable"
 require "minitest/autorun"
 
 describe Unicode::Confusable do
-  it "will detect official confusables" do
-    assert_equal true, Unicode::Confusable.confusable?("1", "l")
-    assert_equal true, Unicode::Confusable.confusable?("ℜ𝘂ᖯʏ", "Ruby")
-    assert_equal true, Unicode::Confusable.confusable?("Michael", "Michae1")
-    assert_equal true, Unicode::Confusable.confusable?("⁇", "??")
+  describe ".confusable?(string1, string2)" do
+    it "will detect official confusables" do
+      assert_equal true, Unicode::Confusable.confusable?("1", "l")
+      assert_equal true, Unicode::Confusable.confusable?("ℜ𝘂ᖯʏ", "Ruby")
+      assert_equal true, Unicode::Confusable.confusable?("Michael", "Michae1")
+      assert_equal true, Unicode::Confusable.confusable?("⁇", "??")
+    end
+
+    it "will return false for non-confusables" do
+      assert_equal false, Unicode::Confusable.confusable?("a", "b")
+      assert_equal false, Unicode::Confusable.confusable?("⁇", "?")
+    end
   end
 
-  it "will return false for non-confusables" do
-    assert_equal false, Unicode::Confusable.confusable?("a", "b")
-    assert_equal false, Unicode::Confusable.confusable?("⁇", "?")
+  describe ".skeleton(string)" do
+    it "returns internal skeleton representation" do
+      assert_equal "Ruby", Unicode::Confusable.skeleton("ℜ𝘂ᖯʏ")
+    end
+
+    it "will remove default ignorable codepoints" do
+      assert_equal "ab", Unicode::Confusable.skeleton("a\u{FE0F}b")
+    end
   end
 
   describe ".list(char)" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unicode-confusable
 version: !ruby/object:Gem::Version
-  version: 1.11.0
+  version: 1.12.0
 platform: ruby
 authors:
 - Jan Lelis
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-13 00:00:00.000000000 Z
+date: 2024-10-30 00:00:00.000000000 Z
 dependencies: []
 description: "[Unicode 16.0.0] Compares two strings if they are visually confusable
   as described in Unicode® Technical Standard #39: Both strings get transformed into
@@ -31,6 +31,7 @@ files:
 - data/confusable.marshal.gz
 - lib/unicode/confusable.rb
 - lib/unicode/confusable/constants.rb
+- lib/unicode/confusable/ignorable.rb
 - lib/unicode/confusable/index.rb
 - lib/unicode/confusable/string_ext.rb
 - sig/unicode-confusable.rbs
@@ -56,7 +57,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
+rubygems_version: 3.5.21
 signing_key:
 specification_version: 4
 summary: Detect characters that look visually similar.