unicode-confusable 1.10.0 → 1.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e001a3462dfe6d2671e8c9e0aa586a5955627379d2a3cadabc2e3e331628c90a
-  data.tar.gz: c4d36984ad85c3d37965b0934c2f21ddc71afd8e1ea50366817cfe72ef6e7679
+  metadata.gz: decb589505c410acb8b064c0aa0b412f4178bac748a7b9eb0fb7a7d6d3b00da7
+  data.tar.gz: 5e60c81b398e29a1914cb356476b47c4c3b95312970d1c22064e7083c133c3b3
 SHA512:
-  metadata.gz: 55bde5498357e1953f35644f4a044fce2dd4cbc35ebc094b74642de73b57fdf755a379edaeb2f1f6d0dffda822894e827ec51fd69aea11f996279165678da62b
-  data.tar.gz: 4c40931571b97cfeb0d5e7ecb488617e9b91ef9f595960f51cf830463619e4c4a22581d80f88c62fd5951506352def6c03408b3e3068241bb3a0b29bca2e7a44
+  metadata.gz: fc871a5e4a4291c95b0218e61f9a7125f872f5e27b94e1f63b5f33f36e6188bf90f147fcc82e24ea4d3c1153ba2cc364284124049f15a25914d6a520da33428c
+  data.tar.gz: de60105cb97d2255cf9dd930a767e019a7e5e0d1e6deb4d2201bc4e726c203cdf20ebf29417ae3c19d552bb93d0a9a90802a181513136e07701aec05c89fb885

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## CHANGELOG
 
+### 1.12.0
+
+- Remove default ignorable codepoints, which is now part of the skeleton algorithm
+- Fix the confusable list for ";" (wrongly contained null bytes)
+
+### 1.11.0
+
+- Unicode 16.0
+
 ### 1.10.0
 
 - Unicode 15.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    unicode-confusable (1.10.0)
+    unicode-confusable (1.12.0)
 
 GEM
   remote: https://rubygems.org/
@@ -30,4 +30,4 @@ DEPENDENCIES
   unicode-confusable!
 
 BUNDLED WITH
-   2.1.4
+   2.5.17

data/MIT-LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2016-2023 Jan Lelis, https://janlelis.com
+Copyright (c) 2016-2024 Jan Lelis, https://janlelis.com
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,14 +1,14 @@
 # Unicode::Confusable [![[version]](https://badge.fury.io/rb/unicode-confusable.svg)](https://badge.fury.io/rb/unicode-confusable)  [![[ci]](https://github.com/janlelis/unicode-confusable/workflows/Test/badge.svg)](https://github.com/janlelis/unicode-confusable/actions?query=workflow%3ATest)
 
-Compares two strings if they are visually confusable as described in [Unicode® Technical Standard #39](https://www.unicode.org/reports/tr39/#Confusable_Detection): Both strings get transformed into a skeleton format before comparing them. The skeleton is generated by normalizing the string ([NFD](http://unicode.org/reports/tr15/#Norm_Forms)), replacing [confusable characters](https://unicode.org/Public/security/12.1.0/confusables.txt), and normalizing the string again.
+Compares two strings if they are visually confusable as described in [Unicode® Technical Standard #39](https://www.unicode.org/reports/tr39/#Confusable_Detection): Both strings get transformed into a skeleton format before comparing them. The skeleton is generated by normalizing the string ([NFD](http://unicode.org/reports/tr15/#Norm_Forms)), removing ignorable characters, replacing [confusable characters](https://unicode.org/Public/security/16.0.0/confusables.txt), and normalizing the string again.
 
-Unicode version: **15.1.0** (September 2023)
+Unicode version: **16.0.0** (September 2024)
 
 \* The Unicode normalization [depends on your Ruby version](https://idiosyncratic-ruby.com/73-unicode-version-mapping.html)
 
-Supported Rubies: **3.2**, **3.1**, **3.0**
+Please note: The TR39 standard now includes detection of confusables based on bidi formatting (i.e. right-to-left text). This is currently not supported by this gen.
 
-Old Rubies which might still work: **2.7**, **2.6**, **2.5**, **2.4**, **2.3**, **2.2**
+Supported Rubies: **3.x** (might stil work: **2.x**)
 
 ## Usage
 
@@ -49,6 +49,10 @@ Unicode::Confusable.list("o")
 # => ["⒪", "ꜵ", "℅", "ᴔ", "ꭁ", "ꭂ", "ﷲ", "№", "ం", "ಂ", "ം", "ං", "०", "੦", "૦", "௦", "౦", "೦", "൦", "๐", "໐", "၀", "٥", "۵", "ｏ", "ℴ", "𝐨", "𝑜", "𝒐", "𝓸", "𝔬", "𝕠", "𝖔", "𝗈", "𝗼", "𝘰", "𝙤", "𝚘", "ᴏ", "ᴑ", "ꬽ", "ο", "𝛐", "𝜊", "𝝄", "𝝾", "𝞸", "σ", "𝛔", "𝜎", "𝝈", "𝞂", "𝞼", "ⲟ", "о", "ჿ", "օ", "ס", "ه", "𞸤", "𞹤", "𞺄", "ﻫ", "ﻬ", "ﻪ", "ﻩ", "ھ", "ﮬ", "ﮭ", "ﮫ", "ﮪ", "ہ", "ﮨ", "ﮩ", "ﮧ", "ﮦ", "ە", "ഠ", "ဝ", "𐓪", "𑣈", "𑣗", "𐐬", "ۿ", "ø", "ꬾ", "ɵ", "ꝋ", "ө", "ѳ", "ꮎ", "ꮻ", "ꭴ", "ﳙ", "ơ", "œ", "ɶ", "∞", "ꝏ", "ꚙ", "ﳗ", "ﱑ", "ﳘ", "ﱒ", "ﶓ", "ﶔ", "ﱓ", "ﱔ", "ൟ", "တ", "ꭣ", "ﲠ", "ﳢ", "ﲥ", "ﳤ", "ﷻ", "ﴱ", "ﳨ", "ﴲ", "ﳪ", "ﷺ", "ﷷ", "ﳍ", "ﳖ", "ﳯ", "ﳞ", "ﳱ", "ﳦ", "ﲛ", "ﳠ", "ﯭ", "ﯬ"]
 ```
 
+## No Bidi-Confusable Check
+
+Testing for bidirectional confusables is currently not supported.
+
 ## No Advanced Detection
 
 TR 39 also describes mechanisms for a more exact recognition of confusables, also within the same string:
@@ -57,11 +61,11 @@ TR 39 also describes mechanisms for a more exact recognition of confusables, als
 - Mixed-script confusable
 - Whole-script confusable
 
-This is currently **not** supported by this gem.
+This is currently not supported by this gem.
 
 See [unicode-x](https://github.com/janlelis/unicode-x) for more Unicode related micro libraries.
 
 ## MIT License
 
-- Copyright (C) 2016-2023 Jan Lelis <https://janlelis.com>. Released under the MIT license.
+- Copyright (C) 2016-2024 Jan Lelis <https://janlelis.com>. Released under the MIT license.
 - Unicode data: https://www.unicode.org/copyright.html#Exhibit1

data/lib/unicode/confusable/constants.rb

@@ -2,8 +2,8 @@
 
 module Unicode
   module Confusable
-    VERSION = "1.10.0"
-    UNICODE_VERSION = "15.1.0"
+    VERSION = "1.12.0"
+    UNICODE_VERSION = "16.0.0"
     DATA_DIRECTORY = File.expand_path(File.dirname(__FILE__) + "/../../../data/").freeze
     INDEX_FILENAME = (DATA_DIRECTORY + "/confusable.marshal.gz").freeze
   end

data/lib/unicode/confusable/ignorable.rb

@@ -0,0 +1,9 @@
+require_relative 'index' unless defined? ::Unicode::Confusable::INDEX
+
+module Unicode
+  module Confusable
+  	IGNORABLE = INDEX[:IGNORABLE].reduce([]){|acc, cur|
+  	  acc + [*(cur.is_a?(Array) ? Range.new(*cur) : cur)]
+  	}.freeze
+  end
+end

data/lib/unicode/confusable.rb

@@ -4,6 +4,8 @@ require 'unicode_normalize/normalize'
 
 module Unicode
   module Confusable
+    autoload :IGNORABLE, File.expand_path('confusable/ignorable', __dir__)
+
     def self.confusable?(string1, string2)
       skeleton(string1) == skeleton(string2)
     end
@@ -12,8 +14,10 @@ module Unicode
       require_relative 'confusable/index' unless defined? ::Unicode::Confusable::INDEX
       UnicodeNormalize.normalize(
         UnicodeNormalize.normalize(string, :nfd).each_codepoint.map{ |codepoint|
-          INDEX[codepoint] || codepoint
-        }.flatten.pack("U*"), :nfd
+          unless IGNORABLE.include?(codepoint)
+            INDEX[:CONFUSABLE][codepoint] || codepoint
+          end
+        }.flatten.compact.pack("U*"), :nfd
       )
     end
 
@@ -21,9 +25,9 @@ module Unicode
       require_relative 'confusable/index' unless defined? ::Unicode::Confusable::INDEX
       codepoint = char.codepoints.first or raise ArgumentError, "no data given to Unicode::Confusable.list"
       if partial_mapping_allowed
-        INDEX.select{ |k,v| v == codepoint || v.is_a?(Array) && v.include?(codepoint) }.keys.map{ |codepoint| [codepoint].pack("U*") }
+        INDEX[:CONFUSABLE].select{ |k,v| v == codepoint || v.is_a?(Array) && v.include?(codepoint) }.keys.map{ |codepoint| [codepoint].pack("U*") }
       else
-        INDEX.select{ |k,v| v == codepoint }.keys.map{ |codepoint| [codepoint].pack("U") }
+        INDEX[:CONFUSABLE].select{ |k,v| v == codepoint }.keys.map{ |codepoint| [codepoint].pack("U") }
       end
     end
   end

data/spec/unicode_confusable_spec.rb

@@ -2,16 +2,28 @@ require_relative "../lib/unicode/confusable"
 require "minitest/autorun"
 
 describe Unicode::Confusable do
-  it "will detect official confusables" do
-    assert_equal true, Unicode::Confusable.confusable?("1", "l")
-    assert_equal true, Unicode::Confusable.confusable?("ℜ𝘂ᖯʏ", "Ruby")
-    assert_equal true, Unicode::Confusable.confusable?("Michael", "Michae1")
-    assert_equal true, Unicode::Confusable.confusable?("⁇", "??")
+  describe ".confusable?(string1, string2)" do
+    it "will detect official confusables" do
+      assert_equal true, Unicode::Confusable.confusable?("1", "l")
+      assert_equal true, Unicode::Confusable.confusable?("ℜ𝘂ᖯʏ", "Ruby")
+      assert_equal true, Unicode::Confusable.confusable?("Michael", "Michae1")
+      assert_equal true, Unicode::Confusable.confusable?("⁇", "??")
+    end
+
+    it "will return false for non-confusables" do
+      assert_equal false, Unicode::Confusable.confusable?("a", "b")
+      assert_equal false, Unicode::Confusable.confusable?("⁇", "?")
+    end
   end
 
-  it "will return false for non-confusables" do
-    assert_equal false, Unicode::Confusable.confusable?("a", "b")
-    assert_equal false, Unicode::Confusable.confusable?("⁇", "?")
+  describe ".skeleton(string)" do
+    it "returns internal skeleton representation" do
+      assert_equal "Ruby", Unicode::Confusable.skeleton("ℜ𝘂ᖯʏ")
+    end
+
+    it "will remove default ignorable codepoints" do
+      assert_equal "ab", Unicode::Confusable.skeleton("a\u{FE0F}b")
+    end
   end
 
   describe ".list(char)" do

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: unicode-confusable
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.12.0
 platform: ruby
 authors:
 - Jan Lelis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-01 00:00:00.000000000 Z
+date: 2024-10-30 00:00:00.000000000 Z
 dependencies: []
-description: "[Unicode 15.1.0] Compares two strings if they are visually confusable
+description: "[Unicode 16.0.0] Compares two strings if they are visually confusable
   as described in Unicode® Technical Standard #39: Both strings get transformed into
   a skeleton format before comparing them. The skeleton is generated by normalizing
   the string, replacing confusable characters, and then normalizing the string again."
@@ -31,6 +31,7 @@ files:
 - data/confusable.marshal.gz
 - lib/unicode/confusable.rb
 - lib/unicode/confusable/constants.rb
+- lib/unicode/confusable/ignorable.rb
 - lib/unicode/confusable/index.rb
 - lib/unicode/confusable/string_ext.rb
 - sig/unicode-confusable.rbs
@@ -41,7 +42,7 @@ licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -56,8 +57,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.4
-signing_key: 
+rubygems_version: 3.5.21
+signing_key:
 specification_version: 4
 summary: Detect characters that look visually similar.
 test_files: