unicode_utils 1.1.2 → 1.2.2

data/lib/unicode_utils/graphic_char_q.rb

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+
+require "unicode_utils/gc"
+
+module UnicodeUtils
+
+  GENERAL_CATEGORY_IS_GRAPHIC_MAP = {
+    Lu: true, Ll: true, Lt: true, Lm: true, Lo: true,
+    Mn: true, Mc: true, Me: true,
+    Nd: true, Nl: true, No: true,
+    Pc: true, Pd: true, Ps: true, Pe: true, Pi: true, Pf: true, Po: true,
+    Sm: true, Sc: true, Sk: true, So: true,
+    Zs: true, Zl: false, Zp: false,
+    Cc: false, Cf: false, Cs: false, Co: false, Cn: false
+  } # :nodoc:
+
+  # Returns true if the given char is a graphic char, false otherwise.
+  # See table 2-3 in section 2.4 of Unicode 6.0.0.
+  #
+  # Examples:
+  #
+  #   require "unicode_utils/graphic_char_q"
+  #   UnicodeUtils.graphic_char?("a")  # => true
+  #   UnicodeUtils.graphic_char?("\n") # => false
+  #   UnicodeUtils.graphic_char?(0x0)  # => false
+  def graphic_char?(char)
+    GENERAL_CATEGORY_IS_GRAPHIC_MAP[UnicodeUtils.gc(char)]
+  end
+  module_function :graphic_char?
+
+end

data/lib/unicode_utils/read_cdata.rb

@@ -8,6 +8,14 @@ module UnicodeUtils
 
   module Impl # :nodoc:
 
+    EAST_ASIAN_WIDTH_SYMBOL_MAP = {
+      1 => :Ambiguous,
+      2 => :Halfwidth,
+      3 => :Wide,
+      4 => :Fullwidth,
+      5 => :Narrow
+    }.freeze
+
     def self.open_cdata_file(filename, &block)
       File.open(File.join(CDATA_DIR, filename), "r:US-ASCII:-", &block)
     end
@@ -113,6 +121,93 @@ module UnicodeUtils
       }
     end
 
+    # Returns a list (array) of pairs (two element Arrays) of Range
+    # (codepoints) and associated integer value.
+    def self.read_range_to_hexdigit_list(filename)
+      Array.new.tap { |list|
+        open_cdata_file(filename) do |input|
+          cp_buffer = "x" * 6
+          cp_buffer.force_encoding(Encoding::US_ASCII)
+          val_buffer = "x"
+          val_buffer.force_encoding(Encoding::US_ASCII)
+          while input.read(6, cp_buffer)
+            list << [
+              Range.new(cp_buffer.to_i(16), input.read(6, cp_buffer).to_i(16)),
+              input.read(1, val_buffer).to_i(16)
+            ]
+          end
+        end
+      }
+    end
+
+    def self.read_east_asian_width_per_cp(filename)
+      # like read_hexdigit_map, but with translation to symbol values
+      Hash.new(:Neutral).tap { |map|
+        open_cdata_file(filename) do |input|
+          buffer = "x" * 6
+          buffer.force_encoding(Encoding::US_ASCII)
+          val_buffer = "x"
+          val_buffer.force_encoding(Encoding::US_ASCII)
+          while input.read(6, buffer)
+            map[buffer.to_i(16)] =
+              EAST_ASIAN_WIDTH_SYMBOL_MAP[input.read(1, val_buffer).to_i(16)]
+          end
+        end
+      }
+    end
+
+    def self.read_east_asian_width_ranges(filename)
+      read_range_to_hexdigit_list(filename).tap { |list|
+        list.each { |pair|
+          pair[1] = EAST_ASIAN_WIDTH_SYMBOL_MAP[pair[1]]
+        }
+      }
+    end
+
+    def self.read_general_category_per_cp(filename)
+      Hash.new.tap { |map|
+        open_cdata_file(filename) do |input|
+          cp_buffer = "x" * 6
+          cp_buffer.force_encoding(Encoding::US_ASCII)
+          cat_buffer = "x" * 2
+          cat_buffer.force_encoding(Encoding::US_ASCII)
+          while input.read(6, cp_buffer)
+            map[cp_buffer.to_i(16)] = input.read(2, cat_buffer).to_sym
+          end
+        end
+      }
+    end
+
+    def self.read_general_category_ranges(filename)
+      Array.new.tap { |list|
+        open_cdata_file(filename) do |input|
+          cp_buffer = "x" * 6
+          cp_buffer.force_encoding(Encoding::US_ASCII)
+          cat_buffer = "x" * 2
+          cat_buffer.force_encoding(Encoding::US_ASCII)
+          while input.read(6, cp_buffer)
+            list << [
+              Range.new(cp_buffer.to_i(16), input.read(6, cp_buffer).to_i(16)),
+              input.read(2, cat_buffer).to_sym
+            ]
+          end
+        end
+      }
+    end
+
+    def self.read_symbol_map(filename)
+      Hash.new.tap { |map|
+        open_cdata_file(filename) do |input|
+          input.each_line { |line|
+            parts = line.split(";")
+            parts[0].strip!
+            parts[1].strip!
+            map[parts[0].to_sym] = parts[1].to_sym
+          }
+        end
+      }
+    end
+
   end
 
 end

data/lib/unicode_utils/version.rb

@@ -11,6 +11,6 @@ module UnicodeUtils
   #
   # A release always has an even PATCHLEVEL. PATCHLEVEL is uneven
   # during development.
-  VERSION = "1.1.2"
+  VERSION = "1.2.2"
 
 end

data/test/test_unicode_utils.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require "test/unit"
+require "stringio"
 
 require "unicode_utils"
 
@@ -223,4 +224,216 @@ class TestUnicodeUtils < Test::Unit::TestCase
       UnicodeUtils.titlecase("i can has 1kg CHEESBURGER", :tr)
   end
 
+  def test_east_asian_width
+    assert_equal :Neutral, UnicodeUtils.east_asian_width("\u{0}")
+    assert_equal :Neutral, UnicodeUtils.east_asian_width("\u{10FFFF}")
+    assert_equal :Neutral, UnicodeUtils.east_asian_width("\u{C5}")
+    assert_equal :Ambiguous, UnicodeUtils.east_asian_width(0xA1)
+    assert_equal :Ambiguous, UnicodeUtils.east_asian_width(0xE000)
+    assert_equal :Ambiguous, UnicodeUtils.east_asian_width(0xF8FF)
+    assert_equal :Ambiguous, UnicodeUtils.east_asian_width(0xFE05)
+    assert_equal :Ambiguous, UnicodeUtils.east_asian_width("ß")
+    assert_equal :Halfwidth, UnicodeUtils.east_asian_width(0xFFA5)
+    assert_equal :Halfwidth, UnicodeUtils.east_asian_width(0xFFEE)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0xB116)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x11A5)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x1100)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x2E94)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x3400)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x4E05)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x4FCD)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0xFA2E)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x3FFFD)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x4DB6)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x4DBF)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x9FCC)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x9FFF)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0xFA2E)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0xFA2F)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0xFA6E)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0xFA6F)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0xFADA)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0xFAFF)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x2A6D7)
+    assert_equal :Wide, UnicodeUtils.east_asian_width(0x2A6FF)
+    assert_equal :Fullwidth, UnicodeUtils.east_asian_width(0x3000)
+    assert_equal :Fullwidth, UnicodeUtils.east_asian_width(0xFF0D)
+    assert_equal :Fullwidth, UnicodeUtils.east_asian_width(0xFFE6)
+    assert_equal :Narrow, UnicodeUtils.east_asian_width(0x20)
+    assert_equal :Narrow, UnicodeUtils.east_asian_width(0x2C)
+    assert_equal :Narrow, UnicodeUtils.east_asian_width(0x7E)
+    assert_equal :Narrow, UnicodeUtils.east_asian_width(0xA6)
+    assert_equal :Narrow, UnicodeUtils.east_asian_width(0x2986)
+  end
+
+  def test_display_width
+    assert_equal 0, UnicodeUtils.display_width("")
+    assert_equal 18, UnicodeUtils.display_width("別れる時に発する語")
+    assert_equal 18, UnicodeUtils.display_width("123456789aBcDeFgHi")
+    assert_equal 6, UnicodeUtils.display_width("Straße")
+    assert_equal 1, UnicodeUtils.display_width("a\u{308}")
+    assert_equal 5, UnicodeUtils.display_width("Now！")
+    assert_equal 2, UnicodeUtils.display_width("a̦b") # General Category Mn
+    assert_equal 4, UnicodeUtils.display_width("a\u{93b}bc") # General Category Mc
+    assert_equal 3, UnicodeUtils.display_width("ab\u{20dd}c") # General Category Me
+    assert_equal 4, UnicodeUtils.display_width("ab\u{a8}c") # General Category Sk
+    assert_equal 4, UnicodeUtils.display_width("ab\u{2000}c") # General Category Zs
+    assert_equal 3, UnicodeUtils.display_width("a b") # Zs
+    assert_equal 3, UnicodeUtils.display_width("a\u{1680}b") # Zs
+  end
+
+  def test_char_display_width
+    assert_equal 2, UnicodeUtils.char_display_width("別")
+    assert_equal 0, UnicodeUtils.char_display_width(0x308)
+    assert_equal 1, UnicodeUtils.char_display_width("a")
+  end
+
+  def test_default_ignorable_char?
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0xad)
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0x34f)
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0x115f)
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0x1160)
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0x2065)
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0x2069)
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0xe008a)
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0xffa0)
+    assert_equal true, UnicodeUtils.default_ignorable_char?(0xe0fff)
+    assert_equal false, UnicodeUtils.default_ignorable_char?("a")
+    assert_equal false, UnicodeUtils.default_ignorable_char?("語")
+    assert_equal false, UnicodeUtils.default_ignorable_char?(" ")
+    assert_equal false, UnicodeUtils.default_ignorable_char?("\u{308}")
+  end
+
+  def test_gc
+    assert_equal :Lu, UnicodeUtils.gc("A")
+    assert_equal :Ll, UnicodeUtils.gc("a")
+    assert_equal :Lt, UnicodeUtils.gc(0x1cb)
+    assert_equal :Lm, UnicodeUtils.gc(0x2b5)
+    assert_equal :Lo, UnicodeUtils.gc(0x10923)
+
+    assert_equal :Mn, UnicodeUtils.gc(0x5a0)
+    assert_equal :Mc, UnicodeUtils.gc(0x93f)
+    assert_equal :Me, UnicodeUtils.gc(0x20dd)
+
+    assert_equal :Nd, UnicodeUtils.gc(0xa901)
+    assert_equal :Nl, UnicodeUtils.gc(0x10144)
+    assert_equal :No, UnicodeUtils.gc(0x10917)
+
+    assert_equal :Pc, UnicodeUtils.gc(0x5f)
+    assert_equal :Pd, UnicodeUtils.gc(0x2011)
+    assert_equal :Ps, UnicodeUtils.gc(0x2329)
+    assert_equal :Pe, UnicodeUtils.gc(0xfe38)
+    assert_equal :Pi, UnicodeUtils.gc(0x201c)
+    assert_equal :Pf, UnicodeUtils.gc(0x201d)
+    assert_equal :Po, UnicodeUtils.gc(0x2e10)
+
+    assert_equal :Sm, UnicodeUtils.gc(0xff0b)
+    assert_equal :Sc, UnicodeUtils.gc(0xa3)
+    assert_equal :Sk, UnicodeUtils.gc(0x2c2)
+    assert_equal :So, UnicodeUtils.gc(0x60f)
+
+    assert_equal :Zs, UnicodeUtils.gc(0x2001)
+    assert_equal :Zl, UnicodeUtils.gc(0x2028)
+    assert_equal :Zp, UnicodeUtils.gc(0x2029)
+
+    assert_equal :Cc, UnicodeUtils.gc(0x0)
+    assert_equal :Cf, UnicodeUtils.gc(0x70f)
+    assert_equal :Cs, UnicodeUtils.gc(0xdb82)
+    assert_equal :Co, UnicodeUtils.gc(0xf1020)
+    assert_equal :Cn, UnicodeUtils.gc(0x10ffff)
+    ### 30 general categories ###
+
+    assert_equal :Lo, UnicodeUtils.gc(0x3400)
+    assert_equal :Lo, UnicodeUtils.gc(0x4000)
+    assert_equal :Lo, UnicodeUtils.gc(0x4db5)
+
+    assert_equal :Lo, UnicodeUtils.gc(0x4e00)
+    assert_equal :Lo, UnicodeUtils.gc(0x9fcb)
+    assert_equal :Lo, UnicodeUtils.gc(0x7111)
+
+    assert_equal :Lo, UnicodeUtils.gc(0xac00)
+    assert_equal :Lo, UnicodeUtils.gc(0xd7a3)
+    assert_equal :Lo, UnicodeUtils.gc(0xb70f)
+
+    assert_equal :Cs, UnicodeUtils.gc(0xd800)
+    assert_equal :Cs, UnicodeUtils.gc(0xdb7f)
+    assert_equal :Cs, UnicodeUtils.gc(0xda00)
+
+    assert_equal :Cs, UnicodeUtils.gc(0xdb80)
+    assert_equal :Cs, UnicodeUtils.gc(0xdbff)
+    assert_equal :Cs, UnicodeUtils.gc(0xdb90)
+
+    assert_equal :Cs, UnicodeUtils.gc(0xdc00)
+    assert_equal :Cs, UnicodeUtils.gc(0xdfff)
+    assert_equal :Cs, UnicodeUtils.gc(0xdc01)
+
+    assert_equal :Co, UnicodeUtils.gc(0xe000)
+    assert_equal :Co, UnicodeUtils.gc(0xf8ff)
+    assert_equal :Co, UnicodeUtils.gc(0xf8fe)
+
+    assert_equal :Lo, UnicodeUtils.gc(0x20000)
+    assert_equal :Lo, UnicodeUtils.gc(0x2a6d6)
+    assert_equal :Lo, UnicodeUtils.gc(0x2b000)
+
+    assert_equal :Lo, UnicodeUtils.gc(0x2a700)
+    assert_equal :Lo, UnicodeUtils.gc(0x2b734)
+    assert_equal :Lo, UnicodeUtils.gc(0x2b800)
+
+    assert_equal :Lo, UnicodeUtils.gc(0x2b740)
+    assert_equal :Lo, UnicodeUtils.gc(0x2b81d)
+    assert_equal :Lo, UnicodeUtils.gc(0x2b810)
+
+    assert_equal :Co, UnicodeUtils.gc(0xf0000)
+    assert_equal :Co, UnicodeUtils.gc(0xffffd)
+    assert_equal :Co, UnicodeUtils.gc(0xffafd)
+
+    assert_equal :Co, UnicodeUtils.gc(0x100000)
+    assert_equal :Co, UnicodeUtils.gc(0x10fffd)
+    assert_equal :Co, UnicodeUtils.gc(0x100ffd)
+
+    assert_equal nil, UnicodeUtils.gc(-1)
+    assert_equal nil, UnicodeUtils.gc(0x110000)
+  end
+
+  def test_general_category
+    assert_equal :Uppercase_Letter, UnicodeUtils.general_category("B")
+    assert_equal :Lowercase_Letter, UnicodeUtils.general_category("b")
+    assert_equal :Control, UnicodeUtils.general_category(0x0)
+    assert_equal nil, UnicodeUtils.general_category(-1)
+  end
+
+  def test_char_type
+    assert_equal :Letter, UnicodeUtils.char_type("Ä")
+    assert_equal :Letter, UnicodeUtils.char_type("ä")
+    assert_equal :Other, UnicodeUtils.char_type(0x0)
+    assert_equal :Number, UnicodeUtils.char_type("1")
+    assert_equal nil, UnicodeUtils.char_type(-1)
+  end
+
+  def test_graphic_char?
+    assert_equal true, UnicodeUtils.graphic_char?("a")
+    assert_equal true, UnicodeUtils.graphic_char?(0x308)
+    assert_equal false, UnicodeUtils.graphic_char?("\n")
+    assert_equal false, UnicodeUtils.graphic_char?(0x0)
+  end
+
+  def test_debug
+    io = StringIO.new
+    UnicodeUtils.debug("", io: io)
+    assert_equal <<-'EOF', io.string
+ Char | Ordinal | Name | General Category | UTF-8
+------+---------+------+------------------+-------
+    EOF
+    io = StringIO.new
+    UnicodeUtils.debug("一 \u{100000}\n", io: io)
+    assert_equal <<-'EOF', io.string
+ Char | Ordinal | Name                       | General Category | UTF-8
+------+---------+----------------------------+------------------+-------------
+ "一" |    4E00 | CJK UNIFIED IDEOGRAPH-4E00 | Other_Letter     | E4 B8 80
+ " "  |      20 | SPACE                      | Space_Separator  | 20
+ N/A  |  100000 | N/A                        | Private_Use      | F4 80 80 80
+ "\n" |       A | <control>                  | Control          | 0A
+    EOF
+  end
+
 end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
-  - 1
   - 2
-  version: 1.1.2
+  - 2
+  version: 1.2.2
 platform: ruby
 authors: 
 - Stefan Lang
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-18 00:00:00 +01:00
+date: 2011-11-27 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -33,10 +33,14 @@ files:
 - lib/unicode_utils/conditional_casing.rb
 - lib/unicode_utils/version.rb
 - lib/unicode_utils/grep.rb
+- lib/unicode_utils/gc.rb
 - lib/unicode_utils/nfkc.rb
+- lib/unicode_utils/char_display_width.rb
 - lib/unicode_utils/nfkd.rb
 - lib/unicode_utils/downcase.rb
 - lib/unicode_utils/case_ignorable_char_q.rb
+- lib/unicode_utils/east_asian_width.rb
+- lib/unicode_utils/general_category.rb
 - lib/unicode_utils/uppercase_char_q.rb
 - lib/unicode_utils/upcase.rb
 - lib/unicode_utils/u.rb
@@ -45,6 +49,7 @@ files:
 - lib/unicode_utils/lowercase_char_q.rb
 - lib/unicode_utils/read_cdata.rb
 - lib/unicode_utils/compatibility_decomposition.rb
+- lib/unicode_utils/display_width.rb
 - lib/unicode_utils/each_word.rb
 - lib/unicode_utils/combining_class.rb
 - lib/unicode_utils/char_name.rb
@@ -54,6 +59,8 @@ files:
 - lib/unicode_utils/cased_char_q.rb
 - lib/unicode_utils/simple_casefold.rb
 - lib/unicode_utils/titlecase_char_q.rb
+- lib/unicode_utils/char_type.rb
+- lib/unicode_utils/default_ignorable_char_q.rb
 - lib/unicode_utils/canonical_equivalents_q.rb
 - lib/unicode_utils/nfd.rb
 - lib/unicode_utils/jamo_short_name.rb
@@ -61,18 +68,23 @@ files:
 - lib/unicode_utils/each_grapheme.rb
 - lib/unicode_utils/simple_upcase.rb
 - lib/unicode_utils/simple_downcase.rb
+- lib/unicode_utils/debug.rb
+- lib/unicode_utils/graphic_char_q.rb
 - lib/unicode_utils/codepoint.rb
 - cdata/prop_set_lowercase
 - cdata/casefold_c_map
 - cdata/combining_class_map
 - cdata/grapheme_break_property
 - cdata/casefold_s_map
+- cdata/prop_set_default_ignorable
+- cdata/east_asian_width_property_ranges
 - cdata/soft_dotted_set
 - cdata/cond_lc_map
 - cdata/simple_tc_map
 - cdata/prop_set_uppercase
 - cdata/jamo_short_names
 - cdata/composition_exclusion_set
+- cdata/general_category_aliases
 - cdata/canonical_decomposition_map
 - cdata/cat_set_titlecase
 - cdata/casefold_f_map
@@ -85,6 +97,9 @@ files:
 - cdata/simple_uc_map
 - cdata/simple_lc_map
 - cdata/special_lc_map
+- cdata/general_category_per_cp
+- cdata/general_category_ranges
+- cdata/east_asian_width_property_per_cp
 - cdata/cond_uc_map
 - cdata/compatibility_decomposition_map
 - test/test_unicode_utils.rb