kyanite 0.7.9 → 0.8.0

data/History.rdoc

@@ -1,3 +1,6 @@
+== 0.8.0 2012-11-17
+* added String#to_ascii with human-like handling of unicode special characters
+
 == 0.7.5 2012-11-14
 * added FSymbol class
 

data/README.rdoc

@@ -4,7 +4,7 @@ http://bklippstein.github.com/kyanite/frames.html
 Welcome to Kyanite. It's a general toolbox like Facets or ActiveSupport.
 
 == Features
-[{String}] Support for special letters like german umlauts. Reduce UTF8 strings to ASCII or less. Compare Strings. Parse nested brackets. Database Helpers. 
+[{String}] Better support for special letters like german umlauts. Reduce UTF8 strings to ASCII or less like humans whould do, not just deleting the accents. Compare Strings. Parse nested brackets. Database Helpers. 
 [{Class}] Tools for reflection. Convert {Class} <=> {String} <=> {Symbol}.
 [{Numeric} {Integer} {Float}] General tools.
 [{Range}] Invert selection of an Array or String.

data/Rakefile.rb

@@ -30,6 +30,7 @@ $hoe = Hoe.spec Drumherum.project_name do
   extra_deps            << ['yard',                     '>= 0.8.3'] 
   extra_deps            << ['yard_klippstein_template', '>= 0.0.37'] 
   extra_deps            << ['hashery',                  '>= 2.0.1'] 
+  extra_deps            << ['unicode_utils',            '>= 1.4.0'] 
   remote_rdoc_dir = '' # Release to root only one project  
   urls                  = [[Drumherum.url_docs], [Drumherum.url_source]]
                     

data/lib/kyanite/array.rb

@@ -33,13 +33,7 @@ require 'kyanite/symbol'                # size
 class Array    
 
 
-  # reverse of {String#to_array_of_codepoints}
-  # @return [String]
-  #  
-  def to_s_utf8
-    self.pack("U*").encode('utf-8')
-  end
- 
+
   
   # Cuts the front portion, and returns the rest.
   # If the remainder is only one element, it' not returned as an array but as single element.

data/lib/kyanite/string/cast.rb

@@ -13,14 +13,7 @@ class String
 
   
   # @!group Cast
-  
-    # reverse of {Array#to_s_utf8}
-    # @return [Array]
-    #
-    def to_array_of_codepoints
-      self.codepoints.to_a
-    end  
-  
+
   
     # Converts a string into the most plausible Identifier
     #
@@ -118,9 +111,10 @@ class String
 
 end
 
-
 # @!endgroup
 
+
+
 class NilClass
   def to_identifier;                  nil;            end
   def to_integer;                     nil;            end

data/lib/kyanite/string/chars.rb

@@ -6,10 +6,10 @@ if $0 == __FILE__
 end
 
 
-require 'kyanite/string/chars_const'
+require 'kyanite/string/chars_const' unless defined? TR_FULL
 require 'kyanite/string/misc'
+require 'unicode_utils/nfkd'
 
- 
 
 
 class String
@@ -19,8 +19,62 @@ class String
     # ---------------------------------------------------------------------------------------------------------------------------------
     # @!group Clear / Format Text
     # See TestKyaniteStringChars for tests and examples.    
+    
+    
+    # reverse of {Array#to_s_utf8}
+    # @return [Array]
+    #
+    def to_a
+      result = []
+      self.each_char do |c|
+      result << c
+      end
+      result
+    end      
+    
+    # reverse of {Array#to_s_utf8}
+    # @return [Array]
+    #
+    def to_array_of_codepoints
+      self.codepoints.to_a
+    end  
+    
+    # @return [Array]    
+    def to_array_of_hex
+      self.unpack('U'*self.length).collect {|x| x.to_s 16}
+    end    
+    
+    
+    # Reduces the string to a ASCII encoding. Example:
+    #  ffi = "\uFB03"
+    #  ix = "\u2168"
+    #  high23="²³"
+    #  high5 = "\u2075"
+    #  all = ffi + ix + high23 + high5 
+    #  all.to_ascii
+    #  => "ffiIX235"
+    # 
+    # Based on +UnicodeUtils.nfkd+, but handles all characters from ISO/IEC 8859-1 and CP1252
+    # like humans do, not just deleting the accents. Example:
+    #  "ÄÖÜäöüß".to_ascii
+    #  => "AeOeUeaeoeuess"
+    #
+    # 1. Converts ÄÖÜäöüßàáâăäãāåạąæảấầắằ etc. to AeOeUeaeoeuessaaaaaaaaaaaaaaaa.
+    # 2. Then removes all non-Ascii-chars.
+    # 3. Then removes all non-printable Ascii-chars.
+    # 4. Caution: Also Newlines are removed.    
+    # About 10 times slower than {#reduce94 reduce94}, but more accurate.     
+    # 
+    def to_ascii
+      result = self.to_ascii_extra_chars 
+      result.tr!(TR_FULL, TR_REDUCED)     # not necessary, only for performance 
+      return UnicodeUtils.nfkd(result).delete('^ -~') # delete is faster than gsub
+    end     
+    
+    
 
     # Reduces the string to a base94 encoding.
+    # About 10 times faster than with +UnicodeUtils+.     
     # 1. Converts àáâăäãāåạąæảấầắằÀÁÂĂÄÃĀÅẠĄÆẢẤẦẮẰ etc. to aaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAA.
     # 2. Then removes all non-Ascii-chars.
     # 3. Then removes all non-printable Ascii-chars.
@@ -74,7 +128,7 @@ class String
       end
       
       self.gsub!( 'ß', options[:german_sz] )        if options[:german_sz]        
-      self.tr!('abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ') 
+      self.tr!('abcdefghijklmnopqrstuvwxyz§', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ') 
       
       self.tr!(TR_FULL, TR_REDUCED.downcase)  
       unless options[:space]
@@ -186,7 +240,16 @@ class String
 
 end
 
+class Array    
 
+  # reverse of {String#to_array_of_codepoints}
+  # @return [String]
+  #  
+  def to_s_utf8
+    self.pack("U*").encode('utf-8')
+  end
+  
+end
 
 
 if defined? TransparentNil
@@ -216,11 +279,17 @@ if $0 == __FILE__ then
   
   #puts "Hallo"
   # puts 'Scheiße'.reduce94(:german_sz => 'z')
-  test_down =       'àáâăäãāåạąæảấầắằабćĉčçċцчďðđдèéêěĕëēėęếеэфĝğġģгĥħхìíîĭïĩīıįijийĵюяķкĺľłļŀлмńňñņŋнòóôŏöõōøőơœопŕřŗрśŝšşсшщţťŧþтùúûŭüũūůűųưувŵýŷÿźżžжз'
-  test_up =         'ÀÁÂĂÄÃĀÅẠĄÆẢẤẦẮẰАБĆĈČÇĊЦЧĎÐĐДÈÉÊĚĔËĒĖĘẾЕЭФĜĞĠĢГĤĦХÌÍÎĬÏĨĪİĮIJИЙĴЮЯĶКĹĽŁĻĿЛМŃŇÑŅŊНÒÓÔŎÖÕŌØŐƠŒОПŔŘŖРŚŜŠŞСШЩŢŤŦÞТÙÚÛŬÜŨŪŮŰŲƯУВŴÝŶŸŹŻŽЖЗ'
+  # test_down =       'àáâăäãāåạąæảấầắằабćĉčçċцчďðđдèéêěĕëēėęếеэфĝğġģгĥħхìíîĭïĩīıįijийĵюяķкĺľłļŀлмńňñņŋнòóôŏöõōøőơœопŕřŗрśŝšşсшщţťŧþтùúûŭüũūůűųưувŵýŷÿźżžжз'
+  # test_up =         'ÀÁÂĂÄÃĀÅẠĄÆẢẤẦẮẰАБĆĈČÇĊЦЧĎÐĐДÈÉÊĚĔËĒĖĘẾЕЭФĜĞĠĢГĤĦХÌÍÎĬÏĨĪİĮIJИЙĴЮЯĶКĹĽŁĻĿЛМŃŇÑŅŊНÒÓÔŎÖÕŌØŐƠŒОПŔŘŖРŚŜŠŞСШЩŢŤŦÞТÙÚÛŬÜŨŪŮŰŲƯУВŴÝŶŸŹŻŽЖЗ'
 
-  puts "hallo".upcase!
+  # puts "hallo".upcase!
 
+    full    = 'àáâăäãāåạąæảấầắằÀÁÂĂÄÃĀÅẠĄÆẢẤẦẮẰ'
+    reduced = 'aaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAA'   
+
+    full.each_char do |c|
+      puts c.noaccents
+    end  
 
 end
 

data/lib/kyanite/string/chars_const.rb

@@ -3,40 +3,112 @@
 if $0 == __FILE__ 
   require 'drumherum'
   smart_init
+  require 'perception'  
 end
 
-
 require 'hashery' 
+require 'unicode_utils/char_type'
 
 
 unless defined?(TR_UPCASE_ALL_REGEXP)
+
+  leerzeichen = "\u2420\u2423\u00a0\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u202f\u205f\u3000"
+  klammer_auf = "\u227a\u226a\u3008\u276c\u2329\u25c1\u25c0"
+  klammer_zu =  "\u227b\u226b\u3009\u276d\u232a\u25b7\u25b6"
+
+  # Sowohl reduce94 als auch to_ascii werden diese Zeichen übersetzen.
+  # Zeichen, die TR_FULL ergänzen und die UnicodeUtils.nfkd nicht korrekt umsetzt.
+  tr_full_b =    %q{£₤¢‹¥›•«×»÷‚‘ƒ’ˆ§´¡„¿“¦”†‡µ′″°¤∗·⋅} + leerzeichen + klammer_auf + klammer_zu
+  tr_reduced_b = %q{LLc"Y"*"*"/''f'^P'!"?"|"~~u'"~~***} + (" "*leerzeichen.length) + ("<"*klammer_auf.length) + (">"*klammer_zu.length) 
+
+  # Nur to_ascii wird diese Zeichen übersetzen.
+  # Zeichen, die in TR_FULL schon drin sind und die UnicodeUtils.nfkd nicht korrekt umsetzt
+  tr_full_c =    %q{ØøðđÐĐħĦıĸłŁŧþŦÞаАбБцчЦЧдДеэЕЭфФгГхХийИЙюяЮЯкКлЛмМнНоОпПрРсшщСШЩтТуУвВжзЖЗ} 
+  tr_reduced_c = %q{OoddDDhHiklLttTTaAbBccCCdDeeEEfFgGhHiiIIjjJJkKlLmMnNoOpPrRsssSSStTuUvVzzZZ}
+
+  
+
+  
+  # Nur to_ascii wird diese Zeichen übersetzen.  
+  TR_EXTRA_CHARS = [
+  [/ß/, 'ss'],  
+  [/Ö/, 'Oe'],  
+  [/Ü/, 'Ue'],  
+  [/Ä/, 'Ae'],  
+  [/ö/, 'oe'],  
+  [/ü/, 'ue'],  
+  [/ä/, 'ae'],  
+  [/€/, 'EUR'], 
+  [/æ/, 'ae'], 
+  [/Æ/, 'AE'],
+  [/œ/, 'oe'],
+  [/Œ/, 'OE'],
+  [/ŋ/, 'nj'],
+  [/Ŋ/, 'NJ'],
+  [/Š/, 'Sh'],
+  [/š/, 'sh'],
+  [/Ž/, 'Zh'],
+  [/ž/, 'zh'],
+  [/Ḃ/, 'Bh'],
+  [/ḃ/, 'bh'],
+  [/Ċ/, 'Ch'],
+  [/ċ/, 'ch'],
+  [/Ḋ/, 'Dh'],
+  [/ḋ/, 'dh'],
+  [/Ḟ/, 'Fh'],
+  [/ḟ/, 'fh'],
+  [/Ġ/, 'Gh'],
+  [/ġ/, 'gh'],
+  [/Ṁ/, 'Mh'],
+  [/ṁ/, 'mh'],
+  [/Ṡ/, 'Sh'],
+  [/ṡ/, 'sh'],
+  [/Ṫ/, 'Th'],
+  [/ṫ/, 'th'],
+  [/©/, '(c)'],
+  [/®/, '(r)'],
+  [/≤/, '<='],
+  [/≥/, '>='],
+  [/±/, '+/-'],
+  [/¼/, '1/4'],
+  [/½/, '1/2'],
+  [/¾/, '3/4'],
+  [/‰/, '%%'],
+  [/˜/, '~'],
+  [/[¬−‐‑‒–—―─]/, '-'] # macht Ärger und muss am Ende bleiben
+  ]
+  patterns = TR_EXTRA_CHARS.collect { |search, replace| search }
+  RE_EXTRA_CHARS = Regexp.union(*patterns)
+
+
+
   base = Hashery::Dictionary.new
-  base['a'] = '     àáâă  äãā  åạą æ ảấầắằ а       '
-  base['A'] = '     ÀÁÂĂ  ÄÃĀ  ÅẠĄ Æ ẢẤẦẮẰ А       '
-  base['b'] = '     б           ' 
-  base['B'] = '     Б                 ' 
+  base['a'] = '     àáâă  äãā  åạą ảấầắằ а ª æ    '
+  base['A'] = '     ÀÁÂĂ  ÄÃĀ  ÅẠĄ ẢẤẦẮẰ А ª Æ    '
+  base['b'] = '     ḃб           ' 
+  base['B'] = '     ḂБ                 ' 
   base['c'] = '     ćĉč çċ цч           '
   base['C'] = '     ĆĈČ ÇĊ ЦЧ            '
-  base['d'] = '     ď ðđ д            '
-  base['D'] = '     Ď ÐĐ Д            '
+  base['d'] = '     ḋď ðđ д            '
+  base['D'] = '     ḊĎ ÐĐ Д            '
   base['e'] = '     èéêěĕ  ëēėę  ế еэ         '
   base['E'] = '     ÈÉÊĚĔ  ËĒĖĘ  Ế ЕЭ         '
-  base['f'] = '     ф  '
-  base['F'] = '     Ф  '
+  base['f'] = '     ḟф  '
+  base['F'] = '     ḞФ  '
   base['g'] = '     ĝğġ ģ г           '
   base['G'] = '     ĜĞĠ Ģ Г           '
   base['h'] = '     ĥħ х                '
   base['H'] = '     ĤĦ Х                '
-  base['i'] = '     ìíîĭ ïĩīı     į  ij ий'
-  base['I'] = '     ÌÍÎĬ ÏĨĪİ     Į  IJ ИЙ'
+  base['i'] = '     ìíîĭ ïĩīı     į   ий'
+  base['I'] = '     ÌÍÎĬ ÏĨĪİ     Į   ИЙ'
   base['j'] = '	    ĵ юя    '
   base['J'] = '	    Ĵ ЮЯ    '
   base['k'] = '     ķĸ к        '
   base['K'] = '     Ķĸ К      '
   base['l'] = '     ĺ ľłļŀ л     '
   base['L'] = '     Ĺ ĽŁĻĿ Л     '
-  base['m'] = '     м   '
-  base['M'] = '     М   '
+  base['m'] = '     ṁм   '
+  base['M'] = '     ṀМ   '
   base['n'] = '     ńň  ñņʼnŋ н     '
   base['N'] = '     ŃŇ  ÑŅʼnŊ Н     '
   base['o'] = '     òóôŏ öõō  øőơ œ о      '
@@ -47,10 +119,10 @@ unless defined?(TR_UPCASE_ALL_REGEXP)
   base['Q'] = nil
   base['r'] = '     ŕř  ŗ р           '
   base['R'] = '     ŔŘ  Ŗ Р           '
-  base['s'] = '     śŝš ßş сшщ          '
-  base['S'] = '     ŚŜŠ ߪ СШЩ          '
-  base['t'] = '     ţťŧþ т        '
-  base['T'] = '     ŢŤŦÞ Т        '
+  base['s'] = '     ṡśŝš ßş сшщ          '
+  base['S'] = '     ṠŚŜŠ ߪ СШЩ          '
+  base['t'] = '     ṫţťŧþ т        '
+  base['T'] = '     ṪŢŤŦÞ Т        '
   base['u'] = '     ùúûŭ üũū  ůűųư у	     '
   base['U'] = '     ÙÚÛŬ ÜŨŪ  ŮŰŲƯ У	     '
   base['v'] = '     в'
@@ -131,51 +203,113 @@ TR_DOWNCASE_ONLY = tr_downcase_only
   end
   
 
-  TR_UPCASE    = tr_upcase2
-  TR_DOWNCASE  = tr_downcase2
-  TR_FULL      = tr_full2
-  TR_REDUCED   = tr_reduced2
+  TR_UPCASE    =        tr_upcase2
+  TR_DOWNCASE  =        tr_downcase2
+  TR_FULL      =        tr_full2  + tr_full_b
+  TR_REDUCED   =        tr_reduced2  + tr_reduced_b
+  TR_FULL_TO_ASCII =    tr_full_b + tr_full_c
+  TR_REDUCED_TO_ASCII = tr_reduced_b + tr_reduced_c
   TR_UPCASE_ALL_REGEXP = /^[A-ZÀÁÂĂÄÃĀÅẠĄÆẢẤẦẮẰАБĆĈČÇĊЦЧĎÐĐДÈÉÊĚĔËĒĖĘẾЕЭФĜĞĠĢГĤĦХÌÍÎĬÏĨĪİĮIJИЙĴЮЯĶКĹĽŁĻĿЛМŃŇÑŅŊНÒÓÔŎÖÕŌØŐƠŒОПŔŘŖРŚŜŠŞСШЩŢŤŦÞТÙÚÛŬÜŨŪŮŰŲƯУВŴÝŶŸŹŻŽЖЗ]/   
   
+  
+
+LANG_SPECIAL_CHARS = {
+  :german =>    ["ÄÖÜäöüß",   "AeOeUeaeoeuess"],
+  :dutch =>     ["IJij",        "IJij"],
+  :estonian =>  ["ŠšŽž",      "ShshZhzh"],
+  :finnish =>   ["ŠšŽž",      "ShshZhzh"],
+  :french =>    ["ŒœŸ",       "OEoeY"],
+  :hungarian => ["ŐőŰű",      "OoUu"],
+  :latin =>     ["ĀāĒēĪīŌōŪū","AaEeIiOoUu"],
+  :finnish =>   ["ĀāĒēĪīŌōŪū","AaEeIiOoUu"],
+  :turkish =>   ["İıĞğŞş",    "IiGgSs"],
+  :welsh =>     ["ẀẁẂẃŴŵŶŷ",  "WwWwWwYy"],
+  :irish  =>    ["ḂḃĊċḊḋḞḟĠġṀṁṠṡṪṫ", "BhbhChchDhdhFhfhGhghMhmhShshThth"]
+}  
+ 
+ # :irish  =>    ["ḂḃḊḋḞḟṀṁṠṡṪṫ", "BhbhChchDhdhFhfhGhghMhmhShshThth"]  
+  
+  
+  
 end # unless defined?
 
 
+  class String
 
+    # @private
+    def to_ascii_extra_chars
+      result = tr(TR_FULL_TO_ASCII, TR_REDUCED_TO_ASCII)  
+      result.gsub(RE_EXTRA_CHARS) do |match|
+        TR_EXTRA_CHARS.detect{ |search, replace| search =~ match}[1]
+      end	
+    end 
+    
+    # @private
+    def to_ascii_minus
+    
+    end
+    
+  end # class
 
 
 
 # -----------------------------------------------------------------------------------------
-# Ausprobieren
+# TR_EXTRA_CHARS und TR_FULL manuell prüfen
 #
 if $0 == __FILE__ then
+  require 'kyanite/string/chars'  
+  require 'kyanite/set'  
 
-
-  puts TR_DOWNCASE_ONLY.inspect
-
-
-  # require 'perception'
-  # rawlog "\n----------------------------------------------------------\n\n"
   
-  # rawlog 'TR_DOWNCASE_ONLY=     '
-  # rawlog TR_DOWNCASE_ONLY
-  # rawlog "\n" 
-  
-  # rawlog 'TR_FULL=     '
-  # rawlog TR_FULL
-  # rawlog "\n"
-  # rawlog 'TR_REDUCED=  '
-  # rawlog TR_REDUCED
-  # rawlog "\n"  
-  
-  # rawlog 'TR_UPCASE=   '
-  # rawlog TR_UPCASE
-  # rawlog "\n"
-  # rawlog 'TR_DOWNCASE= '
-  # rawlog TR_DOWNCASE
-  # rawlog "\n"
-  
-
 
+  # Überprüfe TR_EXTRA_CHARS
+  see
+  see "Überprüfe TR_EXTRA_CHARS"
+  see "========================"
+  see
+  see "defined in", "Dup if <>0", "Trivial?", "Hex Code", "Character", "reduce94", "to_ascii", "Klassifizierung"
+  startline = 14
+  i = 0
+  all = ""
+  TR_EXTRA_CHARS[0..-2].each do | a |
+    c = a[0].to_s[7]
+    all += c
+    see i+startline,                      # Definitionszeile
+    all.to_a.to_set.size-i-1,             # Dup-Detector
+    (c.to_array_of_codepoints[0] <= 127 ? 'TRIVIAL':''), # Trivial-Detector
+    c.to_array_of_hex,                    # sein Code in HEX    
+    c,                                    # das Zeichen
+    c.reduce94,                           # was reduce94 daraus macht
+    c.to_ascii,                           # was to_ascii daraus macht
+    UnicodeUtils.char_type(c)
+    
+    i+=1
+  end
+  
+  # Überprüfe TR_FULL
+  see
+  see
+  see
+  see "Überprüfe TR_FULL"
+  see "================="
+  see  
+  see "Nr", "Dup if <>0", "Trivial?", "Hex Code", "Character", "reduce94", "to_ascii", "Klassifizierung"  
+  i = 0
+  all = ""  
+  #TR_FULL_TO_ASCII.each_char do |c|
+  TR_FULL.each_char do |c|
+    all += c
+    see i,
+    all.to_a.to_set.size-i-1,         # Dup-Detector
+    (c.to_array_of_codepoints[0] <= 127 ? 'TRIVIAL':''), # Trivial-Detector
+    c.to_array_of_hex,                    # sein Code in HEX    
+    c,                                    # das Zeichen
+    c.reduce94,                           # was reduce94 daraus macht    
+    c.to_ascii,                           # was to_ascii daraus macht  
+    UnicodeUtils.char_type(c)
+    
+    i+=1    
+  end
 
 
 

data/test/string/test_cast.rb

@@ -14,11 +14,7 @@ require 'kyanite/array'
 # @!macro string
 class TestKyaniteStringCast < UnitTest
 
-  def test_to_array_of_codepoints
-    test = "H¿llÛ"
-    assert_equal [72, 191, 108, 108, 219],    test.to_array_of_codepoints
-    assert_equal test,                        [72, 191, 108, 108, 219].to_s_utf8
-  end
+
 
   def test_to_nil
   	assert_equal 'e', 'e'.to_nil

data/test/string/test_chars.rb

@@ -3,6 +3,7 @@
 if $0 == __FILE__ 
   require 'drumherum'
   smart_init 
+  require 'perception'
 end
 require 'drumherum/unit_test'
 require 'kyanite/string/chars'                                  
@@ -18,48 +19,175 @@ class TestKyaniteStringChars < UnitTest
 # @!group clear / format text
 #     
 
+  def test_TR_EXTRA_CHARS
+    startline = 23 # Zeilennummer in der TR_EXTRA_CHARS definiert wird
+    i = 0
+    all = ""
+    TR_EXTRA_CHARS.each do | a |
+      c = a[0].to_s[7]
+      all += c
+      assert_equal 0, all.to_a.to_set.size-i-1, "TR_EXTRA_CHARS: Dup in Zeile #{i+startline} Zeichen #{c}"  
+      #assert c.to_array_of_codepoints[0] > 127, "TR_EXTRA_CHARS: Trivialität in Zeile #{i+startline} Zeichen #{c}"   
+      i+=1
+    end
+  end
+  
+  
+  def test_TR_FULL
+    assert_equal TR_FULL.length, TR_REDUCED.length
+    i = 0
+    all = ""
+    TR_FULL.each_char do | c |
+      r = TR_REDUCED[i]    
+      all += c
+      #see "Zeichen Nr. #{i} Zeichen #{c} >> #{r}"      
+      assert_equal 0, all.to_a.to_set.size-i-1, "TR_FULL: Dup in Zeichen Nr. #{i} Zeichen #{c} >> #{r}"  
+      assert c.to_array_of_codepoints[0] > 127, "TR_FULL: Trivialität in Zeichen Nr. #{i} Zeichen #{c} >> #{r}"  
+      assert r.to_array_of_codepoints[0] <= 127, "TR_FULL: Zeichen Nr. #{i} Zeichen #{c} >> #{r} wird nicht in ASCII umgesetzt"        
+      assert_equal c.reduce94, c.to_ascii[0]     
+      i+=1
+    end
+  end  
+  
+  
+  def test_TR_FULL_TO_ASCII
+    assert_equal TR_FULL_TO_ASCII.length, TR_REDUCED_TO_ASCII.length
+    i = 0
+    all = ""
+    TR_FULL_TO_ASCII.each_char do | c |
+      r = TR_REDUCED_TO_ASCII[i]
+      all += c
+      #see "Zeichen Nr. #{i} Zeichen #{c} >> #{r}" 
+      assert_equal 0, all.to_a.to_set.size-i-1,          "TR_FULL_TO_ASCII: Dup in Zeichen Nr. #{i} Zeichen #{c} >> #{r}"  
+      assert c.to_array_of_codepoints[0] > 127,  "TR_FULL_TO_ASCII: Trivialität in Zeichen Nr. #{i} Zeichen #{c} >> #{r}"   
+      assert r.to_array_of_codepoints[0] <= 127, "TR_FULL_TO_ASCII: Zeichen Nr. #{i} Zeichen #{c} >> #{r} wird nicht in ASCII umgesetzt"        
+      i+=1
+    end
+  end    
+   
+
+  def test_to_array_of_codepoints
+    test = "H¿llÛ"
+    assert_equal [72, 191, 108, 108, 219],    test.to_array_of_codepoints
+    assert_equal test,                        [72, 191, 108, 108, 219].to_s_utf8
+  end
+  
+  def test_to_array_of_hex
+    euro = "\u20ac"
+    ffi = "\uFB03"
+    ix = "\u2168"
+    high5 = "\u2075"
+    all = euro + ffi + ix + high5
+    assert_equal ["20ac", "fb03", "2168", "2075"], all.to_array_of_hex
+  end
+  
 
     
-  def test_reduce94_a
-    full    = 'àáâăäãāåạąæảấầắằÀÁÂĂÄÃĀÅẠĄÆẢẤẦẮẰ'
-    reduced = 'aaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAA'   
-    assert_equal reduced,       full.reduce94  
+  def test_to_ascii_a
+    full    = 'ªàáâăãāåạąảấầắằÀÁÂĂÃĀÅẠĄẢẤẦẮẰ'
+    reduced = 'aaaaaaaaaaaaaaaAAAAAAAAAAAAAA' 
+    assert_equal reduced,       full.reduce94      
+    assert_equal reduced,       full.to_ascii      
   end
   
-  def test_reduce94_b
-    full    = 'ćĉčçċĆĈČÇĊďðđĎÐĐèéêěĕëēėęếÈÉÊĚĔËĒĖĘẾ'
-    reduced = 'cccccCCCCCdddDDDeeeeeeeeeeEEEEEEEEEE'   
-    assert_equal reduced,       full.reduce94  
+  def test_to_ascii_b
+    full    =   'ćĉčçċĆĈČÇĊďĎèéêěĕëēėęếÈÉÊĚĔËĒĖĘẾ'
+    reduced1 =  'cccccCCCCCdDeeeeeeeeeeEEEEEEEEEE'  
+    reduced2 =  'ccccchCCCCChdDeeeeeeeeeeEEEEEEEEEE'
+    assert_equal reduced1,       full.reduce94  
+    assert_equal reduced2,       full.to_ascii  
   end  
   
-  def test_reduce94_c
-    full    = 'ĝğġģĜĞĠĢĥħĤĦìíîĭïĩīıįijÌÍÎĬÏĨĪİĮIJĵĴķĸĶĺľłļŀĹĽŁĻĿ'
-    reduced = 'ggggGGGGhhHHiiiiiiiiiiIIIIIIIIIIjJkkKlllllLLLLL'   
-    assert_equal reduced,       full.reduce94  
+  def test_to_ascii_c
+    full    =   'ĝğġģĜĞĠĢĥĤìíîĭïĩīįÌÍÎĬÏĨĪİĮĵĴķĶĺľļŀĹĽĻĿ'
+    reduced1 =  'ggggGGGGhHiiiiiiiiIIIIIIIIIjJkKllllLLLL'  
+    reduced2 =  'ggghgGGGhGhHiiiiiiiiIIIIIIIIIjJkKllllLLLL'
+    assert_equal reduced1,       full.reduce94  
+    assert_equal reduced2,       full.to_ascii      
   end    
   
-  def test_reduce94_e
-    full    = 'ńňñņʼnŋŃŇÑŅŊòóôŏöõōøőơœÒÓÔŎÖÕŌØŐƠŒ'
-    reduced = 'nnnnnnNNNNNoooooooooooOOOOOOOOOOO'   
+  def test_to_ascii_e
+    full    = 'ńňñņʼnŃŇÑŅòóôŏõōőơÒÓÔŎÕŌŐƠ'
+    reduced = 'nnnnnNNNNooooooooOOOOOOOO' 
     assert_equal reduced,       full.reduce94  
+    assert_equal reduced,       full.to_ascii      
   end      
   
-  def test_reduce94_f
-    full    = 'ŕřŗŔŘŖśŝšßşŚŜŠŞţťŧþŢŤŦÞùúûŭüũūůűųưÙÚÛŬÜŨŪŮŰŲƯŵŴýŷÿÝŶŸźżžŹŻŽ'
-    reduced = 'rrrRRRsssssSSSSttttTTTTuuuuuuuuuuuUUUUUUUUUUUwWyyyYYYzzzZZZ'   
-    assert_equal reduced,       full.reduce94  
+  def test_to_ascii_f
+    full    =   'ŕřŗŔŘŖśŝšşŚŜŠŞţťŢŤùúûŭũūůűųưÙÚÛŬŨŪŮŰŲƯŵŴýŷÿÝŶŸźżžŹŻŽ'
+    reduced1 =  'rrrRRRssssSSSSttTTuuuuuuuuuuUUUUUUUUUUwWyyyYYYzzzZZZ'  
+    reduced2 =  'rrrRRRssshsSSShSttTTuuuuuuuuuuUUUUUUUUUUwWyyyYYYzzzhZZZh' 
+    assert_equal reduced1,       full.reduce94  
+    assert_equal reduced2,       full.to_ascii      
   end    
   
+  def test_to_ascii_zusammengesetzt
+    full    = 'ijIJſ…'
+    reduced = 'ijIJs...'  
+    assert_equal reduced,       full.to_ascii      
+  end    
+  
+  def test_to_ascii_same_same
+    same_same    = '^!"$%&/()=?@*+~#<>|,;:.-_ {[]}\\'
+    assert_equal same_same,     same_same.to_ascii 
+    same_same    = "'0123456789"
+    assert_equal same_same,     same_same.to_ascii  
+    same_same    = 'abcdefghijklmnopqrstuvwxyz'
+    assert_equal same_same,     same_same.to_ascii  
+    same_same    = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+    assert_equal same_same,     same_same.to_ascii     
+  end
+  
+  
+  def test_to_ascii_same_same
+    full = '¯¨'
+    reduced = ' ' * full.length
+    assert_equal 2,             full.length
+    assert_equal reduced,       full.to_ascii    
+  end
+  
+  def test_to_ascii_s
+    ffi = "\uFB03"
+    ix = "\u2168"
+    high23="²³"
+    high5 = "\u2075"
+    full = ffi + ix + high23 + high5 + "€ßÖÜÄöüä"
+    reduced1 = "sOUAoua"
+    reduced2 = "ffiIX235EURssOeUeAeoeueae"
+    assert_equal reduced1,       full.reduce94      
+    assert_equal reduced2,       full.to_ascii      
+  end
+  
+  def test_LANG_SPECIAL_CHARS
+    LANG_SPECIAL_CHARS .each do | lang, (full, reduced) |
+      #see lang, full, reduced, full.to_ascii, full.reduce94
+      assert_equal reduced,       full.to_ascii       
+    end  
+  end
+  
+  def test_spaces
+    spaces =  "\u0020\u00a0\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u202f\u205f\u3000\u2420\u2423"
+    assert_equal spaces.to_ascii, " " * spaces.length
+    assert_equal spaces.reduce94, " " * spaces.length
+  end
+  
+  
+  def test_minus_signs
+    minus = "\u00ac\u2212\u2010\u2011\u2012\u2013\u2014\u2015\u2500"
+    assert_equal minus.to_ascii, "-" * minus.length
+    #assert_equal spaces.reduce94, " " * spaces.length    
+  end
+  
 
   
   
   def test_reduce94_full
     full = <<ENDOFSTRING
-àáâăäãāåạąæảấầắằÀÁÂĂÄÃĀÅẠĄÆẢẤẦẮẰćĉčçċĆĈČÇĊďðđĎÐĐèéêěĕëēėęếÈÉÊĚĔËĒĖĘẾĝğġģĜĞĠĢĥħĤĦìíîĭïĩīıįijÌÍÎĬÏĨĪİĮIJĵĴķĶĺľłļŀĹĽŁĻĿńňñņʼnŋŃŇÑŅŊòóôŏöõōøőơœÒÓÔŎÖÕŌØŐƠŒŕřŗŔŘŖśŝšßşŚŜŠŞţťŧþŢŤŦÞùúûŭüũūůűųưÙÚÛŬÜŨŪŮŰŲƯŵŴýŷÿÝŶŸźżžŹŻŽ
+àáâăäãāåạąảấầắằÀÁÂĂÄÃĀÅẠĄẢẤẦẮẰćĉčçċĆĈČÇĊďðđĎÐĐèéêěĕëēėęếÈÉÊĚĔËĒĖĘẾĝğġģĜĞĠĢĥħĤĦìíîĭïĩīıįÌÍÎĬÏĨĪİĮĵĴķĶĺľłļŀĹĽŁĻĿńňñņʼnŋŃŇÑŅŊòóôŏöõōøőơœÒÓÔŎÖÕŌØŐƠŒŕřŗŔŘŖśŝšßşŚŜŠŞţťŧþŢŤŦÞùúûŭüũūůűųưÙÚÛŬÜŨŪŮŰŲƯŵŴýŷÿÝŶŸźżžŹŻŽ
 ENDOFSTRING
 
     reduced = <<ENDOFSTRING
-aaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAcccccCCCCCdddDDDeeeeeeeeeeEEEEEEEEEEggggGGGGhhHHiiiiiiiiiiIIIIIIIIIIjJkKlllllLLLLLnnnnnnNNNNNoooooooooooOOOOOOOOOOOrrrRRRsssssSSSSttttTTTTuuuuuuuuuuuUUUUUUUUUUUwWyyyYYYzzzZZZ
+aaaaaaaaaaaaaaaAAAAAAAAAAAAAAAcccccCCCCCdddDDDeeeeeeeeeeEEEEEEEEEEggggGGGGhhHHiiiiiiiiiIIIIIIIIIjJkKlllllLLLLLnnnnnnNNNNNoooooooooooOOOOOOOOOOOrrrRRRsssssSSSSttttTTTTuuuuuuuuuuuUUUUUUUUUUUwWyyyYYYzzzZZZ
 ENDOFSTRING
 
     full = full.chomp
@@ -160,8 +288,8 @@ ENDOFSTRING
 
   
   def test_downcase_upcase
-    test_down =       'àáâăäãāåạąæảấầắằабćĉčçċцчďðđдèéêěĕëēėęếеэфĝğġģгĥħхìíîĭïĩīıįijийĵюяķкĺľłļŀлмńňñņŋнòóôŏöõōøőơœопŕřŗрśŝšşсшщţťŧþтùúûŭüũūůűųưувŵýŷÿźżžжз'
-    test_up =         'ÀÁÂĂÄÃĀÅẠĄÆẢẤẦẮẰАБĆĈČÇĊЦЧĎÐĐДÈÉÊĚĔËĒĖĘẾЕЭФĜĞĠĢГĤĦХÌÍÎĬÏĨĪİĮIJИЙĴЮЯĶКĹĽŁĻĿЛМŃŇÑŅŊНÒÓÔŎÖÕŌØŐƠŒОПŔŘŖРŚŜŠŞСШЩŢŤŦÞТÙÚÛŬÜŨŪŮŰŲƯУВŴÝŶŸŹŻŽЖЗ'
+    test_down =       'àáâăäãāåạąảấầắằабćĉčçċцчďðđдèéêěĕëēėęếеэфĝğġģгĥħхìíîĭïĩīıįийĵюяķкĺľłļŀлмńňñņŋнòóôŏöõōøőơœопŕřŗрśŝšşсшщţťŧþтùúûŭüũūůűųưувŵýŷÿźżžжз'
+    test_up =         'ÀÁÂĂÄÃĀÅẠĄẢẤẦẮẰАБĆĈČÇĊЦЧĎÐĐДÈÉÊĚĔËĒĖĘẾЕЭФĜĞĠĢГĤĦХÌÍÎĬÏĨĪİĮИЙĴЮЯĶКĹĽŁĻĿЛМŃŇÑŅŊНÒÓÔŎÖÕŌØŐƠŒОПŔŘŖРŚŜŠŞСШЩŢŤŦÞТÙÚÛŬÜŨŪŮŰŲƯУВŴÝŶŸŹŻŽЖЗ'
 
     # Bescheid sagen, sobald Ruby oder ActiveSupport von sich aus funktionieren
     assert_not_equal test_down,       test_up.downcase

data/version.rb

@@ -2,7 +2,7 @@
 
 module Kyanite 
 
-  VERSION  = '0.7.9'
+  VERSION  = '0.8.0'
     
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kyanite
 version: !ruby/object:Gem::Version
-  version: 0.7.9
+  version: 0.8.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-14 00:00:00.000000000 Z
+date: 2012-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drumherum
@@ -139,6 +139,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: unicode_utils
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement