unidecoder 1.1.0

data/lib/unidecoder/version.rb

@@ -0,0 +1,9 @@
+module Unidecoder
+  module Version
+    MAJOR = 1
+    MINOR = 1
+    TINY = 0
+    BUILD = nil
+    STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
+  end
+end

data/test/unicode_point_suite/basic_latin_test.rb

@@ -0,0 +1,144 @@
+$:.unshift File.expand_path("../../lib", File.dirname(__FILE__))
+$:.unshift File.expand_path(File.dirname(__FILE__))
+$:.uniq!
+require "test/unit"
+require "unidecoder"
+require "codepoint_test_helper"
+
+include CodepointTestHelper
+
+class BasicLatinTest < Test::Unit::TestCase
+  # This test suite is just regression test and debugging
+  # to better transliterate the Basic Latin Unicode codepoints
+  #
+  # http://unicode.org/charts/
+  # http://unicode.org/charts/PDF/U0000.pdf
+
+  # NOTE: I can't figure out how to test control characters.
+  # Get weird results trying to pack them to unicode.
+
+  def test_spaces
+    assert_equal_encoded " ", %w{0020 00a0}
+    assert_equal_encoded "",  %w{200b 2060}
+  end
+
+  def test_exclamation_marks
+    assert_equal_encoded "!", %w{0021 2762}
+    assert_equal_encoded "!!", "203c"
+    assert_equal_encoded "", "00a1"
+    assert_equal_encoded "?!", "203d"
+  end
+
+  def test_quotation_marks
+    assert_equal_encoded "\"", %w{0022 02ba 2033 3003}
+  end
+
+  def test_apostrophes
+    assert_equal_encoded "'", %w{0027 02b9 02bc 02c8 2032}
+  end
+
+  def test_asterisks
+    assert_equal_encoded "*", %w{002a 066d 204e 2217 26b9 2731}
+  end
+
+  def test_commas
+    assert_equal_encoded ",", %w{002c 060c}
+  end
+
+  def test_periods
+    assert_equal_encoded ".", %w{002e 06d4}
+  end
+
+  def test_hyphens
+    assert_equal_encoded "-", %w{002d 2010 2011 2012 2212}
+  end
+
+  def test_endash
+    assert_equal_encoded "--", %w{2013 2015}
+  end
+
+  def test_emdash
+    assert_equal_encoded "---", %w{2014}
+  end
+
+  def test_dotleader
+    assert_equal_encoded "..", %w{2025}
+  end
+
+  def test_ellipsis
+    assert_equal_encoded "...", %w{2026}
+  end
+
+  def test_slashes
+    assert_equal_encoded "/", %w{002f 2044 2215}
+    assert_equal_encoded "\\", %w{005c 2216}
+  end
+
+  def test_colons
+    assert_equal_encoded ":", %w{003a 2236}
+  end
+
+  def test_semicolons
+    assert_equal_encoded ";", %w{003b 061b}
+  end
+
+  def test_less_thans
+    assert_equal_encoded "<", %w{003c 2039 2329 27e8 3008}
+  end
+
+  def test_equals
+    assert_equal_encoded "=", "003d"
+  end
+
+  def test_greater_thans
+    assert_equal_encoded ">", %w{003e 203a 232a 27e9 3009}
+  end
+
+  def test_question_marks
+    assert_equal_encoded "?", %w{003f 061f}
+    assert_equal_encoded "", "00bf"
+    assert_equal_encoded "?!", %w{203d 2048}
+    assert_equal_encoded "!?", "2049"
+  end
+
+  def test_circumflexes
+    assert_equal_encoded "^", %w{005e 2038 2303}
+  end
+
+  def test_underscores
+    assert_equal_encoded "_", %w{005f 02cd 2017}
+  end
+
+  def test_grave_accents
+    assert_equal_encoded "`", %w{0060 02cb 2035}
+  end
+
+  def test_bars
+    assert_equal_encoded "|", %w{007c 2223 2758}
+  end
+
+  def test_tildes
+    assert_equal_encoded "~", %w{007e 02dc 2053 223c ff5e}
+  end
+
+  def test_related_letters
+    {
+      "B" => "212c",
+      "C" => %w{2102 212d},
+      "E" => %w{2107 2130},
+      "F" => "2131",
+      "H" => %w{210b 210c 210d},
+      "I" => %w{0130 0406 04c0 2110 2111 2160},
+      "K" => "212a",
+      "L" => "2112",
+      "M" => "2133",
+      "N" => "2115",
+      "P" => "2119",
+      "Q" => "211a",
+      "R" => %w{211b 211c 211d},
+      "Z" => %w{2124 2128}
+    }.each do |expected, encode_mes|
+      assert_equal_encoded expected, encode_mes
+    end
+  end
+end

data/test/unicode_point_suite/codepoint_test_helper.rb

@@ -0,0 +1,28 @@
+# 100% shorthand
+module CodepointTestHelper
+  def assert_equal_encoded(expected, encode_mes)
+    # Killing a duck because Ruby 1.9 doesn't mix Enumerable into String
+    encode_mes = [encode_mes] if encode_mes.is_a?(String)
+    encode_mes.each do |encode_me|
+      encoded = encode(encode_me)
+      actual = encoded.to_ascii
+      if expected == actual
+        # Let's not retest it
+        assert true
+      else
+        message = "<#{expected.inspect}> expected but was\n<#{actual.inspect}>\n"
+        message << "  defined in #{Unidecoder.in_yaml_file(encoded)}"
+        fail message
+      end
+    end
+  end
+
+private
+  def encode(codepoint)
+    Unidecoder.encode(codepoint)
+  end
+
+  def which_yaml(codepoint)
+    Unidecoder.in_yaml_file(encode(codepoint))
+  end
+end

data/test/unidecoder_test.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+$:.unshift File.expand_path("../lib", File.dirname(__FILE__))
+$:.unshift File.expand_path(File.dirname(__FILE__))
+$:.uniq!
+require "test/unit"
+require "unidecoder"
+
+
+class UnidecoderTest < Test::Unit::TestCase
+  # Silly phrases courtesy of Frank da Cruz (http://www.columbia.edu/kermit/utf8.html).
+
+  DONT_CONVERT = [
+    "Vitrum edere possum; mihi non nocet.",               # Latin
+    "Je puis mangier del voirre. Ne me nuit.",            # Old French
+    "Kristala jan dezaket, ez dit minik ematen.",         # Basque
+    "Kaya kong kumain nang bubog at hindi ako masaktan.", # Tagalog
+    "Ich kann Glas essen, ohne mir weh zu tun.",          # German
+    "I can eat glass and it doesn't hurt me.",            # English
+  ]
+
+  CONVERT_PAIRS = {
+    # French
+    "Je peux manger du verre, ça ne me fait pas de mal."     => "Je peux manger du verre, ca ne me fait pas de mal.",
+    # Romanian
+    "Pot să mănânc sticlă și ea nu mă rănește."              => "Pot sa mananc sticla si ea nu ma raneste.",
+    # Icelandic
+    "Ég get etið gler án þess að meiða mig."                 => "Eg get etid gler an thess ad meida mig.",
+    # Albanian
+    "Unë mund të ha qelq dhe nuk më gjen gjë."               => "Une mund te ha qelq dhe nuk me gjen gje.",
+    # Polish
+    "Mogę jeść szkło i mi nie szkodzi."                      => "Moge jesc szklo i mi nie szkodzi.",
+    # Russian
+    "Я могу есть стекло, оно мне не вредит."                 => "Ia moghu iest' stieklo, ono mnie nie vriedit.",
+    # Bulgarian
+    "Мога да ям стъкло, то не ми вреди."                     => "Mogha da iam stklo, to nie mi vriedi.",
+    # Anglo-Saxon
+    "ᛁᚳ᛫ᛗᚨᚷ᛫ᚷᛚᚨᛋ᛫ᛖᚩᛏᚪᚾ᛫ᚩᚾᛞ᛫ᚻᛁᛏ᛫ᚾᛖ᛫ᚻᛖᚪᚱᛗᛁᚪᚧ᛫ᛗᛖ᛬"              => "ic.mag.glas.eotacn.ond.hit.ne.heacrmiacth.me:",
+    # Classical Greek
+    "ὕαλον ϕαγεῖν δύναμαι· τοῦτο οὔ με βλάπτει"              => "ualon phagein dunamai; touto ou me blaptei",
+    # Hindi
+    "मैं काँच खा सकता हूँ और मुझे उससे कोई चोट नहीं पहुंचती" => "maiN kaaNc khaa sktaa huuN aur mujhe usse koii cott nhiiN phuNctii",
+    # Thai
+    "ฉันกินกระจกได้ แต่มันไม่ทำให้ฉันเจ็บ"                   => "chankinkracchkaid aetmanaimthamaihchanecchb",
+    # Chinese
+    "我能吞下玻璃而不伤身体。"                                           => "Wo Neng Tun Xia Bo Li Er Bu Shang Shen Ti . ",
+    # Japanese
+    "私はガラスを食べられます。それは私を傷つけません。"                              => "Si hagarasuwoShi beraremasu. sorehaSi woShang tukemasen. ",
+    "من می توانم بدونِ احساس درد شيشه بخورم" => # Persian
+      "mn my twnm bdwni Hss drd shyshh bkhwrm",
+    "أنا قادر على أكل الزجاج و هذا لا يؤلمن" => # Arabic
+      "'n qdr `l~ 'kl lzjj w hdh l yw'lmn",
+    "אני יכול לאכול זכוכית וזה לא מזיק לי" => # Hebrew
+      "ny ykvl lkvl zkvkyt vzh l mzyq ly",
+  }
+
+  def test_should_raise_error_with_invalid_utf8
+    [
+      "\x80",  # Continuation byte, low (cp125)
+      "\x94",  # Continuation byte, mid (cp125)
+      "\x9F",  # Continuation byte, high (cp125)
+      "\xC0",  # Overlong encoding, start of 2-byte sequence, but codepoint < 128
+      "\xC1",  # Overlong encoding, start of 2-byte sequence, but codepoint < 128
+      "\xC2",  # Start of 2-byte sequence, low
+      "\xC8",  # Start of 2-byte sequence, mid
+      "\xDF",  # Start of 2-byte sequence, high
+      "\xE0",  # Start of 3-byte sequence, low
+      "\xE8",  # Start of 3-byte sequence, mid
+      "\xEF",  # Start of 3-byte sequence, high
+      "\xF0",  # Start of 4-byte sequence
+      "\xF1",  # Unused byte
+      "\xFF",  # Restricted byte
+    ].map do |byte|
+      assert_raise ArgumentError, "#{byte.inspect} did not raise error" do
+        Unidecoder.decode("a#{byte}a")
+      end
+    end
+
+  end
+
+  def test_unidecoder_decode
+    DONT_CONVERT.each do |ascii|
+      assert_equal ascii, Unidecoder.decode(ascii)
+    end
+    CONVERT_PAIRS.each do |unicode, ascii|
+      assert_equal ascii, Unidecoder.decode(unicode)
+    end
+  end
+
+  def test_unidecoder_encode
+    {
+      # Strings
+      "0041" => "A",
+      "00e6" => "æ",
+      "042f" => "Я"
+    }.each do |codepoint, unicode|
+      assert_equal unicode, Unidecoder.encode(codepoint)
+    end
+  end
+
+  def test_unidecoder_in_yaml_file
+    {
+      "A" => "x00.yml (line 67)",
+      "π" => "x03.yml (line 194)",
+      "Я" => "x04.yml (line 49)"
+    }.each do |character, output|
+      assert_equal output, Unidecoder.in_yaml_file(character)
+    end
+  end
+
+  def test_override
+    assert_equal "Juergen", Unidecoder.decode("Jürgen", "ü" => "ue")
+  end
+
+end

metadata

@@ -0,0 +1,255 @@
+--- !ruby/object:Gem::Specification 
+name: unidecoder
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 1
+  - 1
+  - 0
+  version: 1.1.0
+platform: ruby
+authors: 
+- Russell Norris
+- Norman Clarke
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-23 00:00:00 -03:00
+default_executable: 
+dependencies: []
+
+description: A port of Perl's Unidecoder to Ruby. Transliterates Unicode strings to an ASCII approximation.
+email: 
+- rsl@luckysneaks.com
+- norman@njclarke.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/unidecoder/version.rb
+- lib/unidecoder.rb
+- Changelog.md
+- README.md
+- Rakefile
+- test/unicode_point_suite/basic_latin_test.rb
+- test/unicode_point_suite/codepoint_test_helper.rb
+- test/unidecoder_test.rb
+- lib/unidecoder/data/x00.yml
+- lib/unidecoder/data/x01.yml
+- lib/unidecoder/data/x02.yml
+- lib/unidecoder/data/x03.yml
+- lib/unidecoder/data/x04.yml
+- lib/unidecoder/data/x05.yml
+- lib/unidecoder/data/x06.yml
+- lib/unidecoder/data/x07.yml
+- lib/unidecoder/data/x09.yml
+- lib/unidecoder/data/x0a.yml
+- lib/unidecoder/data/x0b.yml
+- lib/unidecoder/data/x0c.yml
+- lib/unidecoder/data/x0d.yml
+- lib/unidecoder/data/x0e.yml
+- lib/unidecoder/data/x0f.yml
+- lib/unidecoder/data/x10.yml
+- lib/unidecoder/data/x11.yml
+- lib/unidecoder/data/x12.yml
+- lib/unidecoder/data/x13.yml
+- lib/unidecoder/data/x14.yml
+- lib/unidecoder/data/x15.yml
+- lib/unidecoder/data/x16.yml
+- lib/unidecoder/data/x17.yml
+- lib/unidecoder/data/x18.yml
+- lib/unidecoder/data/x1e.yml
+- lib/unidecoder/data/x1f.yml
+- lib/unidecoder/data/x20.yml
+- lib/unidecoder/data/x21.yml
+- lib/unidecoder/data/x22.yml
+- lib/unidecoder/data/x23.yml
+- lib/unidecoder/data/x24.yml
+- lib/unidecoder/data/x25.yml
+- lib/unidecoder/data/x26.yml
+- lib/unidecoder/data/x27.yml
+- lib/unidecoder/data/x28.yml
+- lib/unidecoder/data/x2e.yml
+- lib/unidecoder/data/x2f.yml
+- lib/unidecoder/data/x30.yml
+- lib/unidecoder/data/x31.yml
+- lib/unidecoder/data/x32.yml
+- lib/unidecoder/data/x33.yml
+- lib/unidecoder/data/x4d.yml
+- lib/unidecoder/data/x4e.yml
+- lib/unidecoder/data/x4f.yml
+- lib/unidecoder/data/x50.yml
+- lib/unidecoder/data/x51.yml
+- lib/unidecoder/data/x52.yml
+- lib/unidecoder/data/x53.yml
+- lib/unidecoder/data/x54.yml
+- lib/unidecoder/data/x55.yml
+- lib/unidecoder/data/x56.yml
+- lib/unidecoder/data/x57.yml
+- lib/unidecoder/data/x58.yml
+- lib/unidecoder/data/x59.yml
+- lib/unidecoder/data/x5a.yml
+- lib/unidecoder/data/x5b.yml
+- lib/unidecoder/data/x5c.yml
+- lib/unidecoder/data/x5d.yml
+- lib/unidecoder/data/x5e.yml
+- lib/unidecoder/data/x5f.yml
+- lib/unidecoder/data/x60.yml
+- lib/unidecoder/data/x61.yml
+- lib/unidecoder/data/x62.yml
+- lib/unidecoder/data/x63.yml
+- lib/unidecoder/data/x64.yml
+- lib/unidecoder/data/x65.yml
+- lib/unidecoder/data/x66.yml
+- lib/unidecoder/data/x67.yml
+- lib/unidecoder/data/x68.yml
+- lib/unidecoder/data/x69.yml
+- lib/unidecoder/data/x6a.yml
+- lib/unidecoder/data/x6b.yml
+- lib/unidecoder/data/x6c.yml
+- lib/unidecoder/data/x6d.yml
+- lib/unidecoder/data/x6e.yml
+- lib/unidecoder/data/x6f.yml
+- lib/unidecoder/data/x70.yml
+- lib/unidecoder/data/x71.yml
+- lib/unidecoder/data/x72.yml
+- lib/unidecoder/data/x73.yml
+- lib/unidecoder/data/x74.yml
+- lib/unidecoder/data/x75.yml
+- lib/unidecoder/data/x76.yml
+- lib/unidecoder/data/x77.yml
+- lib/unidecoder/data/x78.yml
+- lib/unidecoder/data/x79.yml
+- lib/unidecoder/data/x7a.yml
+- lib/unidecoder/data/x7b.yml
+- lib/unidecoder/data/x7c.yml
+- lib/unidecoder/data/x7d.yml
+- lib/unidecoder/data/x7e.yml
+- lib/unidecoder/data/x7f.yml
+- lib/unidecoder/data/x80.yml
+- lib/unidecoder/data/x81.yml
+- lib/unidecoder/data/x82.yml
+- lib/unidecoder/data/x83.yml
+- lib/unidecoder/data/x84.yml
+- lib/unidecoder/data/x85.yml
+- lib/unidecoder/data/x86.yml
+- lib/unidecoder/data/x87.yml
+- lib/unidecoder/data/x88.yml
+- lib/unidecoder/data/x89.yml
+- lib/unidecoder/data/x8a.yml
+- lib/unidecoder/data/x8b.yml
+- lib/unidecoder/data/x8c.yml
+- lib/unidecoder/data/x8d.yml
+- lib/unidecoder/data/x8e.yml
+- lib/unidecoder/data/x8f.yml
+- lib/unidecoder/data/x90.yml
+- lib/unidecoder/data/x91.yml
+- lib/unidecoder/data/x92.yml
+- lib/unidecoder/data/x93.yml
+- lib/unidecoder/data/x94.yml
+- lib/unidecoder/data/x95.yml
+- lib/unidecoder/data/x96.yml
+- lib/unidecoder/data/x97.yml
+- lib/unidecoder/data/x98.yml
+- lib/unidecoder/data/x99.yml
+- lib/unidecoder/data/x9a.yml
+- lib/unidecoder/data/x9b.yml
+- lib/unidecoder/data/x9c.yml
+- lib/unidecoder/data/x9d.yml
+- lib/unidecoder/data/x9e.yml
+- lib/unidecoder/data/x9f.yml
+- lib/unidecoder/data/xa0.yml
+- lib/unidecoder/data/xa1.yml
+- lib/unidecoder/data/xa2.yml
+- lib/unidecoder/data/xa3.yml
+- lib/unidecoder/data/xa4.yml
+- lib/unidecoder/data/xac.yml
+- lib/unidecoder/data/xad.yml
+- lib/unidecoder/data/xae.yml
+- lib/unidecoder/data/xaf.yml
+- lib/unidecoder/data/xb0.yml
+- lib/unidecoder/data/xb1.yml
+- lib/unidecoder/data/xb2.yml
+- lib/unidecoder/data/xb3.yml
+- lib/unidecoder/data/xb4.yml
+- lib/unidecoder/data/xb5.yml
+- lib/unidecoder/data/xb6.yml
+- lib/unidecoder/data/xb7.yml
+- lib/unidecoder/data/xb8.yml
+- lib/unidecoder/data/xb9.yml
+- lib/unidecoder/data/xba.yml
+- lib/unidecoder/data/xbb.yml
+- lib/unidecoder/data/xbc.yml
+- lib/unidecoder/data/xbd.yml
+- lib/unidecoder/data/xbe.yml
+- lib/unidecoder/data/xbf.yml
+- lib/unidecoder/data/xc0.yml
+- lib/unidecoder/data/xc1.yml
+- lib/unidecoder/data/xc2.yml
+- lib/unidecoder/data/xc3.yml
+- lib/unidecoder/data/xc4.yml
+- lib/unidecoder/data/xc5.yml
+- lib/unidecoder/data/xc6.yml
+- lib/unidecoder/data/xc7.yml
+- lib/unidecoder/data/xc8.yml
+- lib/unidecoder/data/xc9.yml
+- lib/unidecoder/data/xca.yml
+- lib/unidecoder/data/xcb.yml
+- lib/unidecoder/data/xcc.yml
+- lib/unidecoder/data/xcd.yml
+- lib/unidecoder/data/xce.yml
+- lib/unidecoder/data/xcf.yml
+- lib/unidecoder/data/xd0.yml
+- lib/unidecoder/data/xd1.yml
+- lib/unidecoder/data/xd2.yml
+- lib/unidecoder/data/xd3.yml
+- lib/unidecoder/data/xd4.yml
+- lib/unidecoder/data/xd5.yml
+- lib/unidecoder/data/xd6.yml
+- lib/unidecoder/data/xd7.yml
+- lib/unidecoder/data/xf9.yml
+- lib/unidecoder/data/xfa.yml
+- lib/unidecoder/data/xfb.yml
+- lib/unidecoder/data/xfc.yml
+- lib/unidecoder/data/xfd.yml
+- lib/unidecoder/data/xfe.yml
+- lib/unidecoder/data/xff.yml
+has_rdoc: true
+homepage: http://github.com/norman/unidecoder
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: "[none]"
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Transliterates Unicode strings to an ASCII approximation.
+test_files: 
+- test/unicode_point_suite/basic_latin_test.rb
+- test/unidecoder_test.rb