blackwinter-unidecoder 1.1.1.1

data/lib/unidecoder/data/xff.yml

@@ -0,0 +1,257 @@
+---
+- '[?]'
+- '!'
+- '"'
+- '#'
+- $
+- '%'
+- '&'
+- "'"
+- (
+- )
+- '*'
+- +
+- ','
+- -
+- .
+- /
+- 0
+- 1
+- 2
+- 3
+- 4
+- 5
+- 6
+- 7
+- 8
+- 9
+- ':'
+- ;
+- <
+- =
+- '>'
+- '?'
+- '@'
+- A
+- B
+- C
+- D
+- E
+- F
+- G
+- H
+- I
+- J
+- K
+- L
+- M
+- N
+- O
+- P
+- Q
+- R
+- S
+- T
+- U
+- V
+- W
+- X
+- Y
+- Z
+- '['
+- \
+- ']'
+- '^'
+- _
+- '`'
+- a
+- b
+- c
+- d
+- e
+- f
+- g
+- h
+- i
+- j
+- k
+- l
+- m
+- n
+- o
+- p
+- q
+- r
+- s
+- t
+- u
+- v
+- w
+- x
+- y
+- z
+- '{'
+- '|'
+- '}'
+- '~'
+- '[?]'
+- '[?]'
+- .
+- '['
+- ']'
+- ','
+- '*'
+- wo
+- a
+- i
+- u
+- e
+- o
+- ya
+- yu
+- yo
+- tu
+- +
+- a
+- i
+- u
+- e
+- o
+- ka
+- ki
+- ku
+- ke
+- ko
+- sa
+- si
+- su
+- se
+- so
+- ta
+- ti
+- tu
+- te
+- to
+- na
+- ni
+- nu
+- ne
+- no
+- ha
+- hi
+- hu
+- he
+- ho
+- ma
+- mi
+- mu
+- me
+- mo
+- ya
+- yu
+- yo
+- ra
+- ri
+- ru
+- re
+- ro
+- wa
+- n
+- ':'
+- ;
+- ''
+- g
+- gg
+- gs
+- n
+- nj
+- nh
+- d
+- dd
+- r
+- lg
+- lm
+- lb
+- ls
+- lt
+- lp
+- rh
+- m
+- b
+- bb
+- bs
+- s
+- ss
+- ''
+- j
+- jj
+- c
+- k
+- t
+- p
+- h
+- '[?]'
+- '[?]'
+- '[?]'
+- a
+- ae
+- ya
+- yae
+- eo
+- e
+- '[?]'
+- '[?]'
+- yeo
+- ye
+- o
+- wa
+- wae
+- oe
+- '[?]'
+- '[?]'
+- yo
+- u
+- weo
+- we
+- wi
+- yu
+- '[?]'
+- '[?]'
+- eu
+- yi
+- i
+- '[?]'
+- '[?]'
+- '[?]'
+- /C
+- PS
+- '!'
+- -
+- '|'
+- Y=
+- W=
+- '[?]'
+- '|'
+- -
+- '|'
+- -
+- '|'
+- '#'
+- O
+- '[?]'
+- '[?]'
+- '[?]'
+- '[?]'
+- '[?]'
+- '[?]'
+- '[?]'
+- '[?]'
+- '[?]'
+- '[?]'
+- '{'
+- '|'
+- '}'
+- ''
+- ''
+- ''
+- ''

data/lib/unidecoder/version.rb

@@ -0,0 +1,9 @@
+module Unidecoder
+  module Version
+    MAJOR = 1
+    MINOR = 1
+    TINY = 1
+    BUILD = 1
+    STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
+  end
+end

data/test/unicode_point_suite/basic_latin_test.rb

@@ -0,0 +1,144 @@
+$:.unshift File.expand_path("../../lib", File.dirname(__FILE__))
+$:.unshift File.expand_path(File.dirname(__FILE__))
+$:.uniq!
+require "test/unit"
+require "unidecoder"
+require "codepoint_test_helper"
+
+include CodepointTestHelper
+
+class BasicLatinTest < Test::Unit::TestCase
+  # This test suite is just regression test and debugging
+  # to better transliterate the Basic Latin Unicode codepoints
+  #
+  # http://unicode.org/charts/
+  # http://unicode.org/charts/PDF/U0000.pdf
+
+  # NOTE: I can't figure out how to test control characters.
+  # Get weird results trying to pack them to unicode.
+
+  def test_spaces
+    assert_equal_encoded " ", %w{0020 00a0}
+    assert_equal_encoded "",  %w{200b 2060}
+  end
+
+  def test_exclamation_marks
+    assert_equal_encoded "!", %w{0021 2762}
+    assert_equal_encoded "!!", "203c"
+    assert_equal_encoded "", "00a1"
+    assert_equal_encoded "?!", "203d"
+  end
+
+  def test_quotation_marks
+    assert_equal_encoded "\"", %w{0022 02ba 2033 3003}
+  end
+
+  def test_apostrophes
+    assert_equal_encoded "'", %w{0027 02b9 02bc 02c8 2032}
+  end
+
+  def test_asterisks
+    assert_equal_encoded "*", %w{002a 066d 204e 2217 26b9 2731}
+  end
+
+  def test_commas
+    assert_equal_encoded ",", %w{002c 060c}
+  end
+
+  def test_periods
+    assert_equal_encoded ".", %w{002e 06d4}
+  end
+
+  def test_hyphens
+    assert_equal_encoded "-", %w{002d 2010 2011 2012 2212}
+  end
+
+  def test_endash
+    assert_equal_encoded "--", %w{2013 2015}
+  end
+
+  def test_emdash
+    assert_equal_encoded "---", %w{2014}
+  end
+
+  def test_dotleader
+    assert_equal_encoded "..", %w{2025}
+  end
+
+  def test_ellipsis
+    assert_equal_encoded "...", %w{2026}
+  end
+
+  def test_slashes
+    assert_equal_encoded "/", %w{002f 2044 2215}
+    assert_equal_encoded "\\", %w{005c 2216}
+  end
+
+  def test_colons
+    assert_equal_encoded ":", %w{003a 2236}
+  end
+
+  def test_semicolons
+    assert_equal_encoded ";", %w{003b 061b}
+  end
+
+  def test_less_thans
+    assert_equal_encoded "<", %w{003c 2039 2329 27e8 3008}
+  end
+
+  def test_equals
+    assert_equal_encoded "=", "003d"
+  end
+
+  def test_greater_thans
+    assert_equal_encoded ">", %w{003e 203a 232a 27e9 3009}
+  end
+
+  def test_question_marks
+    assert_equal_encoded "?", %w{003f 061f}
+    assert_equal_encoded "", "00bf"
+    assert_equal_encoded "?!", %w{203d 2048}
+    assert_equal_encoded "!?", "2049"
+  end
+
+  def test_circumflexes
+    assert_equal_encoded "^", %w{005e 2038 2303}
+  end
+
+  def test_underscores
+    assert_equal_encoded "_", %w{005f 02cd 2017}
+  end
+
+  def test_grave_accents
+    assert_equal_encoded "`", %w{0060 02cb 2035}
+  end
+
+  def test_bars
+    assert_equal_encoded "|", %w{007c 2223 2758}
+  end
+
+  def test_tildes
+    assert_equal_encoded "~", %w{007e 02dc 2053 223c ff5e}
+  end
+
+  def test_related_letters
+    {
+      "B" => "212c",
+      "C" => %w{2102 212d},
+      "E" => %w{2107 2130},
+      "F" => "2131",
+      "H" => %w{210b 210c 210d},
+      "I" => %w{0130 0406 04c0 2110 2111 2160},
+      "K" => "212a",
+      "L" => "2112",
+      "M" => "2133",
+      "N" => "2115",
+      "P" => "2119",
+      "Q" => "211a",
+      "R" => %w{211b 211c 211d},
+      "Z" => %w{2124 2128}
+    }.each do |expected, encode_mes|
+      assert_equal_encoded expected, encode_mes
+    end
+  end
+end

data/test/unicode_point_suite/codepoint_test_helper.rb

@@ -0,0 +1,28 @@
+# 100% shorthand
+module CodepointTestHelper
+  def assert_equal_encoded(expected, encode_mes)
+    # Killing a duck because Ruby 1.9 doesn't mix Enumerable into String
+    encode_mes = [encode_mes] if encode_mes.is_a?(String)
+    encode_mes.each do |encode_me|
+      encoded = encode(encode_me)
+      actual = encoded.to_ascii
+      if expected == actual
+        # Let's not retest it
+        assert true
+      else
+        message = "<#{expected.inspect}> expected but was\n<#{actual.inspect}>\n"
+        message << "  defined in #{Unidecoder.in_yaml_file(encoded)}"
+        fail message
+      end
+    end
+  end
+
+private
+  def encode(codepoint)
+    Unidecoder.encode(codepoint)
+  end
+
+  def which_yaml(codepoint)
+    Unidecoder.in_yaml_file(encode(codepoint))
+  end
+end

data/test/unidecoder_test.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+$:.unshift File.expand_path("../lib", File.dirname(__FILE__))
+$:.unshift File.expand_path(File.dirname(__FILE__))
+$:.uniq!
+require "test/unit"
+require "unidecoder"
+
+
+class UnidecoderTest < Test::Unit::TestCase
+  # Silly phrases courtesy of Frank da Cruz (http://www.columbia.edu/kermit/utf8.html).
+
+  DONT_CONVERT = [
+    "Vitrum edere possum; mihi non nocet.",               # Latin
+    "Je puis mangier del voirre. Ne me nuit.",            # Old French
+    "Kristala jan dezaket, ez dit minik ematen.",         # Basque
+    "Kaya kong kumain nang bubog at hindi ako masaktan.", # Tagalog
+    "Ich kann Glas essen, ohne mir weh zu tun.",          # German
+    "I can eat glass and it doesn't hurt me.",            # English
+  ]
+
+  CONVERT_PAIRS = {
+    # French
+    "Je peux manger du verre, ça ne me fait pas de mal."     => "Je peux manger du verre, ca ne me fait pas de mal.",
+    # Romanian
+    "Pot să mănânc sticlă și ea nu mă rănește."              => "Pot sa mananc sticla si ea nu ma raneste.",
+    # Icelandic
+    "Ég get etið gler án þess að meiða mig."                 => "Eg get etid gler an thess ad meida mig.",
+    # Albanian
+    "Unë mund të ha qelq dhe nuk më gjen gjë."               => "Une mund te ha qelq dhe nuk me gjen gje.",
+    # Polish
+    "Mogę jeść szkło i mi nie szkodzi."                      => "Moge jesc szklo i mi nie szkodzi.",
+    # Russian
+    "Я могу есть стекло, оно мне не вредит."                 => "Ia moghu iest' stieklo, ono mnie nie vriedit.",
+    # Bulgarian
+    "Мога да ям стъкло, то не ми вреди."                     => "Mogha da iam stklo, to nie mi vriedi.",
+    # Anglo-Saxon
+    "ᛁᚳ᛫ᛗᚨᚷ᛫ᚷᛚᚨᛋ᛫ᛖᚩᛏᚪᚾ᛫ᚩᚾᛞ᛫ᚻᛁᛏ᛫ᚾᛖ᛫ᚻᛖᚪᚱᛗᛁᚪᚧ᛫ᛗᛖ᛬"              => "ic.mag.glas.eotacn.ond.hit.ne.heacrmiacth.me:",
+    # Classical Greek
+    "ὕαλον ϕαγεῖν δύναμαι· τοῦτο οὔ με βλάπτει"              => "ualon phagein dunamai; touto ou me blaptei",
+    # Hindi
+    "मैं काँच खा सकता हूँ और मुझे उससे कोई चोट नहीं पहुंचती" => "maiN kaaNc khaa sktaa huuN aur mujhe usse koii cott nhiiN phuNctii",
+    # Thai
+    "ฉันกินกระจกได้ แต่มันไม่ทำให้ฉันเจ็บ"                   => "chankinkracchkaid aetmanaimthamaihchanecchb",
+    # Chinese
+    "我能吞下玻璃而不伤身体。"                                           => "Wo Neng Tun Xia Bo Li Er Bu Shang Shen Ti . ",
+    # Japanese
+    "私はガラスを食べられます。それは私を傷つけません。"                              => "Si hagarasuwoShi beraremasu. sorehaSi woShang tukemasen. ",
+    "من می توانم بدونِ احساس درد شيشه بخورم" => # Persian
+      "mn my twnm bdwni Hss drd shyshh bkhwrm",
+    "أنا قادر على أكل الزجاج و هذا لا يؤلمن" => # Arabic
+      "'n qdr `l~ 'kl lzjj w hdh l yw'lmn",
+    "אני יכול לאכול זכוכית וזה לא מזיק לי" => # Hebrew
+      "ny ykvl lkvl zkvkyt vzh l mzyq ly",
+  }
+
+  def test_should_raise_error_with_invalid_utf8
+    [
+      "\x80",  # Continuation byte, low (cp125)
+      "\x94",  # Continuation byte, mid (cp125)
+      "\x9F",  # Continuation byte, high (cp125)
+      "\xC0",  # Overlong encoding, start of 2-byte sequence, but codepoint < 128
+      "\xC1",  # Overlong encoding, start of 2-byte sequence, but codepoint < 128
+      "\xC2",  # Start of 2-byte sequence, low
+      "\xC8",  # Start of 2-byte sequence, mid
+      "\xDF",  # Start of 2-byte sequence, high
+      "\xE0",  # Start of 3-byte sequence, low
+      "\xE8",  # Start of 3-byte sequence, mid
+      "\xEF",  # Start of 3-byte sequence, high
+      "\xF0",  # Start of 4-byte sequence
+      "\xF1",  # Unused byte
+      "\xFF",  # Restricted byte
+    ].map do |byte|
+      assert_raise ArgumentError, "#{byte.inspect} did not raise error" do
+        Unidecoder.decode("a#{byte}a")
+      end
+    end
+
+  end
+
+  def test_unidecoder_decode
+    DONT_CONVERT.each do |ascii|
+      assert_equal ascii, Unidecoder.decode(ascii)
+    end
+    CONVERT_PAIRS.each do |unicode, ascii|
+      assert_equal ascii, Unidecoder.decode(unicode)
+    end
+  end
+
+  def test_unidecoder_encode
+    {
+      # Strings
+      "0041" => "A",
+      "00e6" => "æ",
+      "042f" => "Я"
+    }.each do |codepoint, unicode|
+      assert_equal unicode, Unidecoder.encode(codepoint)
+    end
+  end
+
+  def test_unidecoder_in_yaml_file
+    {
+      "A" => "x00.yml (line 67)",
+      "π" => "x03.yml (line 194)",
+      "Я" => "x04.yml (line 49)"
+    }.each do |character, output|
+      assert_equal output, Unidecoder.in_yaml_file(character)
+    end
+  end
+
+  def test_override
+    assert_equal "Juergen", Unidecoder.decode("Jürgen", "ü" => "ue")
+  end
+
+end