twitter_cldr 1.3.6 → 1.4.0

data/spec/normalizers/NormalizationTestShort.txt

@@ -509,71 +509,71 @@ FF35;FF35;FF35;0055;0055; # (Ｕ; Ｕ; Ｕ; U; U; ) FULLWIDTH LATIN CAPITAL LETT
 FF95;FF95;FF95;30E6;30E6; # (ﾕ; ﾕ; ﾕ; ユ; ユ; ) HALFWIDTH KATAKANA LETTER YU
 FFED;FFED;FFED;25A0;25A0; # (￭; ￭; ￭; ■; ■; ) HALFWIDTH BLACK SQUARE
 FFA5;FFA5;FFA5;11AC;11AC; # (ﾥ; ﾥ; ﾥ; ᆬ; ᆬ; ) HALFWIDTH HANGUL LETTER NIEUN-CIEUC
-110AB;110AB;110A5 110BA;110AB;110A5 110BA; # (𑂫; 𑂫; 𑂥◌𑂺; 𑂫; 𑂥◌𑂺; ) KAITHI LETTER VA
-1109C;1109C;1109B 110BA;1109C;1109B 110BA; # (𑂜; 𑂜; 𑂛◌𑂺; 𑂜; 𑂛◌𑂺; ) KAITHI LETTER RHA
-1109A;1109A;11099 110BA;1109A;11099 110BA; # (𑂚; 𑂚; 𑂙◌𑂺; 𑂚; 𑂙◌𑂺; ) KAITHI LETTER DDDHA
-1112F;1112F;11132 11127;1112F;11132 11127; # (◌𑄯; ◌𑄯; ◌𑄲◌𑄧; ◌𑄯; ◌𑄲◌𑄧; ) CHAKMA VOWEL SIGN AU
-1112E;1112E;11131 11127;1112E;11131 11127; # (◌𑄮; ◌𑄮; ◌𑄱◌𑄧; ◌𑄮; ◌𑄱◌𑄧; ) CHAKMA VOWEL SIGN O
-1D15F;1D158 1D165;1D158 1D165;1D158 1D165;1D158 1D165; # (𝅘𝅥; 𝅘𝅥; 𝅘𝅥; 𝅘𝅥; 𝅘𝅥; ) MUSICAL SYMBOL QUARTER NOTE
-1D161;1D158 1D165 1D16F;1D158 1D165 1D16F;1D158 1D165 1D16F;1D158 1D165 1D16F; # (𝅘𝅥𝅯; 𝅘𝅥𝅯; 𝅘𝅥𝅯; 𝅘𝅥𝅯; 𝅘𝅥𝅯; ) MUSICAL SYMBOL SIXTEENTH NOTE
-1D1BB;1D1B9 1D165;1D1B9 1D165;1D1B9 1D165;1D1B9 1D165; # (𝆹𝅥; 𝆹𝅥; 𝆹𝅥; 𝆹𝅥; 𝆹𝅥; ) MUSICAL SYMBOL MINIMA
-1D160;1D158 1D165 1D16E;1D158 1D165 1D16E;1D158 1D165 1D16E;1D158 1D165 1D16E; # (𝅘𝅥𝅮; 𝅘𝅥𝅮; 𝅘𝅥𝅮; 𝅘𝅥𝅮; 𝅘𝅥𝅮; ) MUSICAL SYMBOL EIGHTH NOTE
-1D162;1D158 1D165 1D170;1D158 1D165 1D170;1D158 1D165 1D170;1D158 1D165 1D170; # (𝅘𝅥𝅰; 𝅘𝅥𝅰; 𝅘𝅥𝅰; 𝅘𝅥𝅰; 𝅘𝅥𝅰; ) MUSICAL SYMBOL THIRTY-SECOND NOTE
-1D163;1D158 1D165 1D171;1D158 1D165 1D171;1D158 1D165 1D171;1D158 1D165 1D171; # (𝅘𝅥𝅱; 𝅘𝅥𝅱; 𝅘𝅥𝅱; 𝅘𝅥𝅱; 𝅘𝅥𝅱; ) MUSICAL SYMBOL SIXTY-FOURTH NOTE
-1D1BC;1D1BA 1D165;1D1BA 1D165;1D1BA 1D165;1D1BA 1D165; # (𝆺𝅥; 𝆺𝅥; 𝆺𝅥; 𝆺𝅥; 𝆺𝅥; ) MUSICAL SYMBOL MINIMA BLACK
-1D15E;1D157 1D165;1D157 1D165;1D157 1D165;1D157 1D165; # (𝅗𝅥; 𝅗𝅥; 𝅗𝅥; 𝅗𝅥; 𝅗𝅥; ) MUSICAL SYMBOL HALF NOTE
-1D1BE;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E; # (𝆺𝅥𝅮; 𝆺𝅥𝅮; 𝆺𝅥𝅮; 𝆺𝅥𝅮; 𝆺𝅥𝅮; ) MUSICAL SYMBOL SEMIMINIMA BLACK
-1D164;1D158 1D165 1D172;1D158 1D165 1D172;1D158 1D165 1D172;1D158 1D165 1D172; # (𝅘𝅥𝅲; 𝅘𝅥𝅲; 𝅘𝅥𝅲; 𝅘𝅥𝅲; 𝅘𝅥𝅲; ) MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE
-1D50D;1D50D;1D50D;004A;004A; # (𝔍; 𝔍; 𝔍; J; J; ) MATHEMATICAL FRAKTUR CAPITAL J
-1D538;1D538;1D538;0041;0041; # (𝔸; 𝔸; 𝔸; A; A; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL A
-1D65B;1D65B;1D65B;0066;0066; # (𝙛; 𝙛; 𝙛; f; f; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL F
-1D73E;1D73E;1D73E;03B9;03B9; # (𝜾; 𝜾; 𝜾; ι; ι; ) MATHEMATICAL BOLD ITALIC SMALL IOTA
-1D57F;1D57F;1D57F;0054;0054; # (𝕿; 𝕿; 𝕿; T; T; ) MATHEMATICAL BOLD FRAKTUR CAPITAL T
-1D6FE;1D6FE;1D6FE;03B3;03B3; # (𝛾; 𝛾; 𝛾; γ; γ; ) MATHEMATICAL ITALIC SMALL GAMMA
-1D78A;1D78A;1D78A;03B5;03B5; # (𝞊; 𝞊; 𝞊; ε; ε; ) MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL
-1D7F3;1D7F3;1D7F3;0037;0037; # (𝟳; 𝟳; 𝟳; 7; 7; ) MATHEMATICAL SANS-SERIF BOLD DIGIT SEVEN
-1D778;1D778;1D778;03B9;03B9; # (𝝸; 𝝸; 𝝸; ι; ι; ) MATHEMATICAL SANS-SERIF BOLD SMALL IOTA
-1D512;1D512;1D512;004F;004F; # (𝔒; 𝔒; 𝔒; O; O; ) MATHEMATICAL FRAKTUR CAPITAL O
-1EE27;1EE27;1EE27;062D;062D; # (𞸧; 𞸧; 𞸧; ح; ح; ) ARABIC MATHEMATICAL INITIAL HAH
-1EE89;1EE89;1EE89;064A;064A; # (𞺉; 𞺉; 𞺉; ي; ي; ) ARABIC MATHEMATICAL LOOPED YEH
-1EE08;1EE08;1EE08;0637;0637; # (𞸈; 𞸈; 𞸈; ط; ط; ) ARABIC MATHEMATICAL TAH
-1EEB0;1EEB0;1EEB0;0641;0641; # (𞺰; 𞺰; 𞺰; ف; ف; ) ARABIC MATHEMATICAL DOUBLE-STRUCK FEH
-1EEBA;1EEBA;1EEBA;0638;0638; # (𞺺; 𞺺; 𞺺; ظ; ظ; ) ARABIC MATHEMATICAL DOUBLE-STRUCK ZAH
-1EE11;1EE11;1EE11;0635;0635; # (𞸑; 𞸑; 𞸑; ص; ص; ) ARABIC MATHEMATICAL SAD
-1EE98;1EE98;1EE98;0630;0630; # (𞺘; 𞺘; 𞺘; ذ; ذ; ) ARABIC MATHEMATICAL LOOPED THAL
-1EEA2;1EEA2;1EEA2;062C;062C; # (𞺢; 𞺢; 𞺢; ج; ج; ) ARABIC MATHEMATICAL DOUBLE-STRUCK JEEM
-1EE4D;1EE4D;1EE4D;0646;0646; # (𞹍; 𞹍; 𞹍; ن; ن; ) ARABIC MATHEMATICAL TAILED NOON
-1EE4F;1EE4F;1EE4F;0639;0639; # (𞹏; 𞹏; 𞹏; ع; ع; ) ARABIC MATHEMATICAL TAILED AIN
-1F132;1F132;1F132;0043;0043; # (🄲; 🄲; 🄲; C; C; ) SQUARED LATIN CAPITAL LETTER C
-1F13E;1F13E;1F13E;004F;004F; # (🄾; 🄾; 🄾; O; O; ) SQUARED LATIN CAPITAL LETTER O
-1F11C;1F11C;1F11C;0028 004D 0029;0028 004D 0029; # (🄜; 🄜; 🄜; (M); (M); ) PARENTHESIZED LATIN CAPITAL LETTER M
-1F102;1F102;1F102;0031 002C;0031 002C; # (🄂; 🄂; 🄂; 1,; 1,; ) DIGIT ONE COMMA
-1F16B;1F16B;1F16B;004D 0044;004D 0044; # (🅫; 🅫; 🅫; MD; MD; ) RAISED MD SIGN
-1F11D;1F11D;1F11D;0028 004E 0029;0028 004E 0029; # (🄝; 🄝; 🄝; (N); (N); ) PARENTHESIZED LATIN CAPITAL LETTER N
-1F146;1F146;1F146;0057;0057; # (🅆; 🅆; 🅆; W; W; ) SQUARED LATIN CAPITAL LETTER W
-1F107;1F107;1F107;0036 002C;0036 002C; # (🄇; 🄇; 🄇; 6,; 6,; ) DIGIT SIX COMMA
-1F145;1F145;1F145;0056;0056; # (🅅; 🅅; 🅅; V; V; ) SQUARED LATIN CAPITAL LETTER V
-1F112;1F112;1F112;0028 0043 0029;0028 0043 0029; # (🄒; 🄒; 🄒; (C); (C); ) PARENTHESIZED LATIN CAPITAL LETTER C
-1F231;1F231;1F231;6253;6253; # (🈱; 🈱; 🈱; 打; 打; ) SQUARED CJK UNIFIED IDEOGRAPH-6253
-1F22B;1F22B;1F22B;904A;904A; # (🈫; 🈫; 🈫; 遊; 遊; ) SQUARED CJK UNIFIED IDEOGRAPH-904A
-1F243;1F243;1F243;3014 5B89 3015;3014 5B89 3015; # (🉃; 🉃; 🉃; 〔安〕; 〔安〕; ) TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-5B89
-1F234;1F234;1F234;5408;5408; # (🈴; 🈴; 🈴; 合; 合; ) SQUARED CJK UNIFIED IDEOGRAPH-5408
-1F238;1F238;1F238;7533;7533; # (🈸; 🈸; 🈸; 申; 申; ) SQUARED CJK UNIFIED IDEOGRAPH-7533
-1F247;1F247;1F247;3014 52DD 3015;3014 52DD 3015; # (🉇; 🉇; 🉇; 〔勝〕; 〔勝〕; ) TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-52DD
-1F217;1F217;1F217;5929;5929; # (🈗; 🈗; 🈗; 天; 天; ) SQUARED CJK UNIFIED IDEOGRAPH-5929
-1F248;1F248;1F248;3014 6557 3015;3014 6557 3015; # (🉈; 🉈; 🉈; 〔敗〕; 〔敗〕; ) TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557
-1F224;1F224;1F224;58F0;58F0; # (🈤; 🈤; 🈤; 声; 声; ) SQUARED CJK UNIFIED IDEOGRAPH-58F0
-1F213;1F213;1F213;30C7;30C6 3099; # (🈓; 🈓; 🈓; デ; テ◌゙; ) SQUARED KATAKANA DE
-2F984;440B;440B;440B;440B; # (䐋; 䐋; 䐋; 䐋; 䐋; ) CJK COMPATIBILITY IDEOGRAPH-2F984
-2F9A4;26C36;26C36;26C36;26C36; # (𦰶; 𦰶; 𦰶; 𦰶; 𦰶; ) CJK COMPATIBILITY IDEOGRAPH-2F9A4
-2F910;23F5E;23F5E;23F5E;23F5E; # (𣽞; 𣽞; 𣽞; 𣽞; 𣽞; ) CJK COMPATIBILITY IDEOGRAPH-2F910
-2F9F7;2921A;2921A;2921A;2921A; # (𩈚; 𩈚; 𩈚; 𩈚; 𩈚; ) CJK COMPATIBILITY IDEOGRAPH-2F9F7
-2F97F;8070;8070;8070;8070; # (聰; 聰; 聰; 聰; 聰; ) CJK COMPATIBILITY IDEOGRAPH-2F97F
-2F954;2569A;2569A;2569A;2569A; # (𥚚; 𥚚; 𥚚; 𥚚; 𥚚; ) CJK COMPATIBILITY IDEOGRAPH-2F954
-2F96C;7D63;7D63;7D63;7D63; # (絣; 絣; 絣; 絣; 絣; ) CJK COMPATIBILITY IDEOGRAPH-2F96C
-2FA1B;9F16;9F16;9F16;9F16; # (鼖; 鼖; 鼖; 鼖; 鼖; ) CJK COMPATIBILITY IDEOGRAPH-2FA1B
-2F92D;3EB8;3EB8;3EB8;3EB8; # (㺸; 㺸; 㺸; 㺸; 㺸; ) CJK COMPATIBILITY IDEOGRAPH-2F92D
-2F9C7;88DE;88DE;88DE;88DE; # (裞; 裞; 裞; 裞; 裞; ) CJK COMPATIBILITY IDEOGRAPH-2F9C7
+110AB;110AB;110A5 110BA;110AB;110A5 110BA; # (Ⴋ; Ⴋ; Ⴅ◌Ⴚ; Ⴋ; Ⴅ◌Ⴚ; ) KAITHI LETTER VA
+1109C;1109C;1109B 110BA;1109C;1109B 110BA; # (ႜ; ႜ; ႛ◌Ⴚ; ႜ; ႛ◌Ⴚ; ) KAITHI LETTER RHA
+1109A;1109A;11099 110BA;1109A;11099 110BA; # (ႚ; ႚ; ႙◌Ⴚ; ႚ; ႙◌Ⴚ; ) KAITHI LETTER DDDHA
+1112F;1112F;11132 11127;1112F;11132 11127; # (◌ᄯ; ◌ᄯ; ◌ᄲ◌ᄧ; ◌ᄯ; ◌ᄲ◌ᄧ; ) CHAKMA VOWEL SIGN AU
+1112E;1112E;11131 11127;1112E;11131 11127; # (◌ᄮ; ◌ᄮ; ◌ᄱ◌ᄧ; ◌ᄮ; ◌ᄱ◌ᄧ; ) CHAKMA VOWEL SIGN O
+1D15F;1D158 1D165;1D158 1D165;1D158 1D165;1D158 1D165; # (텟; 텘텥; 텘텥; 텘텥; 텘텥; ) MUSICAL SYMBOL QUARTER NOTE
+1D161;1D158 1D165 1D16F;1D158 1D165 1D16F;1D158 1D165 1D16F;1D158 1D165 1D16F; # (텡; 텘텥텯; 텘텥텯; 텘텥텯; 텘텥텯; ) MUSICAL SYMBOL SIXTEENTH NOTE
+1D1BB;1D1B9 1D165;1D1B9 1D165;1D1B9 1D165;1D1B9 1D165; # (톻; 톹텥; 톹텥; 톹텥; 톹텥; ) MUSICAL SYMBOL MINIMA
+1D160;1D158 1D165 1D16E;1D158 1D165 1D16E;1D158 1D165 1D16E;1D158 1D165 1D16E; # (텠; 텘텥텮; 텘텥텮; 텘텥텮; 텘텥텮; ) MUSICAL SYMBOL EIGHTH NOTE
+1D162;1D158 1D165 1D170;1D158 1D165 1D170;1D158 1D165 1D170;1D158 1D165 1D170; # (텢; 텘텥텰; 텘텥텰; 텘텥텰; 텘텥텰; ) MUSICAL SYMBOL THIRTY-SECOND NOTE
+1D163;1D158 1D165 1D171;1D158 1D165 1D171;1D158 1D165 1D171;1D158 1D165 1D171; # (텣; 텘텥텱; 텘텥텱; 텘텥텱; 텘텥텱; ) MUSICAL SYMBOL SIXTY-FOURTH NOTE
+1D1BC;1D1BA 1D165;1D1BA 1D165;1D1BA 1D165;1D1BA 1D165; # (톼; 톺텥; 톺텥; 톺텥; 톺텥; ) MUSICAL SYMBOL MINIMA BLACK
+1D15E;1D157 1D165;1D157 1D165;1D157 1D165;1D157 1D165; # (텞; 텗텥; 텗텥; 텗텥; 텗텥; ) MUSICAL SYMBOL HALF NOTE
+1D1BE;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E; # (톾; 톺텥텮; 톺텥텮; 톺텥텮; 톺텥텮; ) MUSICAL SYMBOL SEMIMINIMA BLACK
+1D164;1D158 1D165 1D172;1D158 1D165 1D172;1D158 1D165 1D172;1D158 1D165 1D172; # (텤; 텘텥텲; 텘텥텲; 텘텥텲; 텘텥텲; ) MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE
+1D50D;1D50D;1D50D;004A;004A; # (픍; 픍; 픍; J; J; ) MATHEMATICAL FRAKTUR CAPITAL J
+1D538;1D538;1D538;0041;0041; # (픸; 픸; 픸; A; A; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL A
+1D65B;1D65B;1D65B;0066;0066; # (홛; 홛; 홛; f; f; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL F
+1D73E;1D73E;1D73E;03B9;03B9; # (휾; 휾; 휾; ι; ι; ) MATHEMATICAL BOLD ITALIC SMALL IOTA
+1D57F;1D57F;1D57F;0054;0054; # (핿; 핿; 핿; T; T; ) MATHEMATICAL BOLD FRAKTUR CAPITAL T
+1D6FE;1D6FE;1D6FE;03B3;03B3; # (훾; 훾; 훾; γ; γ; ) MATHEMATICAL ITALIC SMALL GAMMA
+1D78A;1D78A;1D78A;03B5;03B5; # (힊; 힊; 힊; ε; ε; ) MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL
+1D7F3;1D7F3;1D7F3;0037;0037; # (ퟳ; ퟳ; ퟳ; 7; 7; ) MATHEMATICAL SANS-SERIF BOLD DIGIT SEVEN
+1D778;1D778;1D778;03B9;03B9; # (흸; 흸; 흸; ι; ι; ) MATHEMATICAL SANS-SERIF BOLD SMALL IOTA
+1D512;1D512;1D512;004F;004F; # (픒; 픒; 픒; O; O; ) MATHEMATICAL FRAKTUR CAPITAL O
+1EE27;1EE27;1EE27;062D;062D; # (; ; ; ح; ح; ) ARABIC MATHEMATICAL INITIAL HAH
+1EE89;1EE89;1EE89;064A;064A; # (; ; ; ي; ي; ) ARABIC MATHEMATICAL LOOPED YEH
+1EE08;1EE08;1EE08;0637;0637; # (; ; ; ط; ط; ) ARABIC MATHEMATICAL TAH
+1EEB0;1EEB0;1EEB0;0641;0641; # (; ; ; ف; ف; ) ARABIC MATHEMATICAL DOUBLE-STRUCK FEH
+1EEBA;1EEBA;1EEBA;0638;0638; # (; ; ; ظ; ظ; ) ARABIC MATHEMATICAL DOUBLE-STRUCK ZAH
+1EE11;1EE11;1EE11;0635;0635; # (; ; ; ص; ص; ) ARABIC MATHEMATICAL SAD
+1EE98;1EE98;1EE98;0630;0630; # (; ; ; ذ; ذ; ) ARABIC MATHEMATICAL LOOPED THAL
+1EEA2;1EEA2;1EEA2;062C;062C; # (; ; ; ج; ج; ) ARABIC MATHEMATICAL DOUBLE-STRUCK JEEM
+1EE4D;1EE4D;1EE4D;0646;0646; # (; ; ; ن; ن; ) ARABIC MATHEMATICAL TAILED NOON
+1EE4F;1EE4F;1EE4F;0639;0639; # (; ; ; ع; ع; ) ARABIC MATHEMATICAL TAILED AIN
+1F132;1F132;1F132;0043;0043; # (; ; ; C; C; ) SQUARED LATIN CAPITAL LETTER C
+1F13E;1F13E;1F13E;004F;004F; # (; ; ; O; O; ) SQUARED LATIN CAPITAL LETTER O
+1F11C;1F11C;1F11C;0028 004D 0029;0028 004D 0029; # (; ; ; (M); (M); ) PARENTHESIZED LATIN CAPITAL LETTER M
+1F102;1F102;1F102;0031 002C;0031 002C; # (; ; ; 1,; 1,; ) DIGIT ONE COMMA
+1F16B;1F16B;1F16B;004D 0044;004D 0044; # (; ; ; MD; MD; ) RAISED MD SIGN
+1F11D;1F11D;1F11D;0028 004E 0029;0028 004E 0029; # (; ; ; (N); (N); ) PARENTHESIZED LATIN CAPITAL LETTER N
+1F146;1F146;1F146;0057;0057; # (; ; ; W; W; ) SQUARED LATIN CAPITAL LETTER W
+1F107;1F107;1F107;0036 002C;0036 002C; # (; ; ; 6,; 6,; ) DIGIT SIX COMMA
+1F145;1F145;1F145;0056;0056; # (; ; ; V; V; ) SQUARED LATIN CAPITAL LETTER V
+1F112;1F112;1F112;0028 0043 0029;0028 0043 0029; # (; ; ; (C); (C); ) PARENTHESIZED LATIN CAPITAL LETTER C
+1F231;1F231;1F231;6253;6253; # (; ; ; 打; 打; ) SQUARED CJK UNIFIED IDEOGRAPH-6253
+1F22B;1F22B;1F22B;904A;904A; # (; ; ; 遊; 遊; ) SQUARED CJK UNIFIED IDEOGRAPH-904A
+1F243;1F243;1F243;3014 5B89 3015;3014 5B89 3015; # (; ; ; 〔安〕; 〔安〕; ) TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-5B89
+1F234;1F234;1F234;5408;5408; # (; ; ; 合; 合; ) SQUARED CJK UNIFIED IDEOGRAPH-5408
+1F238;1F238;1F238;7533;7533; # (; ; ; 申; 申; ) SQUARED CJK UNIFIED IDEOGRAPH-7533
+1F247;1F247;1F247;3014 52DD 3015;3014 52DD 3015; # (; ; ; 〔勝〕; 〔勝〕; ) TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-52DD
+1F217;1F217;1F217;5929;5929; # (; ; ; 天; 天; ) SQUARED CJK UNIFIED IDEOGRAPH-5929
+1F248;1F248;1F248;3014 6557 3015;3014 6557 3015; # (; ; ; 〔敗〕; 〔敗〕; ) TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557
+1F224;1F224;1F224;58F0;58F0; # (; ; ; 声; 声; ) SQUARED CJK UNIFIED IDEOGRAPH-58F0
+1F213;1F213;1F213;30C7;30C6 3099; # (; ; ; デ; テ◌゙; ) SQUARED KATAKANA DE
+2F984;440B;440B;440B;440B; # (濾; 䐋; 䐋; 䐋; 䐋; ) CJK COMPATIBILITY IDEOGRAPH-2F984
+2F9A4;26C36;26C36;26C36;26C36; # (捻; 氶; 氶; 氶; 氶; ) CJK COMPATIBILITY IDEOGRAPH-2F9A4
+2F910;23F5E;23F5E;23F5E;23F5E; # (蘿; 㽞; 㽞; 㽞; 㽞; ) CJK COMPATIBILITY IDEOGRAPH-2F910
+2F9F7;2921A;2921A;2921A;2921A; # (立; 鈚; 鈚; 鈚; 鈚; ) CJK COMPATIBILITY IDEOGRAPH-2F9F7
+2F97F;8070;8070;8070;8070; # (勵; 聰; 聰; 聰; 聰; ) CJK COMPATIBILITY IDEOGRAPH-2F97F
+2F954;2569A;2569A;2569A;2569A; # (凜; 嚚; 嚚; 嚚; 嚚; ) CJK COMPATIBILITY IDEOGRAPH-2F954
+2F96C;7D63;7D63;7D63;7D63; # (塞; 絣; 絣; 絣; 絣; ) CJK COMPATIBILITY IDEOGRAPH-2F96C
+2FA1B;9F16;9F16;9F16;9F16; # (福; 鼖; 鼖; 鼖; 鼖; ) CJK COMPATIBILITY IDEOGRAPH-2FA1B
+2F92D;3EB8;3EB8;3EB8;3EB8; # (來; 㺸; 㺸; 㺸; 㺸; ) CJK COMPATIBILITY IDEOGRAPH-2F92D
+2F9C7;88DE;88DE;88DE;88DE; # (劉; 裞; 裞; 裞; 裞; ) CJK COMPATIBILITY IDEOGRAPH-2F9C7
 #
 @Part2 # Canonical Order Test
 #
@@ -582,7 +582,7 @@ FFA5;FFA5;FFA5;11AC;11AC; # (ﾥ; ﾥ; ﾥ; ᆬ; ᆬ; ) HALFWIDTH HANGUL LETTER
 0061 1DC5 0315 0300 05AE 0062;0061 05AE 1DC5 0300 0315 0062;0061 05AE 1DC5 0300 0315 0062;0061 05AE 1DC5 0300 0315 0062;0061 05AE 1DC5 0300 0315 0062; # (a◌᷅◌̕◌̀◌֮b; a◌֮◌᷅◌̀◌̕b; a◌֮◌᷅◌̀◌̕b; a◌֮◌᷅◌̀◌̕b; a◌֮◌᷅◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING GRAVE-MACRON, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B
 0061 FE20 0315 0300 05AE 0062;0061 05AE FE20 0300 0315 0062;0061 05AE FE20 0300 0315 0062;0061 05AE FE20 0300 0315 0062;0061 05AE FE20 0300 0315 0062; # (a◌︠◌̕◌̀◌֮b; a◌֮◌︠◌̀◌̕b; a◌֮◌︠◌̀◌̕b; a◌֮◌︠◌̀◌̕b; a◌֮◌︠◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LIGATURE LEFT HALF, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B
 0061 0315 0300 05AE 0658 0062;00E0 05AE 0658 0315 0062;0061 05AE 0300 0658 0315 0062;00E0 05AE 0658 0315 0062;0061 05AE 0300 0658 0315 0062; # (a◌̕◌̀◌֮◌٘b; à◌֮◌٘◌̕b; a◌֮◌̀◌٘◌̕b; à◌֮◌٘◌̕b; a◌֮◌̀◌٘◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC MARK NOON GHUNNA, LATIN SMALL LETTER B
-0061 1D182 059A 0316 302A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062; # (a◌𝆂◌֚◌̖◌〪b; a◌〪◌𝆂◌̖◌֚b; a◌〪◌𝆂◌̖◌֚b; a◌〪◌𝆂◌̖◌֚b; a◌〪◌𝆂◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING LOURE, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B
+0061 1D182 059A 0316 302A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062; # (a◌톂◌֚◌̖◌〪b; a◌〪◌톂◌̖◌֚b; a◌〪◌톂◌̖◌֚b; a◌〪◌톂◌̖◌֚b; a◌〪◌톂◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING LOURE, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B
 0061 0315 0300 05AE 1DFE 0062;00E0 05AE 1DFE 0315 0062;0061 05AE 0300 1DFE 0315 0062;00E0 05AE 1DFE 0315 0062;0061 05AE 0300 1DFE 0315 0062; # (a◌̕◌̀◌֮◌᷾b; à◌֮◌᷾◌̕b; a◌֮◌̀◌᷾◌̕b; à◌֮◌᷾◌̕b; a◌֮◌̀◌᷾◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LEFT ARROWHEAD ABOVE, LATIN SMALL LETTER B
 0061 0315 0300 05AE A679 0062;00E0 05AE A679 0315 0062;0061 05AE 0300 A679 0315 0062;00E0 05AE A679 0315 0062;0061 05AE 0300 A679 0315 0062; # (a◌̕◌̀◌֮◌ꙹb; à◌֮◌ꙹ◌̕b; a◌֮◌̀◌ꙹ◌̕b; à◌֮◌ꙹ◌̕b; a◌֮◌̀◌ꙹ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CYRILLIC LETTER YERU, LATIN SMALL LETTER B
 0061 0315 0300 05AE 0F87 0062;00E0 05AE 0F87 0315 0062;0061 05AE 0300 0F87 0315 0062;00E0 05AE 0F87 0315 0062;0061 05AE 0300 0F87 0315 0062; # (a◌̕◌̀◌֮◌྇b; à◌֮◌྇◌̕b; a◌֮◌̀◌྇◌̕b; à◌֮◌྇◌̕b; a◌֮◌̀◌྇◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, TIBETAN SIGN YANG RTAGS, LATIN SMALL LETTER B

data/spec/normalizers/base_spec.rb

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Normalizers
+
+describe Base do
+  describe "#combining_class_for" do
+    it "returns the correct combining class for select code points" do
+      Base.combining_class_for("0303").should == 230  # combining tilde
+      Base.combining_class_for("006E").should == 0    # latin letter n
+    end
+  end
+end

data/spec/normalizers/normalization_spec.rb

@@ -38,6 +38,16 @@ describe 'Unicode Normalization Algorithms' do
     it_behaves_like 'a normalization algorithm'
   end
 
+  describe NFC do
+    let(:invariants) { { 2 => [1, 2, 3], 4 => [4, 5] } }
+    it_behaves_like 'a normalization algorithm'
+  end
+
+  describe NFKC do
+    let(:invariants) { { 4 => [1, 2, 3, 4, 5] } }
+    it_behaves_like 'a normalization algorithm'
+  end
+
   # Runs standard Unicode normalization tests from `file_path` for a given `normalizer`. Expected invariants are
   # specified via `invariants` hash.
   #

data/spec/readme_spec.rb

@@ -74,6 +74,31 @@ describe "README" do
     dt.to_short_s.should == "12/12/11 21:44"
   end
 
+  it "verifies relative time spans" do
+    (DateTime.now - 1).localize.ago.should == "1 day ago"
+    (DateTime.now - 0.5).localize.ago.should == "12 hours ago"  # (i.e. half a day)
+
+    (DateTime.now + 1).localize.until.should == "In 1 day"
+    (DateTime.now + 0.5).localize.until.should == "In 12 hours"
+
+    (DateTime.now - 1).localize(:de).ago.should == "Vor 1 Tag"
+    (DateTime.now + 1).localize(:de).until.should == "In 1 Tag"
+
+    (DateTime.now - 1).localize(:de).ago(:unit => :hour).should == "Vor 24 Stunden"
+    (DateTime.now + 1).localize(:de).until(:unit => :hour).should == "In 24 Stunden"
+
+    # 86400 = 1 day in seconds, 259200 = 3 days in seconds
+    (Time.now + 86400).localize(:de).ago(:unit => :hour, :base_time => (Time.now + 259200)).should == "Vor 48 Stunden"
+
+    ts = TwitterCldr::LocalizedTimespan.new(86400, :de)
+    ts.to_s.should == "In 1 Tag"
+    ts.to_s(:hour).should == "In 24 Stunden"
+
+    ts = TwitterCldr::LocalizedTimespan.new(-86400, :de)
+    ts.to_s.should == "Vor 1 Tag"
+    ts.to_s(:hour).should == "Vor 24 Stunden"
+  end
+
   it "verifies plural rules" do
     1.localize(:ru).plural_rule.should == :one
     2.localize(:ru).plural_rule.should == :few
@@ -149,7 +174,7 @@ describe "README" do
   end
 
   it "verifies code point conversions" do
-    code_point = TwitterCldr::Shared::UnicodeData.for_code_point("1F3E9")
+    code_point = TwitterCldr::Shared::CodePoint.for_hex("1F3E9")
     code_point.name.should             == "LOVE HOTEL"
     code_point.bidi_mirrored.should    == "N"
     code_point.category.should         == "So"

data/spec/shared/code_point_spec.rb

@@ -0,0 +1,152 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Shared
+
+describe CodePoint do
+  describe "#for_hex" do
+    it "should retrieve information for any valid code point" do
+      data = CodePoint.for_hex('0301')
+      data.should be_a(CodePoint)
+      data.values.length.should == 15
+    end
+
+    it "should return nil for invalid code points" do
+      CodePoint.for_hex('abcd').should be_nil
+      CodePoint.for_hex('FFFFFFF').should be_nil
+      CodePoint.for_hex('uytukhil123').should be_nil
+    end
+
+    it "fetches valid information for the specified code point" do
+      test_data = {
+        '17D1' => ['17D1','KHMER SIGN VIRIAM','Mn','0','NSM',"","","","",'N',"","","","",""],
+        'FE91' => ['FE91','ARABIC LETTER BEH INITIAL FORM','Lo','0','AL','<initial> 0628',"","","",'N','GLYPH FOR INITIAL ARABIC BAA',"","","",""],
+        '24B5' => ['24B5','PARENTHESIZED LATIN SMALL LETTER Z','So','0','L','<compat> 0028 007A 0029',"","","",'N',"","","","",""],
+        '2128' => ['2128','BLACK-LETTER CAPITAL Z','Lu','0','L','<font> 005A',"","","",'N','BLACK-LETTER Z',"","","",""],
+        '1F241'=> ['1F241','TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-4E09','So','0','L','<compat> 3014 4E09 3015',"","","",'N',"","","","",""]
+      }
+      test_data.each_pair do |code_point, data|
+        cp_data = CodePoint.for_hex(code_point)
+        cp_data.code_point.should == data[0]
+        cp_data.name.should == data[1]
+        cp_data.category.should == data[2]
+        cp_data.combining_class.should == data[3]
+      end
+    end
+
+    it "fetches valid information for a code point within a range" do
+      test_data = {
+        '4E11' => ["4E11","<CJK Ideograph>","Lo","0","L","","","","","N","","","","",""],
+        'AC55' => ["AC55","<Hangul Syllable>","Lo","0","L","","","","","N","","","","",""],
+        'D7A1' => ["D7A1","<Hangul Syllable>","Lo","0","L","","","","","N","","","","",""],
+        'DAAA' => ["DAAA","<Non Private Use High Surrogate>","Cs","0","L","","","","","N","","","","",""],
+        'F8FE' => ["F8FE","<Private Use>","Co","0","L","","","","","N","","","","",""]
+      }
+
+      test_data.each_pair do |code_point, data|
+        cp_data = CodePoint.for_hex(code_point)
+        cp_data.code_point.should == data[0]
+        cp_data.name.should == data[1]
+        cp_data.category.should == data[2]
+        cp_data.combining_class.should == data[3]
+      end
+    end
+  end
+
+  describe "#for_decomposition" do
+    let(:decomp_map) { { :"YYYY ZZZZ" => "0ABC" } }
+
+    before(:each) do
+      # clear the decomposition map after each test so mocks/stubs work
+      CodePoint.instance_variable_set(:@decomposition_map, nil)
+      stub(CodePoint).for_hex { |code_point| "I'm code point #{code_point}" }
+    end
+
+    after(:each) do
+      # clear the decomposition map after each test so mocks/stubs work
+      CodePoint.instance_variable_set(:@decomposition_map, nil)
+    end
+
+    context "with a stubbed decomposition map" do
+      before(:each) do
+        stub(TwitterCldr).get_resource(:unicode_data, :decomposition_map) { decomp_map }
+      end
+
+      it "should return a code point with the correct value" do
+        CodePoint.for_decomposition(["YYYY", "ZZZZ"]).should == "I'm code point 0ABC"
+      end
+
+      it "should return nil if no decomposition mapping exists" do
+        CodePoint.for_decomposition(["NO"]).should be_nil
+      end
+    end
+
+    it "should cache the decomposition map" do
+      mock(TwitterCldr).get_resource(:unicode_data, :decomposition_map) { decomp_map }.once
+      CodePoint.for_decomposition(["NO"]).should be_nil
+      CodePoint.for_decomposition(["NO"]).should be_nil
+    end
+  end
+
+  describe "#hangul_type" do
+    before(:each) do
+      stub(CodePoint).hangul_blocks { { :lparts => [1..10],
+                                        :vparts => [21..30],
+                                        :tparts => [41..50],
+                                        :compositions => [1..30],
+                                        :decompositions => [31..50] } }
+    end
+
+    it "returns nil if not part of a hangul block" do
+      CodePoint.hangul_type(100.to_s(16)).should == nil
+    end
+
+    it "returns the correct part (i.e. lpart, vpart, or tpart) before composition or decomposition" do
+      CodePoint.hangul_type(5.to_s(16)).should == :lparts
+      CodePoint.hangul_type(30.to_s(16)).should == :vparts
+      CodePoint.hangul_type(41.to_s(16)).should == :tparts
+    end
+
+    it "returns composition or decomposition if no part can be found" do
+      CodePoint.hangul_type(11.to_s(16)).should == :compositions
+      CodePoint.hangul_type(40.to_s(16)).should == :decompositions
+    end
+  end
+
+  describe "#excluded_from_composition?" do
+    it "excludes anything in the list of ranges" do
+      stub(CodePoint).composition_exclusions { [10..10, 13..14, 20..30] }
+      CodePoint.excluded_from_composition?(10.to_s(16)).should be_true
+      CodePoint.excluded_from_composition?(13.to_s(16)).should be_true
+      CodePoint.excluded_from_composition?(14.to_s(16)).should be_true
+      CodePoint.excluded_from_composition?(15.to_s(16)).should be_false
+      CodePoint.excluded_from_composition?(19.to_s(16)).should be_false
+      CodePoint.excluded_from_composition?(100.to_s(16)).should be_false
+    end
+  end
+
+  describe "#get_block" do
+    it "finds the block that corresponds to the code point" do
+      stub(TwitterCldr).get_resource(:unicode_data, :blocks) { [[:klingon, 122..307], [:hirogen, 1337..2200]] }
+      CodePoint.send(:get_block, 200.to_s(16)).should == [:klingon, 122..307]
+      CodePoint.send(:get_block, 2199.to_s(16)).should == [:hirogen, 1337..2200]
+      CodePoint.send(:get_block, 100.to_s(16)).should be_nil
+    end
+  end
+
+  describe "#get_range_start" do
+    it "returns the data for a non-explicit range" do
+      block_data = { "0" => ["1337", "<CJK Ideograph Extension A, First>"] }
+      CodePoint.send(:get_range_start, "ABC", block_data).should == ["ABC", "<CJK Ideograph Extension A>"]
+    end
+
+    it "returns nil if the block data doesn't contain a non-explicit range" do
+      block_data = { "0" => ["1337", "<CJK Ideograph Extension A>"] }
+      CodePoint.send(:get_range_start, "ABC", block_data).should == nil
+    end
+  end
+end

data/spec/tokenizers/base_spec.rb

@@ -147,16 +147,6 @@ describe Base do
     end
   end
 
-  describe "#compute_cache_key" do
-    it "returns a ruby hash of all the pieces concatenated with pipe characters" do
-      @base.send(:compute_cache_key, "space", "the", "final", "frontier").should == "space|the|final|frontier".hash
-    end
-
-    it "returns zero if no arguments are passed" do
-      @base.send(:compute_cache_key).should == 0
-    end
-  end
-
   describe "#traverse" do
     before(:each) do
       @tree = { :admiral => { :captain => { :commander => { :lieutenant => "Found Me!" } } } }

data/spec/utils_spec.rb

@@ -29,4 +29,14 @@ describe TwitterCldr::Utils do
     end
 
   end
+
+  describe "#compute_cache_key" do
+    it "returns a ruby hash of all the pieces concatenated with pipe characters" do
+      TwitterCldr::Utils.compute_cache_key("space", "the", "final", "frontier").should == "space|the|final|frontier".hash
+    end
+
+    it "returns zero if no arguments are passed" do
+      TwitterCldr::Utils.compute_cache_key.should == 0
+    end
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: twitter_cldr
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 7
   prerelease: 
   segments: 
   - 1
-  - 3
-  - 6
-  version: 1.3.6
+  - 4
+  - 0
+  version: 1.4.0
 platform: ruby
 authors: 
 - Cameron Dutro
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-12 00:00:00 Z
+date: 2012-06-15 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: json
@@ -105,7 +105,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 844733855
+        hash: 3424229193
         segments: 
         - 0
         - 9
@@ -221,16 +221,19 @@ files:
 - lib/twitter_cldr/formatters/plurals/rules.rb
 - lib/twitter_cldr/formatters/plurals.rb
 - lib/twitter_cldr/formatters.rb
+- lib/twitter_cldr/normalizers/base.rb
+- lib/twitter_cldr/normalizers/nfc.rb
 - lib/twitter_cldr/normalizers/nfd.rb
+- lib/twitter_cldr/normalizers/nfkc.rb
 - lib/twitter_cldr/normalizers/nfkd.rb
 - lib/twitter_cldr/normalizers.rb
 - lib/twitter_cldr/shared/calendar.rb
+- lib/twitter_cldr/shared/code_point.rb
 - lib/twitter_cldr/shared/currencies.rb
 - lib/twitter_cldr/shared/languages.rb
 - lib/twitter_cldr/shared/numbers.rb
 - lib/twitter_cldr/shared/resources.rb
 - lib/twitter_cldr/shared/timezones.rb
-- lib/twitter_cldr/shared/unicode_data.rb
 - lib/twitter_cldr/shared.rb
 - lib/twitter_cldr/tokenizers/base.rb
 - lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb
@@ -267,17 +270,17 @@ files:
 - spec/formatters/numbers/percent_formatter_spec.rb
 - spec/formatters/plurals/plural_formatter_spec.rb
 - spec/formatters/plurals/rules_spec.rb
-- spec/normalizers/nfd_spec.rb
+- spec/normalizers/base_spec.rb
 - spec/normalizers/normalization_spec.rb
 - spec/normalizers/NormalizationTest.txt
 - spec/normalizers/NormalizationTestShort.txt
 - spec/readme_spec.rb
 - spec/shared/calendar_spec.rb
+- spec/shared/code_point_spec.rb
 - spec/shared/currencies_spec.rb
 - spec/shared/languages_spec.rb
 - spec/shared/numbers_spec.rb
 - spec/shared/resources_spec.rb
-- spec/shared/unicode_data_spec.rb
 - spec/spec_helper.rb
 - spec/tokenizers/base_spec.rb
 - spec/tokenizers/calendars/date_tokenizer_spec.rb
@@ -288,7 +291,7 @@ files:
 - spec/tokenizers/numbers/number_tokenizer_spec.rb
 - spec/tokenizers/token_spec.rb
 - spec/twitter_cldr_spec.rb
-- spec/utils/code_point_spec.rb
+- spec/utils/code_points_spec.rb
 - spec/utils/interpolation_spec.rb
 - spec/utils_spec.rb
 - resources/locales/ar/calendars.yml
@@ -453,6 +456,7 @@ files:
 - resources/unicode_data/bengali.yml
 - resources/unicode_data/block_elements.yml
 - resources/unicode_data/blocks.yml
+- resources/unicode_data/blocks_hangul.yml
 - resources/unicode_data/bopomofo.yml
 - resources/unicode_data/bopomofo_extended.yml
 - resources/unicode_data/box_drawing.yml
@@ -482,6 +486,7 @@ files:
 - resources/unicode_data/combining_diacritical_marks_supplement.yml
 - resources/unicode_data/combining_half_marks.yml
 - resources/unicode_data/common_indic_number_forms.yml
+- resources/unicode_data/composition_exclusions.yml
 - resources/unicode_data/control_pictures.yml
 - resources/unicode_data/coptic.yml
 - resources/unicode_data/counting_rod_numerals.yml
@@ -493,6 +498,7 @@ files:
 - resources/unicode_data/cyrillic_extended_a.yml
 - resources/unicode_data/cyrillic_extended_b.yml
 - resources/unicode_data/cyrillic_supplement.yml
+- resources/unicode_data/decomposition_map.yml
 - resources/unicode_data/deseret.yml
 - resources/unicode_data/devanagari.yml
 - resources/unicode_data/devanagari_extended.yml