latinizer 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 79227b3ecff306b41fc210a377ddb1811a8cdfc2823390bf7ea3a59b6a91c7fc
-  data.tar.gz: 32a70e923290a31c5b8fdc1eeda516c2390333445adcacfac712e8fcd5f2e9ec
+  metadata.gz: 21b15ac5fbb50b85ac397a6b1e58bb4e7e63c5ce9800d7c21eddd02a5fa09d7d
+  data.tar.gz: 36e10b84402711854a0345eaf5ccf488a125ce197690daabdff6443526d20522
 SHA512:
-  metadata.gz: b513f27ef4716f3827aad163ea9ba7c260cf2b64ed20ff455624fef52c174f10f5e763f85b6f62ccf31adf64382aedd1d56d9e0443ac9cf411bb78620411ab53
-  data.tar.gz: 0efcd64fa9ce46b45efdbdcdbd3150e478add0d42566279461f90f145068f201efed0ad4adb43fadddc3cf1327a0b0e8c2d9eea8a35ea2941e8791b27699a8a0
+  metadata.gz: 4c1b6a8eddc5258615cf0017b8209e6ee15ecb4332e0f241457e7a403364597e4ee525a89bcec3a93eb2e89821498ac65cbae893183b54cffffa7e88d3431672
+  data.tar.gz: 3a56c28ccbb182a25ba8e65998d0c95038f613ad4cbdd42d81103e47dc26bcacd0f7eee80549650736d9bb5657e9dd1de9bd824709a1ab9602ca9c77ff9417f1

data/lib/latinizer.rb

@@ -1,293 +1,70 @@
 class Latinizer
-  require 'chinese_pinyin'
-  require 'mecab_standalone'
-  require 'romaji'
   require 'translit'
   require 'unicode/scripts'
   require 'babosa'
+  require_relative './lib/arabic.rb'
+  require_relative './lib/han.rb'
+  require_relative './lib/japanese.rb'
+
+  SUPPORTED_SCRIPTS = [
+    'Arabic',
+    'Cyrillic',
+    'Han',
+    'Japanese'
+  ]
 
   def self.t(text, opt = nil)
-    scripts = Unicode::Scripts.scripts(text) - ['Common', 'Inherited', 'Latin']
-    pinyin_options = {tonemarks: true}
+    scripts = detect_non_latin_scripts(text)
 
-    if opt == :ascii
-      pinyin_options = {}
-    elsif opt == :ja
-      return romanize_japanese(text)
+    if scripts.size == 0
+      return opt == :ascii ? remove_non_ascii(text) : text
+    elsif scripts.size > 1
+      latinized = latinize_script(text, scripts.first, opt)
+      return t(latinized, opt)
     end
 
-    if scripts.size == 1
-      case scripts.first
-      when 'Arabic'
-        return romanize_arabic(text)
-      when 'Cyrillic'
-        latinized = Translit.convert(text, :english)
-        return opt == :ascii ? latinized.to_slug.to_ascii.to_s : latinized
-      when 'Han'
-        return Pinyin.t(text, pinyin_options)
-      end
-    end
+    latinized = latinize_script(text, scripts.first, opt)
+    opt == :ascii ? remove_non_ascii(latinized) : latinized
+  end
 
-    if is_japanese?(scripts)
-      return romanize_japanese(text)
+  def self.latinize_script(text, script, opt = nil)
+    case script
+    when 'Arabic'
+      return Arabic.t(text)
+    when 'Cyrillic'
+      latinized = Translit.convert(text, :english)
+      return opt == :ascii ? remove_diacritics(latinized) : latinized
+    when 'Han'
+      return  Han.t(text, opt)
+    when 'Japanese'
+      return Japanese.t(text)
     end
-
     text
   end
 
-  def self.has_non_latin?(text)
+  def self.detect_non_latin_scripts(text)
     scripts = Unicode::Scripts.scripts(text) - ['Common', 'Inherited', 'Latin']
-    scripts.size > 0 ? true : false
+    if is_japanese?(scripts)
+      scripts -= ['Han', 'Hiragana', 'Katakana']
+      scripts += ['Japanese']
+    end
+    scripts.intersection(SUPPORTED_SCRIPTS)
   end
 
-  def self.is_japanese?(scripts) #fix only kana text
-    (scripts.include?('Han') && (scripts.include?('Hiragana') || scripts.include?('Katakana'))) ||
-    (scripts.include?('Hiragana') || scripts.include?('Katakana'))
+  def self.remove_diacritics(text)
+    text.to_slug.transliterate.to_s
   end
 
-  def self.romanize_japanese(text)
-    Romaji.kana2romaji(parse_japanese(text)
-      .map{|k| k[-1]}
-      .join(' ')
-      .gsub('ー','')
-      .gsub(' 。','.')
-      .gsub(' ・','-')
-      .gsub(' 、',',')
-    )
+  def self.remove_non_ascii(text)
+    text.to_slug.transliterate.to_ascii.to_s
   end
 
-  def self.parse_japanese(text)
-    mecab_parsed = MecabStandalone.parse(text)
-      .split("\n")
-      .map{|k| k.split("\t")}.tap(&:pop)
-      .map{|k| [k[0]].concat(k[1].split(','))}
-      .map{|k| [k[0], k[1], k[-2]]}
-    tokenized_kana = []
-    mecab_parsed.each do |token|
-      if token[1] == "助動詞"
-        tokenized_kana[-1][0] +=  token[0]
-        tokenized_kana[-1][-1] += token[-1]
-      elsif token[-1] == '*'
-        tokenized_kana << [token[0], token[1], token[0]]
-      else
-        tokenized_kana << token
-      end
-    end
-    tokenized_kana
+  def self.has_non_latin?(text)
+    scripts = Unicode::Scripts.scripts(text) - ['Common', 'Inherited', 'Latin']
+    scripts.size > 0 ? true : false
   end
 
-  def self.romanize_arabic(text)
-    text
-    .gsub('،',',') # ARABIC COMMA
-    .gsub('؛',';') # ARABIC SEMICOLON
-    .gsub('؟','?') # ARABIC QUESTION MARK
-    .gsub('ء',"'") # ARABIC LETTER HAMZA
-    .gsub('آ','a') # ARABIC LETTER ALEF WITH MADDA ABOVE
-    .gsub('أ','a') # ARABIC LETTER ALEF WITH HAMZA ABOVE
-    .gsub('ؤ','w') # ARABIC LETTER WAW WITH HAMZA ABOVE
-    .gsub('إ','i') # ARABIC LETTER ALEF WITH HAMZA BELOW
-    .gsub('ئ','ye') # ARABIC LETTER YEH WITH HAMZA ABOVE
-    .gsub('ا','a') # ARABIC LETTER ALEF
-    .gsub('ب','b') # ARABIC LETTER BEH
-    .gsub('ة','a') # ARABIC LETTER TEH MARBUTA
-    .gsub('ت','t') # ARABIC LETTER TEH
-    .gsub('ث','th') # ARABIC LETTER THEH
-    .gsub('ج','j') # ARABIC LETTER JEEM
-    .gsub('ح','h') # ARABIC LETTER HAH
-    .gsub('خ','kh') # ARABIC LETTER KHAH
-    .gsub('د','d') # ARABIC LETTER DAL
-    .gsub('ذ','th') # ARABIC LETTER THAL
-    .gsub('ر','r') # ARABIC LETTER REH
-    .gsub('ز','z') # ARABIC LETTER ZAIN
-    .gsub('س','s') # ARABIC LETTER SEEN
-    .gsub('ش','sh') # ARABIC LETTER SHEEN
-    .gsub('ص','s') # ARABIC LETTER SAD
-    .gsub('ض','d') # ARABIC LETTER DAD
-    .gsub('ط','t') # ARABIC LETTER TAH
-    .gsub('ظ','z') # ARABIC LETTER ZAH
-    .gsub('ع',"'") # ARABIC LETTER AIN
-    .gsub('غ','gh') # ARABIC LETTER GHAIN
-    .gsub('ـ','-') # ARABIC TATWEEL
-    .gsub('ف','f') # ARABIC LETTER FEH
-    .gsub('ق','q') # ARABIC LETTER QAF
-    .gsub('ك','k') # ARABIC LETTER KAF
-    .gsub('ل','l') # ARABIC LETTER LAM
-    .gsub('م','m') # ARABIC LETTER MEEM
-    .gsub('ن','n') # ARABIC LETTER NOON
-    .gsub('ه','h') # ARABIC LETTER HEH
-    .gsub('و','w') # ARABIC LETTER WAW
-    .gsub('ى','a') # ARABIC LETTER ALEF MAKSURA
-    .gsub('ي','y') # ARABIC LETTER YEH
-    .gsub('َ','a') # ARABIC FATHA
-    .gsub('ُ','u') # ARABIC DAMMA
-    .gsub('ِ','i') # ARABIC KASRA
-    .gsub('ْ','') # ARABIC SUKUN
-    .gsub('ٔ',"'") # ARABIC HAMZA ABOVE
-    .gsub('ٕ',"'") # ARABIC HAMZA BELOW
-    .gsub('٠','0') # ARABIC-INDIC DIGIT ZERO
-    .gsub('١','1') # ARABIC-INDIC DIGIT ONE
-    .gsub('٢','2') # ARABIC-INDIC DIGIT TWO
-    .gsub('٣','3') # ARABIC-INDIC DIGIT THREE
-    .gsub('٤','4') # ARABIC-INDIC DIGIT FOUR
-    .gsub('٥','5') # ARABIC-INDIC DIGIT FIVE
-    .gsub('٦','6') # ARABIC-INDIC DIGIT SIX
-    .gsub('٧','7') # ARABIC-INDIC DIGIT SEVEN
-    .gsub('٨','8') # ARABIC-INDIC DIGIT EIGHT
-    .gsub('٩','9') # ARABIC-INDIC DIGIT NINE
-    .gsub('٪','%') # ARABIC PERCENT SIGN
-    .gsub('٫',',') # ARABIC DECIMAL SEPARATOR
-    .gsub('٬',',') # ARABIC THOUSANDS SEPARATOR
-    .gsub('ٮ','b') # ARABIC LETTER DOTLESS BEH
-    .gsub('ٯ','q') # ARABIC LETTER DOTLESS QAF
-    .gsub('ٰ','a') # ARABIC LETTER SUPERSCRIPT ALEF
-    .gsub('ٱ','a') # ARABIC LETTER ALEF WASLA
-    .gsub('ٲ','a') # ARABIC LETTER ALEF WITH WAVY HAMZA ABOVE
-    .gsub('ٳ','a') # ARABIC LETTER ALEF WITH WAVY HAMZA BELOW
-    .gsub('ٷ','u') # ARABIC LETTER U WITH HAMZA ABOVE
-    .gsub('ٹ','tt') # ARABIC LETTER TTEH
-    .gsub('ٺ','tt') # ARABIC LETTER TTEHEH
-    .gsub('ٻ','b') # ARABIC LETTER BEEH
-    .gsub('ټ','t') # ARABIC LETTER TEH WITH RING
-    .gsub('ٽ','t') # ARABIC LETTER TEH WITH THREE DOTS ABOVE DOWNWARDS
-    .gsub('پ','p') # ARABIC LETTER PEH
-    .gsub('ٿ','t') # ARABIC LETTER TEHEH
-    .gsub('ڀ','b') # ARABIC LETTER BEHEH
-    .gsub('ځ','h') # ARABIC LETTER HAH WITH HAMZA ABOVE
-    .gsub('ڂ','h') # ARABIC LETTER HAH WITH TWO DOTS VERTICAL ABOVE
-    .gsub('ڃ','ny') # ARABIC LETTER NYEH
-    .gsub('ڄ','dy') # ARABIC LETTER DYEH
-    .gsub('څ','h') # ARABIC LETTER HAH WITH THREE DOTS ABOVE
-    .gsub('چ','tch') # ARABIC LETTER TCHEH
-    .gsub('ڇ','tch') # ARABIC LETTER TCHEHEH
-    .gsub('ڈ','dd') # ARABIC LETTER DDAL
-    .gsub('ډ','d') # ARABIC LETTER DAL WITH RING
-    .gsub('ڊ','d') # ARABIC LETTER DAL WITH DOT BELOW
-    .gsub('ڋ','d') # ARABIC LETTER DAL WITH DOT BELOW AND SMALL TAH
-    .gsub('ڌ','d') # ARABIC LETTER DAHAL
-    .gsub('ڍ','dd') # ARABIC LETTER DDAHAL
-    .gsub('ڎ','d') # ARABIC LETTER DUL
-    .gsub('ڏ','d') # ARABIC LETTER DAL WITH THREE DOTS ABOVE DOWNWARDS
-    .gsub('ڐ','d') # ARABIC LETTER DAL WITH FOUR DOTS ABOVE
-    .gsub('ڑ','rr') # ARABIC LETTER RREH
-    .gsub('ڒ','r') # ARABIC LETTER REH WITH SMALL V
-    .gsub('ړ','r') # ARABIC LETTER REH WITH RING
-    .gsub('ڔ','r') # ARABIC LETTER REH WITH DOT BELOW
-    .gsub('ڕ','r') # ARABIC LETTER REH WITH SMALL V BELOW
-    .gsub('ږ','r') # ARABIC LETTER REH WITH DOT BELOW AND DOT ABOVE
-    .gsub('ڗ','r') # ARABIC LETTER REH WITH TWO DOTS ABOVE
-    .gsub('ژ','j') # ARABIC LETTER JEH
-    .gsub('ڙ','r') # ARABIC LETTER REH WITH FOUR DOTS ABOVE
-    .gsub('ښ','s') # ARABIC LETTER SEEN WITH DOT BELOW AND DOT ABOVE
-    .gsub('ڛ','s') # ARABIC LETTER SEEN WITH THREE DOTS BELOW
-    .gsub('ڜ','s') # ARABIC LETTER SEEN WITH THREE DOTS BELOW AND THREE DOTS ABOVE
-    .gsub('ڝ','s') # ARABIC LETTER SAD WITH TWO DOTS BELOW
-    .gsub('ڞ','s') # ARABIC LETTER SAD WITH THREE DOTS ABOVE
-    .gsub('ڟ','t') # ARABIC LETTER TAH WITH THREE DOTS ABOVE
-    .gsub('ڠ','n') # ARABIC LETTER AIN WITH THREE DOTS ABOVE
-    .gsub('ڡ','f') # ARABIC LETTER DOTLESS FEH
-    .gsub('ڢ','f') # ARABIC LETTER FEH WITH DOT MOVED BELOW
-    .gsub('ڣ','f') # ARABIC LETTER FEH WITH DOT BELOW
-    .gsub('ڤ','v') # ARABIC LETTER VEH
-    .gsub('ڥ','f') # ARABIC LETTER FEH WITH THREE DOTS BELOW
-    .gsub('ڦ','p') # ARABIC LETTER PEHEH
-    .gsub('ڧ','q') # ARABIC LETTER QAF WITH DOT ABOVE
-    .gsub('ڨ','q') # ARABIC LETTER QAF WITH THREE DOTS ABOVE
-    .gsub('ک','k') # ARABIC LETTER KEHEH
-    .gsub('ڪ','k') # ARABIC LETTER SWASH KAF
-    .gsub('ګ','k') # ARABIC LETTER KAF WITH RING
-    .gsub('ڬ','k') # ARABIC LETTER KAF WITH DOT ABOVE
-    .gsub('ڭ','ng') # ARABIC LETTER NG
-    .gsub('ڮ','k') # ARABIC LETTER KAF WITH THREE DOTS BELOW
-    .gsub('گ','g') # ARABIC LETTER GAF
-    .gsub('ڰ','g') # ARABIC LETTER GAF WITH RING
-    .gsub('ڱ','ng') # ARABIC LETTER NGOEH
-    .gsub('ڲ','g') # ARABIC LETTER GAF WITH TWO DOTS BELOW
-    .gsub('ڳ','g') # ARABIC LETTER GUEH
-    .gsub('ڴ','g') # ARABIC LETTER GAF WITH THREE DOTS ABOVE
-    .gsub('ڵ','l') # ARABIC LETTER LAM WITH SMALL V
-    .gsub('ڶ','l') # ARABIC LETTER LAM WITH DOT ABOVE
-    .gsub('ڷ','l') # ARABIC LETTER LAM WITH THREE DOTS ABOVE
-    .gsub('ڸ','l') # ARABIC LETTER LAM WITH THREE DOTS BELOW
-    .gsub('ڹ','n') # ARABIC LETTER NOON WITH DOT BELOW
-    .gsub('ں','n') # ARABIC LETTER NOON GHUNNA
-    .gsub('ڻ','rn') # ARABIC LETTER RNOON
-    .gsub('ڼ','n') # ARABIC LETTER NOON WITH RING
-    .gsub('ڽ','n') # ARABIC LETTER NOON WITH THREE DOTS ABOVE
-    .gsub('ھ','h') # ARABIC LETTER HEH DOACHASHMEE
-    .gsub('ڿ','tch') # ARABIC LETTER TCHEH WITH DOT ABOVE
-    .gsub('ۀ','h') # ARABIC LETTER HEH WITH YEH ABOVE
-    .gsub('ہ','h') # ARABIC LETTER HEH GOAL
-    .gsub('ۂ','h') # ARABIC LETTER HEH GOAL WITH HAMZA ABOVE
-    .gsub('ۃ','a') # ARABIC LETTER TEH MARBUTA GOAL
-    .gsub('ۄ','w') # ARABIC LETTER WAW WITH RING
-    .gsub('ۅ','oe') # ARABIC LETTER KIRGHIZ OE
-    .gsub('ۆ','oe') # ARABIC LETTER OE
-    .gsub('ۇ','u') # ARABIC LETTER U
-    .gsub('ۈ','yu') # ARABIC LETTER YU
-    .gsub('ۉ','yu') # ARABIC LETTER KIRGHIZ YU
-    .gsub('ۊ','w') # ARABIC LETTER WAW WITH TWO DOTS ABOVE
-    .gsub('ۋ','v') # ARABIC LETTER VE
-    .gsub('ی','y') # ARABIC LETTER FARSI YEH
-    .gsub('ۍ','y') # ARABIC LETTER YEH WITH TAIL
-    .gsub('ێ','y') # ARABIC LETTER YEH WITH SMALL V
-    .gsub('ۏ','w') # ARABIC LETTER WAW WITH DOT ABOVE
-    .gsub('ې','e') # ARABIC LETTER E
-    .gsub('ۑ','y') # ARABIC LETTER YEH WITH THREE DOTS BELOW
-    .gsub('ے','y') # ARABIC LETTER YEH BARREE
-    .gsub('ۓ','y') # ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
-    .gsub('۔','.') # ARABIC FULL STOP
-    .gsub('ە','ae') # ARABIC LETTER AE
-    .gsub('ۮ','d') # ARABIC LETTER DAL WITH INVERTED V
-    .gsub('ۯ','r') # ARABIC LETTER REH WITH INVERTED V
-    .gsub('۰','0') # EXTENDED ARABIC-INDIC DIGIT ZERO
-    .gsub('۱','1') # EXTENDED ARABIC-INDIC DIGIT ONE
-    .gsub('۲','2') # EXTENDED ARABIC-INDIC DIGIT TWO
-    .gsub('۳','3') # EXTENDED ARABIC-INDIC DIGIT THREE
-    .gsub('۴','4') # EXTENDED ARABIC-INDIC DIGIT FOUR
-    .gsub('۵','5') # EXTENDED ARABIC-INDIC DIGIT FIVE
-    .gsub('۶','6') # EXTENDED ARABIC-INDIC DIGIT SIX
-    .gsub('۷','7') # EXTENDED ARABIC-INDIC DIGIT SEVEN
-    .gsub('۸','8') # EXTENDED ARABIC-INDIC DIGIT EIGHT
-    .gsub('۹','9') # EXTENDED ARABIC-INDIC DIGIT NINE
-    .gsub('ۺ','sh') # ARABIC LETTER SHEEN WITH DOT BELOW
-    .gsub('ۻ','d') # ARABIC LETTER DAD WITH DOT BELOW
-    .gsub('ۼ','gh') # ARABIC LETTER GHAIN WITH DOT BELOW
-    .gsub('۽','&') # ARABIC SIGN SINDHI AMPERSAND
-    .gsub('ﷲ','Allah') # ARABIC LIGATURE ALLAH ISOLATED FORM
-    .gsub('و','w') # Arabic letter waw
-    .gsub('ء',"'") # hamza
-    .gsub('ٔ',"'") # hamza above
-    .gsub('ٕ',"'") # hamza below
-    .gsub('ع',"'") # ain
-    .gsub('آ','a') # alef madda
-    .gsub('إ','i') # alef with hamza below
-    .gsub('ٱ','a') # alef wasla
-    .gsub('ة','a') # teh marbuta
-    .gsub('ۃ','a') # teh marbuta goal
-    .gsub('ي','y') # Arabic yeh
-    .gsub('ى','a') # alef maksura
-    .gsub('ﻯ','a') # alef maksura isolated form
-    .gsub('ﻰ','a') # alef maksura final form
-    .gsub('ﯨ','a') # Uighur Kazach Kirghiz alef maksura initial form
-    .gsub('ﯩ','a') # Uighur Kazach Kirghiz alef maksura medial form
-    .gsub('ٰ','a ') # Arabic letter superscript alef
-    .gsub('ـ','') # tatweel (filler)
-    .gsub('َ','a') # fatha ("-a")
-    .gsub('ُ','u') # damma ("-u")
-    .gsub('ِ','i') # kasra ("-i")
-    .gsub('ْ','') # sukun (no vowel)
-    .gsub('ۡ','') # comment small high dotless head of khah; like sukun (no vowel); used in Kashmiri, Assamese
-    .gsub('اً','an') # alef + fathatan
-    .gsub('ً','') # fathatan ("-an")
-    .gsub('ٌ','') # dammatan ("-un")
-    .gsub('ٍ','') # kasratan ("-in")
-    .gsub('ّ','') # shadda (consonant doubler)
-    .gsub('ڃ','ny') # Arabic letter nyeh U+0683 (used in Sindhi (snd))
-    .gsub('ڄ','dy') # Arabic letter dyeh U+0684 (used in Sindhi (snd))
-    .gsub('۾','men') # Sindhi postposition men
-    .gsub('ؑ','alayhe wasallam') # "upon him be peace"
-    .gsub('ﷴ','Mohammad') # "Mohammad"
-    .gsub('ﷸ','wasallam') # "and peace"
-    .gsub('ﷺ','sallallahou alayhe wasallam') # "prayer of God be upon him and his family and peace"
+  def self.is_japanese?(scripts)
+    scripts.include?('Hiragana') || scripts.include?('Katakana')
   end
 end

data/lib/lib/arabic.rb

@@ -0,0 +1,199 @@
+class Arabic
+
+  ARABIC = {
+    '،' => ',',   # ARABIC COMMA
+    '؛' => ';',   # ARABIC SEMICOLON
+    '؟' => '?',   # ARABIC QUESTION MARK
+    'ء' => "'",   # ARABIC LETTER HAMZA
+    'آ' => 'a',   # ARABIC LETTER ALEF WITH MADDA ABOVE
+    'أ' => 'a',   # ARABIC LETTER ALEF WITH HAMZA ABOVE
+    'ؤ' => 'w',   # ARABIC LETTER WAW WITH HAMZA ABOVE
+    'إ' => 'i',   # ARABIC LETTER ALEF WITH HAMZA BELOW
+    'ئ' => 'ye',  # ARABIC LETTER YEH WITH HAMZA ABOVE
+    'ا' => 'a',   # ARABIC LETTER ALEF
+    'ب' => 'b',   # ARABIC LETTER BEH
+    'ة' => 'a',   # ARABIC LETTER TEH MARBUTA
+    'ت' => 't',   # ARABIC LETTER TEH
+    'ث' => 'th',  # ARABIC LETTER THEH
+    'ج' => 'j',   # ARABIC LETTER JEEM
+    'ح' => 'h',   # ARABIC LETTER HAH
+    'خ' => 'kh',  # ARABIC LETTER KHAH
+    'د' => 'd',   # ARABIC LETTER DAL
+    'ذ' => 'th',  # ARABIC LETTER THAL
+    'ر' => 'r',   # ARABIC LETTER REH
+    'ز' => 'z',   # ARABIC LETTER ZAIN
+    'س' => 's',   # ARABIC LETTER SEEN
+    'ش' => 'sh',  # ARABIC LETTER SHEEN
+    'ص' => 's',   # ARABIC LETTER SAD
+    'ض' => 'd',   # ARABIC LETTER DAD
+    'ط' => 't',   # ARABIC LETTER TAH
+    'ظ' => 'z',   # ARABIC LETTER ZAH
+    'ع' => "'",   # ARABIC LETTER AIN
+    'غ' => 'gh',  # ARABIC LETTER GHAIN
+    'ـ' => '-',   # ARABIC TATWEEL
+    'ف' => 'f',   # ARABIC LETTER FEH
+    'ق' => 'q',   # ARABIC LETTER QAF
+    'ك' => 'k',   # ARABIC LETTER KAF
+    'ل' => 'l',   # ARABIC LETTER LAM
+    'م' => 'm',   # ARABIC LETTER MEEM
+    'ن' => 'n',   # ARABIC LETTER NOON
+    'ه' => 'h',   # ARABIC LETTER HEH
+    'و' => 'w',   # ARABIC LETTER WAW
+    'ى' => 'a',   # ARABIC LETTER ALEF MAKSURA
+    'ي' => 'y',   # ARABIC LETTER YEH
+    'َ' => 'a',    # ARABIC FATHA
+    'ُ' => 'u',    # ARABIC DAMMA
+    'ِ' => 'i',    # ARABIC KASRA
+    'ْ' => '',     # ARABIC SUKUN
+    'ٔ' => "'",    # ARABIC HAMZA ABOVE
+    'ٕ' => "'",    # ARABIC HAMZA BELOW
+    '٠' => '0',   # ARABIC-INDIC DIGIT ZERO
+    '١' => '1',   # ARABIC-INDIC DIGIT ONE
+    '٢' => '2',   # ARABIC-INDIC DIGIT TWO
+    '٣' => '3',   # ARABIC-INDIC DIGIT THREE
+    '٤' => '4',   # ARABIC-INDIC DIGIT FOUR
+    '٥' => '5',   # ARABIC-INDIC DIGIT FIVE
+    '٦' => '6',   # ARABIC-INDIC DIGIT SIX
+    '٧' => '7',   # ARABIC-INDIC DIGIT SEVEN
+    '٨' => '8',   # ARABIC-INDIC DIGIT EIGHT
+    '٩' => '9',   # ARABIC-INDIC DIGIT NINE
+    '٪' => '%',   # ARABIC PERCENT SIGN
+    '٫' => ' => ',  # ARABIC DECIMAL SEPARATOR
+    '٬' => ' => ',  # ARABIC THOUSANDS SEPARATOR
+    'ٮ' => 'b',   # ARABIC LETTER DOTLESS BEH
+    'ٯ' => 'q',   # ARABIC LETTER DOTLESS QAF
+    'ٰ' => 'a',   # ARABIC LETTER SUPERSCRIPT ALEF
+    'ٱ' => 'a',   # ARABIC LETTER ALEF WASLA
+    'ٲ' => 'a',   # ARABIC LETTER ALEF WITH WAVY HAMZA ABOVE
+    'ٳ' => 'a',   # ARABIC LETTER ALEF WITH WAVY HAMZA BELOW
+    'ٷ' => 'u',   # ARABIC LETTER U WITH HAMZA ABOVE
+    'ٹ' => 'tt',  # ARABIC LETTER TTEH
+    'ٺ' => 'tt',  # ARABIC LETTER TTEHEH
+    'ٻ' => 'b',   # ARABIC LETTER BEEH
+    'ټ' => 't',   # ARABIC LETTER TEH WITH RING
+    'ٽ' => 't',   # ARABIC LETTER TEH WITH THREE DOTS ABOVE DOWNWARDS
+    'پ' => 'p',   # ARABIC LETTER PEH
+    'ٿ' => 't',   # ARABIC LETTER TEHEH
+    'ڀ' => 'b',   # ARABIC LETTER BEHEH
+    'ځ' => 'h',   # ARABIC LETTER HAH WITH HAMZA ABOVE
+    'ڂ' => 'h',   # ARABIC LETTER HAH WITH TWO DOTS VERTICAL ABOVE
+    'ڃ' => 'ny',  # ARABIC LETTER NYEH
+    'ڄ' => 'dy',  # ARABIC LETTER DYEH
+    'څ' => 'h',   # ARABIC LETTER HAH WITH THREE DOTS ABOVE
+    'چ' => 'tch',   # ARABIC LETTER TCHEH
+    'ڇ' => 'tch',   # ARABIC LETTER TCHEHEH
+    'ڈ' => 'dd',  # ARABIC LETTER DDAL
+    'ډ' => 'd',   # ARABIC LETTER DAL WITH RING
+    'ڊ' => 'd',   # ARABIC LETTER DAL WITH DOT BELOW
+    'ڋ' => 'd',   # ARABIC LETTER DAL WITH DOT BELOW AND SMALL TAH
+    'ڌ' => 'd',   # ARABIC LETTER DAHAL
+    'ڍ' => 'dd',  # ARABIC LETTER DDAHAL
+    'ڎ' => 'd',   # ARABIC LETTER DUL
+    'ڏ' => 'd',   # ARABIC LETTER DAL WITH THREE DOTS ABOVE DOWNWARDS
+    'ڐ' => 'd',   # ARABIC LETTER DAL WITH FOUR DOTS ABOVE
+    'ڑ' => 'rr',  # ARABIC LETTER RREH
+    'ڒ' => 'r',   # ARABIC LETTER REH WITH SMALL V
+    'ړ' => 'r',   # ARABIC LETTER REH WITH RING
+    'ڔ' => 'r',   # ARABIC LETTER REH WITH DOT BELOW
+    'ڕ' => 'r',   # ARABIC LETTER REH WITH SMALL V BELOW
+    'ږ' => 'r',   # ARABIC LETTER REH WITH DOT BELOW AND DOT ABOVE
+    'ڗ' => 'r',   # ARABIC LETTER REH WITH TWO DOTS ABOVE
+    'ژ' => 'j',   # ARABIC LETTER JEH
+    'ڙ' => 'r',   # ARABIC LETTER REH WITH FOUR DOTS ABOVE
+    'ښ' => 's',   # ARABIC LETTER SEEN WITH DOT BELOW AND DOT ABOVE
+    'ڛ' => 's',   # ARABIC LETTER SEEN WITH THREE DOTS BELOW
+    'ڜ' => 's',   # ARABIC LETTER SEEN WITH THREE DOTS BELOW AND THREE DOTS ABOVE
+    'ڝ' => 's',   # ARABIC LETTER SAD WITH TWO DOTS BELOW
+    'ڞ' => 's',   # ARABIC LETTER SAD WITH THREE DOTS ABOVE
+    'ڟ' => 't',   # ARABIC LETTER TAH WITH THREE DOTS ABOVE
+    'ڠ' => 'n',   # ARABIC LETTER AIN WITH THREE DOTS ABOVE
+    'ڡ' => 'f',   # ARABIC LETTER DOTLESS FEH
+    'ڢ' => 'f',   # ARABIC LETTER FEH WITH DOT MOVED BELOW
+    'ڣ' => 'f',   # ARABIC LETTER FEH WITH DOT BELOW
+    'ڤ' => 'v',   # ARABIC LETTER VEH
+    'ڥ' => 'f',   # ARABIC LETTER FEH WITH THREE DOTS BELOW
+    'ڦ' => 'p',   # ARABIC LETTER PEHEH
+    'ڧ' => 'q',   # ARABIC LETTER QAF WITH DOT ABOVE
+    'ڨ' => 'q',   # ARABIC LETTER QAF WITH THREE DOTS ABOVE
+    'ک' => 'k',   # ARABIC LETTER KEHEH
+    'ڪ' => 'k',   # ARABIC LETTER SWASH KAF
+    'ګ' => 'k',   # ARABIC LETTER KAF WITH RING
+    'ڬ' => 'k',   # ARABIC LETTER KAF WITH DOT ABOVE
+    'ڭ' => 'ng',  # ARABIC LETTER NG
+    'ڮ' => 'k',   # ARABIC LETTER KAF WITH THREE DOTS BELOW
+    'گ' => 'g',   # ARABIC LETTER GAF
+    'ڰ' => 'g',   # ARABIC LETTER GAF WITH RING
+    'ڱ' => 'ng',  # ARABIC LETTER NGOEH
+    'ڲ' => 'g',   # ARABIC LETTER GAF WITH TWO DOTS BELOW
+    'ڳ' => 'g',   # ARABIC LETTER GUEH
+    'ڴ' => 'g',   # ARABIC LETTER GAF WITH THREE DOTS ABOVE
+    'ڵ' => 'l',   # ARABIC LETTER LAM WITH SMALL V
+    'ڶ' => 'l',   # ARABIC LETTER LAM WITH DOT ABOVE
+    'ڷ' => 'l',   # ARABIC LETTER LAM WITH THREE DOTS ABOVE
+    'ڸ' => 'l',   # ARABIC LETTER LAM WITH THREE DOTS BELOW
+    'ڹ' => 'n',   # ARABIC LETTER NOON WITH DOT BELOW
+    'ں' => 'n',   # ARABIC LETTER NOON GHUNNA
+    'ڻ' => 'rn',  # ARABIC LETTER RNOON
+    'ڼ' => 'n',   # ARABIC LETTER NOON WITH RING
+    'ڽ' => 'n',   # ARABIC LETTER NOON WITH THREE DOTS ABOVE
+    'ھ' => 'h',   # ARABIC LETTER HEH DOACHASHMEE
+    'ڿ' => 'tch', # ARABIC LETTER TCHEH WITH DOT ABOVE
+    'ۀ' => 'h',   # ARABIC LETTER HEH WITH YEH ABOVE
+    'ہ' => 'h',   # ARABIC LETTER HEH GOAL
+    'ۂ' => 'h',   # ARABIC LETTER HEH GOAL WITH HAMZA ABOVE
+    'ۃ' => 'a',   # ARABIC LETTER TEH MARBUTA GOAL
+    'ۄ' => 'w',   # ARABIC LETTER WAW WITH RING
+    'ۅ' => 'oe',  # ARABIC LETTER KIRGHIZ OE
+    'ۆ' => 'oe',  # ARABIC LETTER OE
+    'ۇ' => 'u',   # ARABIC LETTER U
+    'ۈ' => 'yu',  # ARABIC LETTER YU
+    'ۉ' => 'yu',  # ARABIC LETTER KIRGHIZ YU
+    'ۊ' => 'w',   # ARABIC LETTER WAW WITH TWO DOTS ABOVE
+    'ۋ' => 'v',   # ARABIC LETTER VE
+    'ی' => 'y',   # ARABIC LETTER FARSI YEH
+    'ۍ' => 'y',   # ARABIC LETTER YEH WITH TAIL
+    'ێ' => 'y',   # ARABIC LETTER YEH WITH SMALL V
+    'ۏ' => 'w',   # ARABIC LETTER WAW WITH DOT ABOVE
+    'ې' => 'e',   # ARABIC LETTER E
+    'ۑ' => 'y',   # ARABIC LETTER YEH WITH THREE DOTS BELOW
+    'ے' => 'y',   # ARABIC LETTER YEH BARREE
+    'ۓ' => 'y',   # ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
+    '۔' => '.',   # ARABIC FULL STOP
+    'ە' => 'ae',  # ARABIC LETTER AE
+    'ۮ' => 'd',   # ARABIC LETTER DAL WITH INVERTED V
+    'ۯ' => 'r',   # ARABIC LETTER REH WITH INVERTED V
+    '۰' => '0',   # EXTENDED ARABIC-INDIC DIGIT ZERO
+    '۱' => '1',   # EXTENDED ARABIC-INDIC DIGIT ONE
+    '۲' => '2',   # EXTENDED ARABIC-INDIC DIGIT TWO
+    '۳' => '3',   # EXTENDED ARABIC-INDIC DIGIT THREE
+    '۴' => '4',   # EXTENDED ARABIC-INDIC DIGIT FOUR
+    '۵' => '5',   # EXTENDED ARABIC-INDIC DIGIT FIVE
+    '۶' => '6',   # EXTENDED ARABIC-INDIC DIGIT SIX
+    '۷' => '7',   # EXTENDED ARABIC-INDIC DIGIT SEVEN
+    '۸' => '8',   # EXTENDED ARABIC-INDIC DIGIT EIGHT
+    '۹' => '9',   # EXTENDED ARABIC-INDIC DIGIT NINE
+    'ۺ' => 'sh',  # ARABIC LETTER SHEEN WITH DOT BELOW
+    'ۻ' => 'd',   # ARABIC LETTER DAD WITH DOT BELOW
+    'ۼ' => 'gh',  # ARABIC LETTER GHAIN WITH DOT BELOW
+    '۽' => '&',   # ARABIC SIGN SINDHI AMPERSAND
+    'ﷲ' => 'Allah',   # ARABIC LIGATURE ALLAH ISOLATED FORM
+    'ۡ' => '',      # comment small high dotless head of khah; like sukun (no vowel); used in Kashmiri =>  Assamese
+    'اً' => 'an',  # alef + fathatan
+    'ً' => '',   # fathatan ("-an")
+    'ٌ' => '',   # dammatan ("-un")
+    'ٍ' => '',   # kasratan ("-in")
+    'ّ' => '',     # shadda (consonant doubler)
+    '۾' => 'men',   # Sindhi postposition men
+    'ؑ' => 'alayhe wasallam',  # "upon him be peace"
+    'ﷴ' => 'Mohammad',      # "Mohammad"
+    'ﷸ' => 'wasallam',    # "and peace"
+    'ﷺ' => 'sallallahou alayhe wasallam',   # "prayer of God be upon him and his family and peace"
+}.freeze
+
+  def self.t(text)
+    latin = text.dup
+    ARABIC.each { |k,v| latin.gsub!(k, v)}
+    latin
+  end
+
+end

data/lib/lib/han.rb

@@ -0,0 +1,18 @@
+class Han
+  require 'chinese_pinyin'
+
+  def self.t(text, opt = nil)
+    latin = []
+    chars = text.split("")
+    chars.each_with_index do |char, index|
+      if char =~ /\p{Han}/
+        converted_char = Pinyin.t(char, opt == :ascii ? {} : {tonemarks: true})
+        latin << ' '
+        latin << converted_char
+      else
+        latin << char
+      end
+    end
+    latin.join('').gsub('  ', ' ')
+  end
+end

data/lib/lib/japanese.rb

@@ -0,0 +1,61 @@
+class Japanese
+  require 'mecab_standalone'
+  require 'romaji'
+
+  JAPANESE_PONCTUATION = {
+    '　' => ' ',
+    '、' => ',',
+    '。' => '.',
+    '：' => ':',
+    '！' => '!',
+    '？' => '?',
+    '〜' => '~',
+    '…' => '...',
+    '‥' => '..',
+    '「 ' => ' \'',
+    '」' => '\'',
+    '『 ' => ' "',
+    '』' => '"',
+    '〝 ' => ' "',
+    '〟' => '"',
+    '（ ' => ' (',
+    '）' => ')',
+    '【 ' => ' [',
+    '】' => ']',
+    '｛ ' => ' {',
+    '｝' => '}',
+}.freeze
+
+  def self.t(text)
+    latin = text.dup
+    parsed = parse(text)
+    parsed.each do |token|
+      if token[-1]=~ /\p{Katakana}/
+        latin.sub!(token[0], ' ' + Romaji.kana2romaji(token[-1]) )
+      end
+    end
+    JAPANESE_PONCTUATION.each { |k,v| latin.gsub!(k, v)}
+    latin
+  end
+
+  def self.parse(text)
+    mecab_parsed = MecabStandalone.parse(text)
+      .split("\n")
+      .map{|k| k.split("\t")}.tap(&:pop)
+      .map{|k| [k[0]].concat(k[1].split(','))}
+      .map{|k| [k[0], k[1], k[-2]]}
+    tokenized_kana = []
+    mecab_parsed.each do |token|
+      if token[1] == "助動詞"
+        tokenized_kana[-1][0] +=  token[0]
+        tokenized_kana[-1][-1] += token[-1]
+      elsif token[-1] == '*'
+        tokenized_kana << [token[0], token[1], token[0]]
+      else
+        tokenized_kana << token
+      end
+    end
+    tokenized_kana
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: latinizer
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - William Yugue
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-09 00:00:00.000000000 Z
+date: 2020-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chinese_pinyin
@@ -108,6 +108,9 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/latinizer.rb
+- lib/lib/arabic.rb
+- lib/lib/han.rb
+- lib/lib/japanese.rb
 homepage: https://github.com/wyugue/latinizer
 licenses:
 - MIT