zhongwen_tools 0.12.4 → 0.15.1

data/lib/zhongwen_tools/number.rb

@@ -0,0 +1,221 @@
+# encoding: utf-8
+require 'zhongwen_tools/regex'
+require 'zhongwen_tools/zhongwen'
+require 'zhongwen_tools/romanization/pinyin'
+require 'zhongwen_tools/number/number_table'
+
+# Number.to_pyn, to_i, to_zhs, etc.
+module ZhongwenTools
+  module Number
+    def self.number?(obj)
+      klass = obj.class
+
+      if klass == String
+        regex = /([\d]|#{ZhongwenTools::Regex.zh_numbers}){1,}/
+          "#{obj}".gsub(regex, '') == ''
+      elsif klass == Integer
+        true
+      elsif klass == Fixnum
+        true
+      end
+    end
+
+    #needs to be a class method
+    %w(i zhs zht pyn).each do |action|
+      define_singleton_method("to_#{ action }") do |*args|
+        obj, from, separator = args
+        from ||= number_type(obj)
+
+        convert(obj, action.to_sym, from.to_sym, separator)
+      end
+    end
+
+    def self.to_zh(obj, type = :zhs, from = nil)
+      type = type.to_sym
+
+      if type == :zht
+        to_zht(obj, from)
+      else
+        to_zhs(obj, from)
+      end
+    end
+
+    private
+
+    def self.convert(obj, to, from, separator = '')
+      fail ArgumentError unless [:zhs, :zht, :i, :pyn].include?(to.to_sym)
+      fail ArgumentError unless [String, Integer, Fixnum].include?(obj.class)
+
+      number =   convert_from from, to, obj
+
+      if to == :i
+        combine_integers(number)
+      elsif to == :pyn
+        regex = /#{ %w(yi4 wan4 qian1 bai2 shi2).map{ |x| 'ling2\-' + x }.join('|')}/
+        finalize_number(number, '-').gsub(regex, '').gsub(/\-+/, '-').gsub(/\-$/, '')
+      else
+        finalize_number(number)
+      end
+    end
+
+    def self.number_type(obj)
+      klass = obj.class
+
+      if klass == Fixnum || klass == Integer
+        :i
+      else
+        if ZhongwenTools::Zhongwen.zh?(obj)
+          # need to check zhs or zht
+          if zht?(obj)
+            :zht
+          else
+            :zhs
+          end
+        else #assume it is pyn
+          #if ZhongwenTools.const_defined?(:Romanization) && ZhongwenTools::Romanization.romanization?(obj)
+          # might need to convert to pyn
+          :pyn
+        end
+      end
+    end
+
+    def self.zht?(str)
+      str[/#{ZhongwenTools::Regex.zht_numbers }*/] == str
+    end
+
+    def self.zhs?(str)
+      !zht?(str)
+    end
+
+    def self.convert_from(from, to, number)
+      if from == :zht
+        convert_from_zh(to, number)
+      elsif from == :zhs
+        convert_from_zh(to, number)
+      elsif from == :i
+        convert_from_integer(to, number)
+      elsif from == :pyn
+        convert_from_pyn(to, number)
+      end
+    end
+
+    def self.convert_from_pyn(to, pyn)
+      # convert to pyn
+      # split the pyn and then
+      pyns = ZhongwenTools::Romanization::Pinyin.split_pyn(pyn)
+
+      pyns.map do |p|
+        convert_number(p).fetch( to ){ p }
+      end
+    end
+
+    def self.convert_from_zh(to, number)
+      converted_number = number.chars.map do |zh|
+        convert_number(zh).fetch(to){ zh }
+      end
+
+      converted_number
+    end
+
+    def self.combine_integers(integers)
+      return combine_year(integers) if year?(integers)
+
+      number = 0
+      length = integers.size
+      skipped = false
+
+      integers.each_with_index do |curr_num, i|
+        next if skipped == i
+
+        if (i+2) <= length
+          number, i = combine_integer(integers, number, curr_num, i)
+          skipped = i + 1
+        else
+          number = adjust_integer(number, curr_num)
+        end
+      end
+
+      number
+    end
+
+    def self.year?(integers)
+      integers.select{ |i| i < 10 }.size == integers.size
+    end
+
+    def self.combine_year(integers)
+      integers.map{ |i| i.to_s }.join.to_i
+    end
+
+    def self.combine_integer integers, result, curr_num, i
+      next_number = integers[i + 1]
+      if number_multiplier? next_number
+        result += next_number * curr_num
+      end
+
+      [result, i]
+    end
+
+    def self.adjust_integer(number, curr_num)
+      number_multiplier?(curr_num) ? number * curr_num : number + curr_num
+    end
+
+    def self.number_multiplier?(number)
+      [10,100,1_000,10_000,100_000_000].include? number
+    end
+
+
+    def self.convert_from_integer to, int
+      # FIXME: this will fail for numbers over 1 billion.
+      result = []
+      nums = convert_integer_to_reversed_array_of_integers(int)
+
+      nums.each_with_index do |num, i|
+        wan =  wan_level(wan, i)
+
+        if i == 0
+          result << convert_integer(num, to) unless num == 0
+        else
+          result << convert_wan_level(i, to)
+          # checks the wan level and ...
+          result << convert_integer(num, to) if wan_ok?(num, wan, i)
+        end
+      end
+
+      result.reverse!
+    end
+
+    def self.convert_integer_to_reversed_array_of_integers(int)
+      int.to_s.chars.to_a.reverse.map{ |x| x.to_i }
+    end
+
+    def self.wan_ok?(num, wan, i)
+      (num == 1 && (10**(i) / 10_000 ** wan) != 10) || num != 1
+    end
+
+    def self.wan_level(wan, i)
+      wan ||= 0
+      wan += 1 if (i + 1) % 5 == 0
+
+      wan
+    end
+
+    def self.convert_wan_level(i, to)
+      convert_integer((10**(i)), to) || convert_integer((10**(i) / 10_000), to) || convert_integer((10**(i) / 10_000**2), to)
+    end
+
+    def self.convert_integer(int, to)
+      NUMBERS_TABLE.find{ |x| x[:i] == int }.fetch(to){ 0 }
+    end
+
+    def self.convert_number(number)
+      NUMBERS_TABLE.find{ |x|  x[:zhs] == number || x[:zht] == number  || x[:pyn] == number }
+    end
+
+    def self.finalize_number(number, separator = '')
+      # FIXME: is finalize_number the best name you can think of?
+      # NOTE: Figuring out usage of "liang" vs. "er" is pretty
+      #       difficult, so always use "er" instead.
+      number.join(separator).gsub(/零#{ZhongwenTools::Regex.zh_number_multiple}/u,'')
+    end
+  end
+end

data/lib/zhongwen_tools/regex.rb

@@ -1,51 +1,68 @@
 # encoding: utf-8
+
 module ZhongwenTools
   module Regex
-    extend self
-
-    def pyn
-      /(#{pyn_regexes.values.join('|')}|r)([1-5])?([\s\-]+)?/
+    def self.pyn
+      /(#{pyn_regexes.values.join('|')}|r)([1-5])([\s\-]+)?/
     end
 
-    def py
+    def self.py
       # FIXME: need to detect Ālābó
       # ([ĀÁǍÀA][io]?|[io]?|[][āáǎàaēéěèeūúǔùu]?o?|[ĒÉĚÈE]i?|[]i?|[ŌÓǑÒO]u?|[]u?|u[āáǎàaēoēéěèe]?i?|[]e?)(n?g?r?)){1,}
-      /(#{pyn_regexes.map{|k,v| v.to_s[7..-2].gsub_with_hash(/[aeiouv]/,py_tones)}.join('|')}([\s\-])?)/
+      /(#{pyn_regexes.map{|k,v| v.to_s[7..-2].gsub(/[aeiouv]/,py_tones)}.join('|')}([\s\-])?)/
     end
 
-    def pinyin_num
+    def self.pinyin_num
       /(([BPMFDTNLGKHZCSRJQXWYbpmfdtnlgkhzcsrjqxwy]?[h]?)(A[io]?|a[io]?|i[aeu]?o?|Ei?|ei?|Ou?|ou?|u[aoe]?i?|ve?)?(n?g?)(r?)([1-5])(\-+)?)/
     end
 
-    def fullwidth
+    def self.pinyin_toneless
+      /(#{pyn_regexes.values.join('|')}|r)([\s\-]+)?/
+    end
+
+    def self.fullwidth
       /[０-９Ａ-Ｚａ-ｚ％．：＃＄＆＋－／＼＝；＜＞]/
     end
 
-    def capital_letters
-      /(#{Regexp.union(ZhongwenTools::UNICODE_CAPS.keys)})/
+    def self.capital_letters
+      /(#{Regexp.union(ZhongwenTools::Caps::CAPS.keys)})/
     end
 
-    def lowercase_letters
-      /(#{Regexp.union(ZhongwenTools::UNICODE_CAPS.values)})/
+    def self.lowercase_letters
+      /(#{Regexp.union(ZhongwenTools::Caps::CAPS.values)})/
     end
 
-    def zh
+    def self.zh
       /[\u2E80-\u2E99]|[\u2E9B-\u2EF3]|[\u2F00-\u2FD5]|[\u3005|\u3007]|[\u3021-\u3029]|[\u3038-\u303B]|[\u3400-\u4DB5]|[\u4E00-\u9FCC]|[\uF900-\uFA6D]|[\uFA70-\uFAD9]/
     end
 
-    def punc
+    def self.punc
       /[\u0021-\u0023]|[\u0025-\u002A]|[\u002C-\u002F]|[\u003A\u003B\u003F\u0040]|[\u005B-\u005D\u005F\u007B\u007D\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387]/
     end
 
-    def zh_punc
+    def self.zh_punc
       # TODO: includes non-zh punctuation codes. Should only include punctuation in CJK ranges.
       /[\u2E00-\u2E2E]|[\u2E30-\u2E3B]|[\u3001-\u3003]|[\u3008-\u3011]|[\u3014-\u301F]|[\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF]|[\uA60D-\uA60F]|[\uA673\uA67E]|[\uA6F2-\uA6F7]|[\uA874-\uA877]|[\uA8CE\uA8CF]|[\uA8F8-\uA8FA]|[\uA92E\uA92F\uA95F]|[\uA9C1-\uA9CD]|[\uA9DE\uA9DF]|[\uAA5C-\uAA5F]|[\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F]|[\uFE10-\uFE19]|[\uFE30-\uFE52]|[\uFE54-\uFE61]|[\uFE63\uFE68\uFE6A\uFE6B]|[\uFF01-\uFF03]|[\uFF05-\uFF0A]|[\uFF0C-\uFF0F]|[\uFF1A\uFF1B\uFF1F\uFF20]|[\uFF3B-\uFF3D]|[\uFF3F\uFF5B\uFF5D]|[\uFF5F-\uFF65]/
     end
 
-    def zh_numbers
+    def self.zh_numbers
       # TODO: include numbers like yotta, etc.
       # 垓	秭	穰	溝	澗	正	載 --> beyond 100,000,000!
-      /[〇零一壹幺二贰貳两兩三弎叁參四肆䦉五伍六陆陸七柒八捌九玖十拾廿百佰千仟万萬亿億]/
+      # Regional: Dong Guai
+      /[〇零一壹幺二贰貳两兩三弎叁參仨四肆䦉五伍六陆陸七柒八捌九玖十拾廿卅百佰千仟万萬亿億]/
+    end
+
+    def self.zhs_numbers
+      # TODO: check if 佰,仟 are the financial numbers in zhs
+      /[〇零一壹幺二贰两三弎叁仨四肆䦉五伍六陆七柒八捌九玖十拾廿卅百佰千仟万亿]/
+    end
+
+    def self.zht_numbers
+      /[〇零一壹幺二貳兩三弎參仨四肆䦉五伍六陸七柒八捌九玖十拾廿卅佰千仟萬億]/
+    end
+
+    def self.zh_number_multiple
+      /[拾十百佰千仟仟万萬亿億]/
     end
 
     # Public: A Regex for bopomofo, a.k.a. Zhuyin Fuhao 注音符号.
@@ -56,12 +73,13 @@ module ZhongwenTools
     #   bopomofo #=> <Regex>
     #
     # Returns a Regex.
-    def bopomofo
+    def self.bopomofo
       /[ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩ]/
     end
 
     private
-    def pyn_regexes
+
+    def self.pyn_regexes
       # http://stackoverflow.com/questions/20736291/regex-for-matching-pinyin
       # https://www.debuggex.com/r/_9kbxA6f00gIGiVo
       # NOTE: you might need to change the order of these regexes for more accurate matching of some pinyin.
@@ -81,7 +99,7 @@ module ZhongwenTools
       }
     end
 
-    def py_tones
+    def self.py_tones
       py_tones = {
         'a' => '[āáǎàa]',
         'e' => '[ēéěèe]',
@@ -93,5 +111,3 @@ module ZhongwenTools
     end
   end
 end
-
-require File.expand_path("../regex/ruby18", __FILE__) if RUBY_VERSION < '1.9'

data/lib/zhongwen_tools/romanization/pinyin.rb

@@ -0,0 +1,231 @@
+# encoding: utf-8
+require 'zhongwen_tools/regex'
+require 'zhongwen_tools/caps'
+require 'zhongwen_tools/romanization'
+
+module ZhongwenTools
+  module Romanization
+
+    def self.convert_to_py(str, from)
+      str =  convert_romanization(str, from, :pyn) if from != :pyn
+      ZhongwenTools::Romanization::Pinyin.convert_pyn_to_pinyin(str)
+    end
+
+    def self.convert_to_pyn(str, from)
+      orig_str = str.dup
+
+      if from == :py
+        str = ZhongwenTools::Romanization::Pinyin.convert_pinyin_to_pyn(str)
+      else
+        str = convert_romanization(str, from, :pyn)
+      end
+
+      str = ZhongwenTools::Romanization::Pinyin.add_hyphens_to_pyn(str) if hyphenated?(orig_str)
+
+      str
+    end
+
+    module Pinyin
+      %w(pinyin py pyn).each do |romanization|
+        define_singleton_method("to_#{romanization}") do |*args|
+          str, from = args
+          from ||= ZhongwenTools::Romanization.romanization? str
+
+          #_convert_romanization str, _set_type(type.to_sym), _set_type(from)
+          ZhongwenTools::Romanization.convert str, py_type(romanization), (py_type(from) || from)
+        end
+      end
+
+      def self.split_pyn(str)
+        # FIXME: ignore punctuation
+        regex = str[/[1-5]/].nil? ?  /(#{ZhongwenTools::Regex.pinyin_toneless})/ : /(#{ZhongwenTools::Regex.pyn})/
+
+        str.scan(regex).map{ |arr| arr[0].strip.gsub('-','') }.flatten
+      end
+
+      def self.split_py(str)
+        words = str.split(' ')
+
+        results = words.map do |word|
+          word, is_capitalized = normalize_pinyin(word)
+          result = word.split(/['\-]/).flatten.map do |x|
+            find_py(x)
+          end
+
+          recapitalize(result.flatten, is_capitalized)
+        end
+
+        results.flatten
+      end
+
+      # Public: checks if a string is pinyin.
+      #         http://en.wikipedia.org/wiki/Pinyin
+      #
+      # Examples
+      #   py?('nǐ hǎo')
+      #   # => true
+      #
+      # Returns Boolean.
+      def self.py?(str)
+        # NOTE: py regex does not include capitals with tones.
+        #ZhongwenTools::Caps.downcase(str).gsub(ZhongwenTools::Regex.punc,'').gsub(Regex.py, '').gsub(/[\s\-]/,'').strip == ''
+        regex = /(#{ ZhongwenTools::Regex.punc }|#{ ZhongwenTools::Regex.py }|[\s\-])/
+          ZhongwenTools::Caps.downcase(str).gsub(regex, '').strip == ''
+      end
+
+      # Public: checks if a string is pinyin.
+      #
+      # Examples
+      #   pyn?('pin1-yin1')
+      #   # => true
+      #
+      # Returns Boolean.
+      def self.pyn?(str)
+        # FIXME: use strip_punctuation method
+        normalized_str = ZhongwenTools::Caps.downcase(str.gsub(ZhongwenTools::Regex.punc,'').gsub(/[\s\-]/,''))
+        pyn_arr = split_pyn(normalized_str).map{ |p| p }
+
+        pyn_matches_properly?(pyn_arr, normalized_str) &&
+          are_all_pyn_syllables_complete?(pyn_arr)
+      end
+
+      def self.add_hyphens_to_pyn(str)
+        results = str.split(' ').map do |s|
+          split_pyn(s).join('-')
+        end
+
+        results.join(' ')
+      end
+
+      private
+
+      def self.pyn_matches_properly?(pyn_arr, normalized_str)
+        pyn_arr.join('') == normalized_str
+      end
+
+      def self.are_all_pyn_syllables_complete?(pyn_arr)
+        pyns = ROMANIZATIONS_TABLE.map{ |r| r[:pyn] }
+
+        pyn_syllables = pyn_arr.select do |p|
+          pyns.include?(p.gsub(/[1-5]/, ''))
+        end
+
+        pyn_arr.size == pyn_syllables.size
+      end
+
+      def self.py_type(romanization)
+        romanization = romanization.to_s.downcase.to_sym
+
+        { pyn: :pyn, py: :py, pinyin: :py }[romanization]
+      end
+
+
+      def self.normalize_pinyin(pinyin)
+        [ZhongwenTools::Caps.downcase(pinyin), capitalized?(pinyin)]
+      end
+
+      def self.find_py(str)
+        str.scan(ZhongwenTools::Regex.py).map{ |x| (x - [nil])[0] }
+
+      end
+
+      def self.recapitalize(obj, capitalized)
+        return obj unless capitalized
+
+        if obj.class == String
+          ZhongwenTools::Caps.capitalize(obj)
+        elsif obj.class == Array
+          [ZhongwenTools::Caps.capitalize(obj[0]), obj[1..-1]].flatten
+        end
+      end
+
+      # Internal: converts real pinyin to pinyin number string.
+      #
+      # pinyin - A String for the pinyin.
+      #
+      # Examples
+      #
+      #   convert_pinyin_to_pyn('Nǐ hǎo ma') #=> 'Ni3 hao3 ma5?'
+      #
+      # Returns a String in pinyin number format.
+      def self.convert_pinyin_to_pyn(pinyin)
+        words = pinyin.split(' ')
+
+        pyn = words.map do |word|
+          # NOTE: if a word is upcase, then it will be converted the same
+          #       as a word that is only capitalized.
+          word, is_capitalized = normalize_pinyin(word)
+
+          pys = split_py(word)
+          #is_capitalized ? ZhongwenTools::Caps.capitalize(result) : result
+          recapitalize(current_pyn(word, pys), is_capitalized)
+        end
+
+        pyn.join(' ')
+      end
+
+      def self.capitalized?(str)
+        str[0] != ZhongwenTools::Caps.downcase(str[0])
+      end
+
+      def self.current_pyn(pyn, pinyin_arr)
+        replacements = []
+        pinyin_arr.each do |pinyin|
+          replace =  pinyin_replacement(pinyin)
+          match = pinyin
+          if replacements.size > 0
+            pyn = pyn.sub(/(#{replacements.join('.*')}.*)#{match}/){ $1 + replace }
+          else
+            pyn = pyn.sub(/#{match}/){ "#{$1}#{replace}"}
+            end
+          replacements << replace
+        end
+
+        pyn.gsub("'", '')
+      end
+
+      def self.pinyin_replacement(py)
+        matches = PYN_PY.values.select do |x|
+          py.include? x
+        end
+        match = select_pinyin_match(matches)
+        replace = PYN_PY.find{|k,v| k if v == match}[0]
+
+        py.gsub(match, replace).gsub(/([^\d ]*)(\d)([^\d ]*)/){$1 + $3 + $2}
+      end
+
+      def self.select_pinyin_match(matches)
+        # take the longest pinyin match. Use bytes because 'è' is prefered over 'n' or 'r' or 'm'
+        match = matches.sort{|x,y| x.bytes.to_a.length <=> y.bytes.to_a.length}[-1]
+
+        # Edge case.. en/eng pyn -> py conversion is one way only.
+        match[/^(ē|é|ě|è|e)n?g?/].nil? ? match : match.chars[0]
+      end
+
+
+      #  Internal: Replaces numbered pinyin with actual pinyin. Pinyin separated with hyphens are combined as one word.
+      #
+      #  str - A String to replace with actual pinyin
+      #
+      #  Examples
+      #
+      #    convert_pyn_to_pinyin 'Ni3 hao3 ma5?' # => "Nǐ hǎo ma?"
+      #
+      #
+      #  Returns a string with actual pinyin
+      def self.convert_pyn_to_pinyin(str)
+        regex = Regex.pinyin_num
+        # Using gsub is ~8x faster than using scan and each.
+        # Explanation: if it's pinyin without vowels, e.g. m, ng, then convert,
+        #              otherwise, check if it needs an apostrophe (http://www.pinyin.info/romanization/hanyu/apostrophes.html).
+        #              If it does, add it and then convert. Otherwise, just convert.
+        #              Oh, and if it has double hyphens, replace with one hyphen.
+        #              And finally, correct those apostrophes at the very end.
+        #              It's like magic.
+        str.gsub(regex) do
+          ($3.nil? ? "#{PYN_PY[$1]}" : ($2 == '' && ['a','e','o'].include?($3[0,1]))? "'#{PYN_PY["#{$3}#{$6}"]}#{$4}#{$5}" : "#{$2}#{PYN_PY["#{$3}#{$6}"]}#{$4}#{$5}") + (($7.to_s.length > 1) ? '-' : '')
+        end.gsub("-'","-").sub(/^'/,'')
+      end
+    end
+  end
+end

data/lib/zhongwen_tools/romanization/{pyn_to_py.rb → pinyin_table.rb}

@@ -2,7 +2,8 @@
 
 # NOTE: This table works for pyn -> pinyin conversion, but it introduces
 #       mistakes when converting pinyin to pyn. In practice, pinyin can't
-#       be converted to pyn properly unless it's properly formatted.
+#       be converted to pyn with complete accuracy unless it is properly 
+#       formatted.
 module ZhongwenTools
   module Romanization
     PYN_PY = {