ruby-pinyin-ez 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 92a6c23a0492659ffbe8d356f6d845d060a4ac7c801669d9d59f742bac7c4bf6
+  data.tar.gz: 9cbf2d4733d3ffe853cf8968d916a647d9f934f10de77eeecf6176c375edb135
+SHA512:
+  metadata.gz: 9e2af60065a7a96c17bef6d5f9e3ccb069b575710f2c18cd561785bea898d5edd0d5bd6ea7a6a70375246e82c8f6804c5fe1959c7e117304dfeb7a17984fce9e
+  data.tar.gz: 5845a4e2a627c32cc5dfe90490acdce961d3ebfa3f71b8ab58325110c47fc762744b8965fdbfbfac62c71c7d91ab335c5e0371bb55a2858dbb66b3a667295a91

data/LICENSE

@@ -0,0 +1,10 @@
+Copyright (c) 2012, Jan Xie <jan.h.xie@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+    * Neither the name of Jan Xie nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.markdown

@@ -0,0 +1,100 @@
+# ruby-pinyin: 支持多音字的汉字转拼音工具
+[![Build Status](https://travis-ci.org/janx/ruby-pinyin.svg?branch=master)](https://travis-ci.org/janx/ruby-pinyin)
+
+ruby-pinyin: zhī chí duō yīn zì de hàn zì zhuǎn pīn yīn gōng jù
+
+ruby-pinyin可以把汉字转化为对应的拼音，并能够较好的处理多音字的情况。比如：
+
+        PinYin.of_string('南京市长江大桥', :ascii)
+
+能够正确的将“长”转为"chang2", 而不是"zhang3".
+
+## Features
+
+* 支持多音字
+* 使用最新的UNICODE数据(6.3.0 published at 2013/02/26)
+* 能够显示数字或者UNICODE音调(eg: 'cao1', 'cāo')
+* 丰富的API
+* 支持中英文标点混合字符串
+* 中文标点转为英文标点
+* 支持自定义读音
+
+## Installation
+
+        gem install ruby-pinyin
+
+或者把ruby-pinyin加入你的Gemfile:
+
+        gem 'ruby-pinyin'
+
+## Examples
+
+        # encoding: utf-8
+        require 'ruby-pinyin'
+
+        # return ['jie', 'cao']
+        PinYin.of_string('节操')
+
+        # return ['jie2', 'cao1']
+        PinYin.of_string('节操', true)
+        PinYin.of_string('节操', :ascii)
+
+        # return ["jié", "cāo"]
+        PinYin.of_string('节操', :unicode)
+
+        # 正确处理多音字: return ["nán", "jīng", "shì", "cháng", "jiāng", "dà", "qiáo"]
+        PinYin.of_string('南京市长江大桥', :unicode)
+
+        # return %w(gan xie party gan xie guo jia)
+        PinYin.of_string('感谢party感谢guo jia')
+
+        # return 'gan-xie-party-gan-xie-guo-jia'
+        PinYin.permlink('感谢party感谢guo jia')
+
+        # return 'gxpartygxguojia'
+        PinYin.abbr('感谢party感谢guo jia')
+
+        # return 'gan xie party, gan xie guo jia!'
+        # PinYin.sentence保留标点符号, 同时用对应英文标点代替中文标点
+        PinYin.sentence('感谢party， 感谢guo家！')
+
+        # override readings with your own data file
+        PinYin.override_files = [File.expand_path('../my.dat', __FILE__)]
+
+更多的例子和参数请参考[测试用例](https://github.com/janx/ruby-pinyin/blob/master/test/pinyin_test.rb)
+
+## 配置 ##
+
+ruby-pinyin有两个PinYin::Backend: `PinYin::Backend::Simple` 以及`PinYin::Backend::MMSeg`. 默认是使用MMSeg backend, 支持多音字识别。如果你不需要多音字识别，或是对内存使用要求很高，或是有其它任何原因想要fallback到Simple backend, 可以如下配置：
+
+```ruby
+PinYin.backend = PinYin::Backend::Simple.new
+```
+
+## 自定义发音 ##
+
+通过`PinYin.override_files`可以自定义某些字的发音。自定义的数据以普通文本文件存放，每行定义一个字的发音，以ASCII空格将汉字的unicode编码和拼音隔开。格式可参考[lib/ruby-pinyin/data/Mandarin.dat](https://github.com/janx/ruby-pinyin/blob/master/lib/ruby-pinyin/data/Mandarin.dat)文件。
+
+## 欢迎任何帮助 ##
+
+如果你喜欢这个项目，请通过(不限)以下方式帮助她!
+
+* 各种使用
+* 各种宣传
+* 各种报告bug, 提供建议  (github issue tracker)
+* 各种修bug, 实现feature (github pull request)
+
+## LICENSE ##
+
+[BSD LICENSE](https://github.com/janx/ruby-pinyin/blob/master/LICENSE)
+
+ruby-pinyin中的拼音数据由作者整理自互联网，你可以在ruby-pinyin之外的地方任意使用，但是请注明数据来自ruby-pinyin :-)
+
+## Contributors ##
+
+* [Martin91](https://github.com/Martin91)
+* [jaxi](https://github.com/jaxi)
+* [jiangxin](https://github.com/jiangxin)
+* [forresty](https://github.com/forresty)
+* [pzpz](https://github.com/pzpz)
+* [Eric Guo](https://github.com/Eric-Guo)

data/lib/ruby-pinyin/backend/ezseg.rb

@@ -0,0 +1,101 @@
+module PinYin
+  module Backend
+    class EZSeg
+
+      def initialize(override_files=[])
+        @override_files = override_files || []
+      end
+
+
+
+      def romanize(str, tone=nil, include_punctuations=false)
+        return [] unless str && str.length > 0
+        words = segment str
+
+        res = []
+        words.each do |word|
+          if str && !str.empty?
+            word.unpack('U*').each_with_index do |t,idx|
+              code = sprintf('%x',t).upcase
+              readings = codes[code]
+
+              if readings
+                multiple_arr = readings.collect{|one| Value.new(format([one], tone), false)}
+                res << (multiple_arr.length > 1 ? multiple_arr : multiple_arr[0])
+              else
+                val = [t].pack('U*')
+                if val =~ /^[0-9a-zA-Z\s]*$/ # 复原，去除特殊字符,如全角符号等。
+                  if res.last && res.last.respond_to?(:english?) && res.last.english?
+                    res.last << Value.new(val, true)
+                  elsif val != ' '
+                    res << Value.new(val, true)
+                  end
+                elsif include_punctuations
+                  val = [Punctuation[code]].pack('H*') if Punctuation.include?(code)
+                  (res.last ? res.last : res) << Value.new(val, false)
+                end
+              end
+            end
+          end
+        end
+        res
+      end
+
+      private
+
+      def codes
+        return @codes if @codes
+
+        @codes = {}
+        src = File.expand_path('../../data/Mandarin.dat', __FILE__)
+        @override_files.unshift(src).each do |file|
+          load_codes_from(file)
+        end
+        @codes
+      end
+
+      def load_codes_from(file)
+        File.readlines(file).map do |line|
+          code, readings = line.split(' ')
+          @codes[code] = readings.split(',')
+        end
+      end
+
+      def format(readings, tone)
+        case tone
+        when :unicode
+          readings[0]
+        when :ascii, true
+          PinYin::Util.to_ascii(readings[0])
+        else
+          PinYin::Util.to_ascii(readings[0], false)
+        end
+      end
+
+      def segment(str)
+        words = []
+        str.split('').each do |s|
+          words.push(s) unless s =~ Punctuation.chinese_regexp
+        end
+
+        words
+      end
+
+      # def apply(base, patch)
+      #   result = []
+      #   base.each_with_index do |char, i|
+      #     if patch[i].nil?
+      #       result.push char
+      #     elsif char =~ Punctuation.regexp
+      #       result.push Value.new("#{patch[i]}#{$1}", char.english?)
+      #     else
+      #       result.push Value.new(patch[i], char.english?)
+      #     end
+      #   end
+      #   result
+      # end
+
+
+    end
+  end
+end

data/lib/ruby-pinyin/backend/mmseg.rb

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+require 'rmmseg-cpp-new'
+
+module PinYin
+  module Backend
+    class MMSeg
+
+      def initialize(override_files=[])
+        @simple = Simple.new override_files
+
+        RMMSeg::Dictionary.dictionaries.delete_if {|(type, path)| type == :words}
+        RMMSeg::Dictionary.dictionaries.push [:words, File.expand_path('../../data/words.dic', __FILE__)]
+        RMMSeg::Dictionary.load_dictionaries
+      end
+
+      def romanize(str, tone=nil, include_punctuations=false)
+        return [] unless str && str.length > 0
+
+        words = segment str
+
+        base = @simple.romanize(str, tone, include_punctuations)
+        patch = words.map {|w| format(w, tone) }.flatten
+
+        if base.size != patch.size
+          base.compact!
+          patch.compact!
+        end
+
+        apply base, patch
+      end
+
+      def segment(str)
+        algor = RMMSeg::Algorithm.new str
+
+        words = []
+        while token = algor.next_token
+          s = token.text.force_encoding("UTF-8")
+          words.push(s) unless s =~ Punctuation.chinese_regexp
+        end
+        words
+      end
+
+      private
+
+      def dictionary
+        return @dict if @dict
+
+        @dict = {}
+        src = File.expand_path('../../data/words.dat', __FILE__)
+        File.readlines(src).map do |line|
+          word, unicode = line.strip.split(',')
+          @dict[word] = unicode
+        end
+
+        @dict
+      end
+
+      def get_pinyin(word, tone)
+        return unless dictionary[word]
+
+        case tone
+        when :unicode
+          dictionary[word]
+        when :ascii, true
+          to_ascii dictionary[word], true
+        else
+          to_ascii dictionary[word], false
+        end
+      end
+
+      def to_ascii(word, with_tone)
+        word.split(' ').map do |reading|
+          PinYin::Util.to_ascii(reading, with_tone)
+        end.join(' ')
+      end
+
+      def format(word, tone)
+        pinyin = get_pinyin(word, tone)
+        return pinyin.split(' ') if pinyin
+
+        #如果是个英文单词，直接返回，否则返回与词等长的nil数组
+        if word =~ /^[_0-9a-zA-Z\s]*$/
+          word
+        elsif word.respond_to? :force_encoding
+          # word has been encoded in UTF-8 already
+          [nil] * word.size
+        else
+          # For ruby 1.8, there is no native utf-8 support
+          [nil] * word.unpack('U*').size
+        end
+      end
+
+      def apply(base, patch)
+        result = []
+        base.each_with_index do |char, i|
+          if patch[i].nil?
+            result.push char
+          elsif char =~ Punctuation.regexp
+            result.push Value.new("#{patch[i]}#{$1}", char.english?)
+          else
+            result.push Value.new(patch[i], char.english?)
+          end
+        end
+        result
+      end
+
+    end
+  end
+end

data/lib/ruby-pinyin/backend/simple.rb

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+module PinYin
+  module Backend
+    class Simple
+
+      def initialize(override_files=[])
+        @override_files = override_files || []
+      end
+
+      def romanize(str, tone=nil, include_punctuations=false)
+        res = []
+        return res unless str && !str.empty?
+
+        str.unpack('U*').each_with_index do |t,idx|
+          code = sprintf('%x',t).upcase
+          readings = codes[code]
+
+          if readings
+            res << Value.new(format(readings, tone), false)
+          else
+            val = [t].pack('U*')
+            if val =~ /^[0-9a-zA-Z\s]*$/ # 复原，去除特殊字符,如全角符号等。
+              if res.last && res.last.english?
+                res.last << Value.new(val, true)
+              elsif val != ' '
+                res << Value.new(val, true)
+              end
+            elsif include_punctuations
+              val = [Punctuation[code]].pack('H*') if Punctuation.include?(code)
+              (res.last ? res.last : res) << Value.new(val, false)
+            end
+          end
+        end
+
+        res.map {|phrase| phrase.split(/\s+/)}.flatten
+      end
+
+      private
+
+      def codes
+        return @codes if @codes
+
+        @codes = {}
+        src = File.expand_path('../../data/Mandarin.dat', __FILE__)
+        @override_files.unshift(src).each do |file|
+          load_codes_from(file)
+        end
+        @codes
+      end
+
+      def load_codes_from(file)
+        File.readlines(file).map do |line|
+          code, readings = line.split(' ')
+          @codes[code] = readings.split(',')
+        end
+      end
+
+      def format(readings, tone)
+        case tone
+        when :unicode
+          readings[0]
+        when :ascii, true
+          PinYin::Util.to_ascii(readings[0])
+        else
+          PinYin::Util.to_ascii(readings[0], false)
+        end
+      end
+
+    end
+  end
+end

data/lib/ruby-pinyin/backend.rb

@@ -0,0 +1,7 @@
+module PinYin
+  module Backend
+    autoload :Simple, 'ruby-pinyin/backend/simple'
+    autoload :MMSeg,  'ruby-pinyin/backend/mmseg'
+    autoload :EZSeg,  'ruby-pinyin/backend/ezseg'
+  end
+end