ting 0.2.0

data/History.txt

@@ -0,0 +1,16 @@
+== 0.2 / 2010-04-04
+
+* Added support for superscript numeral tones and IPA tone marks
+
+== 0.1.3 / 2008-07-18
+
+* made compatible with the latest release of Facets
+
+== 0.1.0 / 2007-12-14
+
+* Converted to Hoe, bugfixes
+
+== 0.0.1 / 2007-07-26
+
+* Birthday!
+

data/README.rdoc

@@ -0,0 +1,94 @@
+= Ting
+
+Ting can convert between various systems for phonetically
+writing Mandarin Chinese. It can also handle various representation
+of tones, so it can be used to convert pinyin with numbers
+to pinyin with tones.
+
+Hanyu Pinyin, Bopomofo, Wade-Giles, Tongyong Pinyin
+and International Phonetic Alphabet (IPA) are supported.
+
+== SYNOPSIS
+
+To parse your strings create a +Reader+ object. Ting.reader() takes two
+parameters : the transliteration format, and the way that tones are represented.
+
+To some extent these can be mixed and matched.
+
+To generate pinyin/wade-giles/etc. create a +Writer+ object. Use Ting.writer()
+
+=== Formats
+
+* +:hanyu+ Hanyu Pinyin
+* +:zhuyin+ Zhuyin Fuhao (a.k.a. Bopomofo)
+* +:wadegiles+ Wade Giles
+* +:ipa+ International Phonetic Alphabet
+* +:tongyong+ Tongyong Pinyin
+
+=== Tones
+
+* +:numbers+ Simply put a number after the syllable, easy to type
+* +:accents+ Use diacritics, follows the Hanyu Pinyin rules, there needs to be at least one vowel to apply this to, not usable with IPA or Bopomofo
+* +:supernum+ Superscript numerals, typically used for Wade-Giles
+* +:marks+ Tone mark after the syllable, typically used for Bopomofo
+* +:ipa+ IPA tone marks
+* +:no_tones+ Use no tones
+
+== Examples
+
+Parse Hanyu Pinyin
+
+   require 'ting'
+
+   reader = Ting.reader(:hanyu, :numbers)
+   reader << "wo3 ai4 ni3"
+    # => [<Ting::Syllable <initial=Empty, final=Uo, tone=3>>, 
+    #     <Ting::Syllable <initial=Empty, final=Ai, tone=4>>, 
+    #     <Ting::Syllable <initial=Ne, final=I, tone=3>>]
+
+Generate Bopomofo
+
+   zhuyin = Ting.writer(:zhuyin, :marks)
+   zhuyin << (reader << "wo3 ai4 ni3")
+   # => "ㄨㄛˇ ㄞˋ ㄋㄧˇ"
+
+Generate Wade-Giles
+
+   wadegiles = Ting.writer(:wadegiles, :supernum)
+   wadegiles << (reader << "yi1 ge5 bu2 gou4")
+   # => "i¹ ko pu² kou⁴"
+
+Generate IPA
+
+   ipa = Ting.writer.new(:ipa, :ipa)
+   ipa << (reader << "you3 peng2 zi4 yuan2 fang1 lai2")
+   # => "iou˧˩˧ pʰeŋ˧˥ ts˥˩ yɛn˧˥ faŋ˥˥ lai˧˥"
+
+Since this is such a common use case, a convenience method to add diacritics to pinyin.
+
+   require 'ting/string'
+
+   "wo3 ai4 ni3".pretty_tones
+   # => "wǒ ài nǐ"
+
+Note that syllables need to be separated by spaces, feeding "peng2you3" to the parser
+does not work. The String#pretty_tones method does handle these things a bit more gracefully.
+
+If you need to parse input that does not conform, consider using a regexp to scan for valid
+syllables, then feed the syllables to the parser one by one. Have a look at #pretty_tones for
+an example of how to do this.
+
+== REQUIREMENTS
+
+* $KCODE has to be set to "UTF8" for everything to work correctly
+
+== INSTALL
+
+* gem install ting
+
+== LICENSE
+
+Copyright (c) 2004-2010, Arne Brasseur. (http://www.arnebrasseur.net)
+
+Available as Free Software under the GPLv3 License, see LICENSE.txt for details
+

data/Rakefile

@@ -0,0 +1,15 @@
+require 'rubygems'
+
+require 'rake'
+require 'rake/testtask'
+
+task :default => [:"test:ting"]
+
+namespace "test" do
+  Rake::TestTask.new("ting") do |t|
+    $: << File.dirname(__FILE__) + '/lib'
+    t.pattern = 'test/*.rb'
+    t.verbose = true
+    t.warning = true
+  end
+end

data/TODO

@@ -0,0 +1,15 @@
+- Additional transcription systems
+  - MSP2 (or how do you call that)
+  - Palladiy (To make things interesting)
+  - Gwoyueh
+  - Yale
+
+- Research some rare pinyin syllables : lo, yo ^e, yai
+- Get a definitive answer about ong/ueng/weng
+- Add a README to the data/ directory with info on sources, contents and purposes
+- More tests
+- Add remembering of parameters to cgiform example, other examples
+
+The core lib basically does translation on the syllable level. It can handle strings with syllables nicely seperated by spaces. Successive layers should make it possible to convert a sentence with interpunction into a different system. It should be possible to write compound words together in Hanyu, and have the syllables seperated by dashes when converting to WG. For instance:
+
+Wǒ de péngyǒu, shì dàifu.  =>  Wǒ te p`éng-yǔ, shih tài-fu.

data/examples/cgiform/cgiform.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/ruby -w
+
+
+require 'cgi'
+require 'erb'
+
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'pinyin'
+
+cgi=CGI.new("xhtml1")
+
+params=cgi.params
+begin
+  if params['pinyin'] && params['pinyin'] != '' && params['pinyin'] != []
+    @converted = Pinyin::Writer.new(params['to'], params['to_tone']) << (Pinyin::Reader.new(params['from'],params['from_tone']) << params['pinyin'].first)
+  end
+rescue
+  cgi.out{$!.to_s}
+  cgi.out{params['pinyin'].inspect}
+end
+
+cgi.out("text/html; charset=utf-8") do
+  ERB.new(IO.read('template.rhtml')).result(binding)
+end

data/examples/cgiform/template.rhtml

@@ -0,0 +1,69 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Ruby Pinyin CGIForm example</title>
+    <style type='text/css'>
+      body {
+        font-family: sans-serif;
+      }
+
+      div#wrap {
+        width: 40%;
+        margin: 0 auto;
+      }
+
+      table {
+        width: 100%;
+      }      
+      div#converted_text {
+        border: 1px dotted #000;
+      }
+
+      textarea {
+	width: 100%;
+	height: 10em;
+	margin: 0 auto;
+      }
+    </style>
+  </head>
+  <body>
+    <div id='wrap'> 
+    <h2>Pinyin example application</h2>	
+    <h3>Enter some pinyin text and choose your format</h3>	
+    <table>
+      <form method='post'>
+        <tr>
+          <td colspan='2'>
+            <textarea name='pinyin'><%=params['pinyin'].first if params['pinyin'] != []%></textarea>
+          </td>
+        </tr>
+
+        <tr><td>From</td><td>To</td></tr>
+        <% Pinyin::Conversions::All.each do |f|%>
+          <tr>
+            <td><input type='radio' name='from' value='<%=f%>'><%=f.capitalize%></input></td>
+            <td><input type='radio' name='to' value='<%=f%>'><%=f.capitalize%></input></td>
+          </tr>
+        <% end %>
+        <tr><td>From tone</td><td>To tone</td></tr>
+        <% Pinyin::Tones::All.each do |f|%>
+          <tr>
+            <td><input type='radio' name='from_tone' value='<%=f%>'><%=f.capitalize%></input></td>
+            <td><input type='radio' name='to_tone' value='<%=f%>'><%=f.capitalize%></input></td>
+          </tr>
+        <% end %>
+        <tr>
+          <td><input type='submit'></input></td>
+          <td>&nbsp;</td>
+        </tr>
+      </form>
+    </table>
+    <% if @converted %>
+      <h2>Converted:</h2>
+      <div id='converted_text'>
+        <%= @converted %>
+      </div>
+    <% end %>  
+    </div>
+  </body>
+</html>

data/examples/hello.rb

@@ -0,0 +1,12 @@
+$: << File.join(File.dirname(__FILE__), '../lib')
+
+require 'pinyin'
+
+conv1 = Pinyin::Converter.new(:hanyu, :numbers, :wadegiles, :accents)
+conv2 = Pinyin::Converter.new(:wadegiles, :accents, :zhuyin, :marks)
+
+pinyin    = 'wo3 de2 peng2 you3 shi4 dai4 fu'
+wadegiles = conv1 << pinyin
+zhuyin    = conv2 << wadegiles
+
+puts pinyin, wadegiles, zhuyin

data/lib/ting.rb

@@ -0,0 +1,93 @@
+# Handle several romanization systems for Mandarin Chinese
+#
+# Author::     Arne Brasseur (arne@arnebrasseur.net)
+# Copyright::  Copyright (c) 2007-2010, Arne Brasseur
+# Licence::    GNU General Public License, v3
+
+$: << File.dirname(__FILE__)
+
+require 'ting/support'
+require 'ting/groundwork'
+require 'ting/exception'
+
+require 'ting/tones'
+require 'ting/conversion'
+require 'ting/conversions'
+require 'ting/conversions/hanyu'
+
+module Ting
+  VERSION = "0.2.0"
+
+  class Reader
+    def initialize(conv, tone)
+      @conv = conv.to_s
+      @tone = Tones.const_get tone.to_s.camelcase
+      @cache = {}
+    end
+
+    def parse(str)
+      return @cache[str] ||= Conversions.tokenize(str).map do |s, pos|
+        tone,syll = @tone.pop_tone(s)
+        tsyll = Conversions.parse(@conv,syll)
+        ini, fin = tsyll.initial, tsyll.final
+        unless tone && fin && ini
+          raise ParseError.new(s,pos),"Illegal syllable <#{s}> in input <#{str}> at position #{pos}." 
+        end
+        Syllable.new(ini, fin, tone)
+      end
+    rescue Object => e
+      raise ParseError.new(str,0), "Parsing of #{str.inspect} failed : #{e}"
+    end
+
+    alias :<< :parse
+  end
+
+  class Writer
+    def initialize(conv, tone)
+      @conv = conv.to_s
+      @tone = Tones.const_get tone.to_s.camelcase
+      @cache = {}
+    end
+
+    def generate(py)
+      conv=lambda {|syll| @tone.add_tone(Conversions.unparse(@conv,syll),syll.tone)}
+      return @cache[py] ||= if py.respond_to? :map
+        py.map(&conv).join(' ')
+      else
+        conv.call(py)
+      end
+    end
+
+    alias :<< :generate
+    alias :unparse :generate
+  end
+
+  class Converter
+    def initialize(from, from_tone, to, to_tone)
+      @reader = Reader.new(from, from_tone)
+      @writer = Writer.new(to, to_tone)
+    end
+
+    def convert(str)
+      @writer.unparse @reader.parse(str)
+    end
+
+    alias :<< :convert
+  end
+  
+  class <<self
+    READERS={}
+    WRITERS={}
+
+    def reader(format, tones)
+      return READERS[[format, tones]] ||= Reader.new(format,tones)
+    end
+    def writer(format, tones)
+      return WRITERS[[format, tones]] ||= Writer.new(format,tones)
+    end
+  end
+
+end
+
+
+Pinyin = Ting #legacy support

data/lib/ting/conversion.rb

@@ -0,0 +1,51 @@
+module Ting
+  
+  #
+  # Base class for conversions like Hanyu pinyin, 
+  # Wade-Giles, etc.
+  #
+  class Conversion
+
+    # Separator between syllables in the same word
+    # For Wade-Giles this is a dash, Hanyu pinyin
+    # uses a single quote in certain situations
+    attr_reader :syllable_separator
+    
+    # The tone handling object
+    attr_reader :tones
+
+    # An optional lambda that preprocesses input
+    attr_reader :preprocessor
+
+    # The name of this conversion, the same name used
+    # in the data file and that is also available as
+    # a method name on Initial and Final objects.
+    #
+    # By default the underscorized class name
+    attr_reader :name
+
+    def initialize(tone = :numbers, options = {})
+      @preprocessor = options[:preprocessor] || lambda {|s| s}
+
+      if Tone === tone
+        @tone = tone
+      else
+        @tone = Ting::Tones.const_get(tone.to_s.camelcase)
+      end
+
+      @name = self.class.name.underscore
+    end
+
+    # Converts a string into an array of strings and
+    # syllable objects.
+    def parse(string)
+    end
+
+    # Converts an array of strings and syllable objects
+    # into a string
+    def unparse(array)
+    end
+
+  end
+end
+    

data/lib/ting/conversions.rb

@@ -0,0 +1,75 @@
+require 'csv'
+require 'yaml'
+
+module Ting
+  module Conversions
+    All=[]
+
+    DATA_DIR=File.dirname(__FILE__)+'/data/'
+
+    #Load various representations for initials and finals
+    %w(Initial Final).each do |c|
+      klazz=Ting.const_get c
+      begin
+        CSV.open(DATA_DIR+c.downcase+'.csv', 'r').each do |name, *values|
+          All << name.to_s unless All.index name || name =~ /name|standalone/i
+          klazz.class_eval {attr_accessor name.to_sym}
+          values.each_with_index do |v,i|
+            klazz::All[i].send(name+'=', v)
+          end
+        end
+      rescue
+        puts "Bad data in #{c.downcase}.csv : " + $!
+        raise
+      end
+      
+    end
+
+    #Substitution rules 
+    @@rules=YAML::load(IO.read(DATA_DIR+'rules.yaml'))
+
+    def self.parse(type, string)
+      if (fin = Final::All.find {|f| f.respond_to?("#{type}_standalone") && f.send("#{type}_standalone") == string})
+        TonelessSyllable.new(Initial::Empty, fin)
+      else
+        Initial::All.find do |ini|
+          Final::All.find do |fin|
+            next                                  if TonelessSyllable.illegal?(ini,fin)
+            return TonelessSyllable.new(ini,fin)  if apply_rules(type, (ini.send(type)||'') + (fin.send(type)||'')) == string
+          end
+        end
+      end
+    end
+    
+    def self.unparse(type, tsyll)
+      if tsyll.initial.send(type)
+        apply_rules(type, tsyll.initial.send(type) + (tsyll.final.send(type) || ''))
+      elsif tsyll.final.respond_to?(type.to_s+'_standalone') && standalone = tsyll.final.send(type.to_s+'_standalone')
+        standalone
+      else
+        apply_rules(type, tsyll.final.send(type))
+      end
+    end
+
+    def self.tokenize(str)
+      returning [] do |ary|
+        str,pos = str.dup, 0
+        while s=str.slice!(/[^' ]*/) and s != ""
+          ary << [s.strip, pos]
+          pos+=s.length
+          str.slice!(/[' ]/)
+        end
+      end
+    end
+    
+    private
+      def self.apply_rules(type, string)
+        returning string.dup do |s|
+          @@rules[type] && @@rules[type].each do |rule|
+            s.gsub!(Regexp.new(rule['match']),rule['subst'])
+          end
+        end
+      end
+
+  end
+end