pinyin 0.0.1 → 0.1.4

data/Rakefile

@@ -0,0 +1,36 @@
+require 'rubygems'
+
+require 'rake'
+require 'rake/testtask'
+require 'hoe'
+
+$:.unshift './lib'
+
+require 'pinyin'
+
+Hoe.new('pinyin', Pinyin::VERSION) do |p|
+  p.rubyforge_name = 'pinyin'
+  p.summary = "A conversion library for Chinese transcription methods like Hanyu Pinyin, Bopomofo and Wade-Giles"
+  p.description = p.paragraphs_of('README', 2).join
+  p.url = "http://rubyforge.org/projects/pinyin"
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.email = "pinyin@arnebrasseur.net"
+  p.author = 'Arne Brasseur'
+  p.remote_rdoc_dir=""
+  p.spec_extras = {
+    :extra_rdoc_files => ["README", "History.txt"],
+    :rdoc_options => ["--main", "README"],
+    :platform => Gem::Platform::RUBY
+  }
+end
+
+task :default => [:test_units]
+
+namespace "test" do
+  Rake::TestTask.new("pinyin") do |t|
+    $: << File.dirname(__FILE__) + '/lib'
+    t.pattern = 'test/*.rb'
+    t.verbose = true
+    t.warning = true
+  end
+end

data/TODO

@@ -11,6 +11,7 @@
   - Yale
 
 - Research some rare pinyin syllables : lo, yo ^e, yai
+- Get a definitive answer about ong/ueng/weng
 - Add a general README as rdoc start page
 - Add a README to the data/ directory with info on sources, contents and purposes
 - More tests
@@ -19,4 +20,4 @@
 !More
 The core lib basically does translation on the syllable level. It can handle strings with syllables nicely seperated by spaces. Successive layers should make it possible to convert a sentence with interpunction into a different system. It should be possible to write compound words together in Hanyu, and have the syllables seperated by dashes when converting to WG. For instance:
 
-Wǒ de péngyǒu, shì dàifu.  =>  Wǒ te p`éng-yǔ, shih tài-fu.
+Wǒ de péngyǒu, shì dàifu.  =>  Wǒ te p`éng-yǔ, shih tài-fu.>>>>>>> .r211

data/lib/pinyin.rb

@@ -6,21 +6,24 @@
 
 $: << File.dirname(__FILE__)
 
-require 'support'
-require 'groundwork'
-require 'exception'
+require "facets/string/camelcase"
 
-require 'tones'
-Pinyin::Tones::All.each{|m| require 'tones/'+m}
-
-require 'conversions'
+require 'pinyin/support'
+require 'pinyin/groundwork'
+require 'pinyin/exception'
 
+require 'pinyin/tones'
+require 'pinyin/conversion'
+require 'pinyin/conversions'
+require 'pinyin/conversions/hanyu'
 
 module Pinyin
+  VERSION = "0.1.4"
+
   class Reader
     def initialize(conv, tone)
-      @conv = conv.to_s #Conversions.const_get conv.to_s.camelize
-      @tone = Tones.const_get tone.to_s.camelize
+      @conv = conv.to_s
+      @tone = Tones.const_get tone.to_s.camelcase
     end
 
     def parse(str)
@@ -28,9 +31,13 @@ module Pinyin
         tone,syll = @tone.pop_tone(s)
         tsyll = Conversions.parse(@conv,syll)
         ini, fin = tsyll.initial, tsyll.final
-        raise ParseError.new(s,pos),"Illegal syllable <#{s}> in input <#{str}> at position #{pos}." unless tone && fin && ini
+        unless tone && fin && ini
+          raise ParseError.new(s,pos),"Illegal syllable <#{s}> in input <#{str}> at position #{pos}." 
+        end
         Syllable.new(ini, fin, tone)
       end
+    rescue Object => e
+      raise ParseError.new(str,0), "Parsing of #{str.inspect} failed : #{e}"
     end
 
     alias :<< :parse
@@ -38,8 +45,8 @@ module Pinyin
 
   class Writer
     def initialize(conv, tone)
-      @conv = conv.to_s #Conversions.const_get conv.to_s.camelize
-      @tone = Tones.const_get tone.to_s.camelize
+      @conv = conv.to_s
+      @tone = Tones.const_get tone.to_s.camelcase
     end
 
     def unparse(py)
@@ -66,6 +73,18 @@ module Pinyin
 
     alias :<< :convert
   end
+  
+  class <<self
+    Conversions::All.each do |c|
+      define_method "#{c.to_s.camelcase}Reader" do |tone|
+        Reader.new(c, tone)
+      end
+
+      define_method "#{c.to_s.camelcase}Writer" do |tone|
+        Writer.new(c, tone)
+      end
+    end
+  end
 end
 
 

data/lib/pinyin/conversion.rb

@@ -0,0 +1,51 @@
+module Pinyin
+  
+  #
+  # Base class for conversions like Hanyu pinyin, 
+  # Wade-Giles, etc.
+  #
+  class Conversion
+
+    # Separator between syllables in the same word
+    # For Wade-Giles this is a dash, Hanyu pinyin
+    # uses a single quote in certain situations
+    attr_reader :syllable_separator
+    
+    # The tone handling object
+    attr_reader :tones
+
+    # An optional lambda that preprocesses input
+    attr_reader :preprocessor
+
+    # The name of this conversion, the same name used
+    # in the data file and that is also available as
+    # a method name on Initial and Final objects.
+    #
+    # By default the underscorized class name
+    attr_reader :name
+
+    def initialize(tone = :numbers, options = {})
+      @preprocessor = options[:preprocessor] || lambda {|s| s}
+
+      if Tone === tone
+        @tone = tone
+      else
+        @tone = Pinyin::Tones.const_get(tone.to_s.camelcase)
+      end
+
+      @name = self.class.name.underscore
+    end
+
+    # Converts a string into an array of strings and
+    # syllable objects.
+    def parse(string)
+    end
+
+    # Converts an array of strings and syllable objects
+    # into a string
+    def unparse(array)
+    end
+
+  end
+end
+    

data/lib/{conversions.rb → pinyin/conversions.rb}

@@ -12,7 +12,7 @@ module Pinyin
       klazz=Pinyin.const_get c
       begin
         CSV.open(DATA_DIR+c.downcase+'.csv', 'r').each do |name, *values|
-          All << name unless All.index name || name =~ /name|standalone/i
+          All << name.to_s unless All.index name || name =~ /name|standalone/i
           klazz.class_eval {attr_accessor name.to_sym}
           values.each_with_index do |v,i|
             klazz::All[i].send(name+'=', v)
@@ -63,12 +63,13 @@ module Pinyin
     end
     
     private
-    def self.apply_rules(type, string)
-      returning string.dup do |s|
-        @@rules[type] && @@rules[type].each do |rule|
-          s.gsub!(Regexp.new(rule['match']),rule['subst'])
+      def self.apply_rules(type, string)
+        returning string.dup do |s|
+          @@rules[type] && @@rules[type].each do |rule|
+            s.gsub!(Regexp.new(rule['match']),rule['subst'])
+          end
         end
       end
-    end
+
   end
 end

data/lib/pinyin/conversions/hanyu.rb

@@ -0,0 +1,77 @@
+module Pinyin
+  module Conversions
+    class Hanyu
+      def initialize(tone = :numbers, options = {})
+        @options = options
+        @options[:preprocess] ||= lambda {|s| s.gsub(/u:|Ü/, 'ü').downcase }
+        
+        if Class === tone
+          @tone = tone
+        else
+          @tone = Pinyin::Tones.const_get(tone.to_s.camelcase)
+        end
+      end
+
+      def valid_character_regexp
+        @valid_character_regexp ||= valid_character_regexp!
+      end
+      
+      def valid_character_regexp!
+        valid_chars = []
+        Pinyin.valid_combinations do |i,f|
+          1.upto(5) do |tone|
+            valid_chars += @tone.add_tone(Conversions.unparse(:hanyu,TonelessSyllable.new(i,f)), tone).chars
+          end
+        end
+        valid_chars.sort!.uniq!
+        Regexp.new(valid_chars.map{|ch| Regexp.escape(ch)}.join('|'))
+      end
+
+      def parse(string)
+        result = []
+        looking_at = []
+        string.chars.each do |ch|
+          head, syll = parse_tail(looking_at)
+          looking_at << ch
+          if syll && !parse_tail(looking_at)
+            puts "-> #{syll.inspect}"
+            result << head.to_s unless head.empty?
+            result << syll
+            looking_at = [ch]
+          end
+        end
+        result
+      end
+
+      def parse_tail(chars)
+        7.downto(1) do |i|
+          head = chars[0...-i]
+          tail = chars[-i..-1]
+          syll = parse_syllable( tail )
+          return head, syll if syll
+        end
+        nil
+      end
+
+      def parse_syllable(tone_syll)
+        tone_syll = tone_syll.to_s
+        tone_syll = @options[:preprocess].call(tone_syll) if @options[:preprocess]
+#        p tone_syll
+        tone, syll = @tone.pop_tone(tone_syll)
+        if tone && syll
+          ini_fini = Conversions.parse(:hanyu,syll)
+          if ini_fini
+            p tone, syll, ini_fini
+            ini, fini = ini_fini.initial, ini_fini.final
+          end
+
+          return Syllable.new(ini, fini, tone) if tone && ini && fini
+        end
+      end
+
+       #     self.gsub('u:','ü').gsub(/[A-Za-züÜ]{1,5}\d/) do |m|
+       #Pinyin.HanyuWriter(:accents) << Pinyin.HanyuReader(:numbers).parse(m.downcase)
+       #end
+    end
+  end
+end

data/lib/{data → pinyin/data}/valid_pinyin.yaml

@@ -2,9 +2,8 @@
 # Hpricoted from http://en.wikipedia.org/wiki/Pinyin_table
 # Hand-edited for Ueng/weng which is under Ong in the table
 #
-# Note that, in order to only use 7-bit ASCII characters, 
-# the u with two dots (umlaut) is written v, and the e with
-# a little hat on top (circumflex) is written E.
+# TODO : This is actually wrong, Ong should be removed in favor
+#        of -ueng/weng
 ---
 V:
   Ne: nü

data/lib/{groundwork.rb → pinyin/groundwork.rb}

@@ -6,7 +6,11 @@
 #  * ILLEGAL_COMBINATIONS
 
 module Pinyin
+
+  #
   # A Chinese initial (start of a syllable)
+  #
+
   class Initial
     attr :name
     def initialize(n)
@@ -41,8 +45,10 @@ module Pinyin
     end
   end
 
-
+  #
   # A Chinese final (end of a syllable)
+  #
+  
   class Final
     attr :name
     def initialize(n)
@@ -72,8 +78,11 @@ module Pinyin
   end
 
 
+  #
   # Combination of an initial and a final
   # Not to be confused with a syllable that has the neutral tone
+  #
+
   class TonelessSyllable
     attr_accessor :initial, :final
 
@@ -98,7 +107,10 @@ module Pinyin
   end
 
 
+  #
   # Syllable : initial, final and tone
+  #
+
   class Syllable < TonelessSyllable
     attr_accessor :tone
 
@@ -114,9 +126,11 @@ module Pinyin
     alias :to_s :inspect
   end
 
-
+  #
   # Some groups of initials and finals may not be combined
   # This list is not exhaustive but is sufficient to resolve ambiguity
+  #
+
   ILLEGAL_COMBINATIONS=
     [               
      [Initial::Group_0, Final::Group_0],
@@ -135,14 +149,35 @@ module Pinyin
      [Initial::Group_1, Final::Group_V],
      [Initial::Group_3, Final::Group_V],
 
-     [Initial::Group_2, [Final::O]],  #Only bo, po, mo and fo are valid -o combinations
+     #2008.05.26 lo is also valid!
+     #[Initial::Group_2, [Final::O]],  #Only bo, po, mo and fo are valid -o combinations
      [Initial::Group_3, [Final::O]],
      [Initial::Group_4, [Final::O]],
      [Initial::Group_5, [Final::O]],
      [Initial::Group_6, [Final::O]],
 
-     [[Initial::Empty], [Final::Ong]] # Some say ong and ueng is actually the same final, zhuyin uses the same representation, but ueng only has standalone form weng
-
+     [[Initial::Empty], [Final::Ong]] 
+       # TODO: Ong is actually the same as Ueng, in Hanyu Pinyin : -ong or weng
     ]
 
+  class <<self
+
+
+    #
+    # Yields a block for any valid initial/final pair
+    #
+
+    def valid_combinations
+      require 'yaml'
+      inp = YAML::load(IO.read(File.join(File.dirname(__FILE__), 'data', 'valid_pinyin.yaml')))
+      inp.each do |final, initials|
+        final = Final.const_get(final)
+        initials.each do |initial, pinyin|
+          initial = Initial.const_get(initial)
+          yield(initial, final)
+        end
+      end
+    end
+  end
+                       
 end

data/lib/pinyin/string.rb

@@ -0,0 +1,14 @@
+class String
+  def pretty_tones
+    self.gsub('u:','ü').gsub(/[A-Za-züÜ]{1,5}\d/) do |m|
+      Pinyin.HanyuWriter(:accents) << Pinyin.HanyuReader(:numbers).parse(m.downcase)
+    end
+  end
+
+  def bpmf
+    self.gsub('u:','ü').scan(/[A-Za-züÜ]{1,5}\d/).map do |m|
+      Pinyin.ZhuyinWriter(:marks) << 
+        (Pinyin.HanyuReader(:numbers) << m.downcase)
+    end.join(' ')
+  end
+end

data/lib/{support.rb → pinyin/support.rb}

@@ -1,14 +1,10 @@
 class String
-  def camelize
-    self.split(/_/).map{|p|p.capitalize}.join
-  end
-
   def chars
     self.unpack('U*').map{|c| [c].pack('U')}
   end
 end
 
-class Object
+module Kernel
   def returning(s)
     yield(s)
     s