ting 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 832c9fb34c5415f06ee7a3c11063930e377ba9cf
-  data.tar.gz: 599204c5cec9e06962568922c4c477dd868911ba
+SHA256:
+  metadata.gz: 6e563c103a40efc2e917643626b84bd6f46f72ab054d9c8c090b1fca418f5226
+  data.tar.gz: 79b68c136dfc2bc4b6424f2cd8ac09f861f0ca4df6bd86f54fea24b6bf5c2da1
 SHA512:
-  metadata.gz: c35bc7b3cd2a40edb9fa89fe00089a1d3791eb640600a17b7de419beaa902e8047c20d24dda3d9fdb11346eeca7b9ecee3688d7010949edfabdc14a16187d11a
-  data.tar.gz: eb3868c3100089e6b9d55e71a7cb0d2f157a7a67ccf81e660422e285954897c3c09b3dd970a9264492823a762b64fdcd7a247fe84bd54ffed041dd1f78dce6c5
+  metadata.gz: 28b3a045d027467fd22f47c20835bcc1576809b40c6e239b0a40bfc965deb5b4c2ac554d4050758268b8b7bcf063b16a94aa87f85b4a568366824cab9ecf6695
+  data.tar.gz: 32f06a2130fb4d530ae63084142ceb82bb54047adaa9ea81cb61691401548342f62b54244e19529996c735c448cb414773c7d793c2859a5550ebcdbae167075f

data/CHANGELOG.md

@@ -9,16 +9,23 @@ All notable changes to this project will be documented in this file. This change
 
 ## [Unreleased]
 
+## [0.12.0] - released 2018-10-18
+
+### Changed
+- `Ting.pretty_tones` inserts apostrophes before syllables beginning with a, e, o
+- `Ting::HanyuPinyinParser` is better at splitting up compound words correctly
+
 ## [0.11.0] - released 2017-11-06
 
 ### Changed
 - Make `Ting.pretty_tones` work with bopomofo
-- Correct name of Palladius (Cyrilic transcription)
+- Correct name of Palladius (Cyrillic transcription)
 - Add missing IPA finals
 - Change Palladius transcription of "hui" to хуэй
 
 ### Added
 - `bin/ting_table` script
 
-[Unreleased]: https://github.com/lambdaisland/uri/compare/v0.11.0...HEAD
+[Unreleased]: https://github.com/lambdaisland/uri/compare/v0.12.0...HEAD
+[0.12.0]: https://github.com/plexus/ting/compare/v0.12.0...v0.11.0
 [0.11.0]: https://github.com/plexus/ting/compare/v0.11.0...v0.10.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ting (0.10.0)
+    ting (0.12.0)
 
 GEM
   remote: https://rubygems.org/
@@ -239,4 +239,4 @@ DEPENDENCIES
   ting!
 
 BUNDLED WITH
-   1.15.4
+   1.16.6

data/lib/ting.rb

@@ -50,15 +50,30 @@ module Ting
       )
     end
 
+    # The longest syllables are six letters long (chuang, shuang, zhuang).
+    SYLLABLE_REGEXP = /[A-Za-züÜ]{1,6}\d?/
 
     def pretty_tones(string)
-      string.gsub('u:','ü').gsub(/[A-Za-züÜ]{1,7}\d?/) do |syll|
-        SYLLABLE_CACHE[syll]
+      string = string.gsub('u:', 'ü') # (note that this implicitly dups the string)
+      # Scan through the string, replacing syllable by syllable.
+      pos = 0
+      while match = string.match(SYLLABLE_REGEXP, pos)
+        syllable = match[0]
+        replacement = SYLLABLE_CACHE[syllable]
+        match_pos = match.begin(0)
+        # If this syllable starts with a vowel and is preceded by a letter (not whitespace or
+        # control characters), insert an apostrophe before it.
+        if match_pos > 0 && string[match_pos - 1] =~ /[[:alpha:]]/ && syllable =~ /^[AEOaoe]/
+          replacement = "'" + replacement
+        end
+        string[match_pos, syllable.length] = replacement
+        pos = match_pos + replacement.length
       end
+      string
     end
 
     def bpmf(string)
-      string.gsub('u:','ü').scan(/[A-Za-züÜ]{1,7}\d?/).map do |m|
+      string.gsub('u:', 'ü').scan(SYLLABLE_REGEXP).map do |m|
         Ting.writer(:zhuyin, :marks).(
           Ting.reader(:hanyu, :numbers).(m.downcase)
         )

data/lib/ting/groundwork.rb

@@ -111,6 +111,8 @@ module Ting
     alias :to_s :inspect
 
     def ==( other )
+      return false unless other.is_a? Syllable
+
       [ other.initial, other.final, other.tone, other.capitalized ] ==
         [ self.initial, self.final, self.tone, self.capitalized ]
     end

data/lib/ting/hanyu_pinyin_parser.rb

@@ -14,26 +14,59 @@ module Ting
       @all_syllables ||= Ting.all_syllables.map(&hanyu_writer).sort_by(&:length).reverse
     end
 
-    def sylls_with_erhua
-      @with_erhua ||= all_syllables.map{|p| p + 'r'}
+    def consonant_syllables
+      @consonant_syllables ||= all_syllables.grep(/^[bcdfghjklmnpqrstwxyz]/i)
     end
 
     def pinyin_regexp
-      @pinyin_regexp ||= Regexp.union(*sylls_with_erhua, *all_syllables)
+      # This will parse a cluster of pinyin, i.e. an uninterrupted string of pinyin characters without punctuation.
+      @pinyin_cluster_regexp ||= /\A
+        # Every syllable can appear at the start of a cluster.
+        (#{Regexp.union(all_syllables)})
+        # However, only syllables starting with a consonant can follow, as syllables starting with a vowel have to
+        # be prefixed with an apostrophe.
+        # Since it is common to omit the apostrophe when there is no ambiguity, also allow syllables starting with
+        # a vowel after all letters except n and g, and after -ong, since -on does not appear at the end of a valid
+        # syllable.
+        (#{Regexp.union(consonant_syllables)}|(?<=[^ng]|[ōóǒòo]ng)#{Regexp.union(all_syllables)})*
+        (r)?
+        \Z/x
     end
 
-    def split_pinyin(pinyin)
-      pinyin.scan(pinyin_regexp).flat_map do |syll|
-        if sylls_with_erhua.include?(syll) && ! all_syllables.include?(syll)
-          [ syll[0..-2], 'er']
-        else
-          [ syll ]
+    def pinyin_separator_regexp
+      # A regular expression that matches every character that can *not* appear in pinyin.
+      @pinyin_separator_regexp ||= Regexp.new("[^#{all_syllables.join.downcase.split("").sort.uniq.join}]+")
+    end
+
+    def parse_cluster(pinyin)
+      syllables = []
+
+      # Chop off one syllable at a time from the end by continuously matching the same regular expression.
+      # This ensures the pinyin will be split into only valid pinyin syllables. Because a match capture will
+      # only contain the *last* content it has matched, we have to use a loop.
+      while match = pinyin_regexp.match(pinyin)
+        # If an 'r' at the end was matched, this implies that all other parts of the string were matched as
+        # syllables, and this cluster uses erhua.
+        if 'r' == match[3]
+          syllables << 'er'
+          pinyin = pinyin.chop
         end
+        last_syllable = match[2] || match[1]
+        syllables << last_syllable
+        pinyin = pinyin[0, pinyin.length - last_syllable.length]
       end
+
+      raise ArgumentError, "Unparseable pinyin fragment encountered: #{pinyin}" if !pinyin.empty?
+
+      syllables.reverse
     end
 
     def parse(pinyin)
-      split_pinyin(pinyin).map(&hanyu_reader)
+      # hanyu_reader cannot parse uppercase pinyin.
+      pinyin = pinyin.downcase
+
+      clusters = pinyin.split(pinyin_separator_regexp)
+      clusters.flat_map {|cluster| parse_cluster(cluster)}.flat_map(&hanyu_reader)
     end
     alias call parse
 

data/lib/ting/tones/accents.rb

@@ -27,7 +27,7 @@ module Ting
         when /a/
           syll.sub(/a/, tone_glyph(:a,tone))
         when /e/
-          syll.sub(/e/, tone_glyph(:e,tone))
+          syll.sub(/e/, tone_glyph(:e,tone)).sub('v', 'ü')
         when /o/
           syll.sub(/o/, tone_glyph(:o,tone))
         when /(i|u|v)\z/

data/lib/ting/version.rb

@@ -1,3 +1,3 @@
 module Ting
-  VERSION = '0.11.0'
+  VERSION = '0.12.0'
 end

data/spec/hanyu_pinyin_parser_spec.rb

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Ting::HanyuPinyinParser do
+  let(:parser) { Ting::HanyuPinyinParser.new }
+
+  it 'should be able to parse boring characters' do
+    pinyin = "xíbié de hǎi'àn"
+    expect(parser.parse(pinyin)).to eq([
+      Ting::Syllable.new( Ting::Initial::Xi,    Ting::Final::I,  2 ),
+      Ting::Syllable.new( Ting::Initial::Bo,    Ting::Final::Ie, 2 ),
+      Ting::Syllable.new( Ting::Initial::De,    Ting::Final::E,  5 ),
+      Ting::Syllable.new( Ting::Initial::He,    Ting::Final::Ai, 3 ),
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::An, 4 ),
+    ])
+  end
+
+  it 'should be able to parse erhua' do
+    pinyin = "wèir Wèir"
+    expect(parser.parse(pinyin)).to eq([
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Ui, 4 ),
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Er, 5 ),
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Ui, 4 ),
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Er, 5 ),
+    ])
+  end
+
+  it 'should be able to discern erhua from other Ri syllables' do
+    pinyin = Ting.pretty_tones 'yang2rou4'
+    expect(parser.parse(pinyin)).to eq([
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Iang, 2 ),
+      Ting::Syllable.new( Ting::Initial::Ri,    Ting::Final::Ou,   4 ),
+    ])
+
+    pinyin = Ting.pretty_tones 'sui1ran2'
+    expect(parser.parse(pinyin)).to eq([
+      Ting::Syllable.new( Ting::Initial::Si, Ting::Final::Ui, 1 ),
+      Ting::Syllable.new( Ting::Initial::Ri, Ting::Final::An, 2 ),
+    ])
+  end
+
+  it 'should parse er4 and her2 correctly' do
+    expect(parser.parse('èr')).to eq([
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Er, 4 ),
+    ])
+    expect(parser.parse('hér')).to eq([
+      Ting::Syllable.new( Ting::Initial::He,    Ting::Final::E,  2 ),
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Er, 5 ),
+    ])
+  end
+
+  it 'should parse ou1zhou1 correctly' do
+    pinyin = Ting.pretty_tones('ou1zhou1')
+    expect(parser.parse(pinyin)).to eq([
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Ou, 1 ),
+      Ting::Syllable.new( Ting::Initial::Zhi,   Ting::Final::Ou, 1 ),
+    ])
+  end
+
+  it 'should parse sheng3lve4 correctly' do
+    expect(parser.parse('shěnglüè')).to eq([
+      Ting::Syllable.new( Ting::Initial::Shi, Ting::Final::Eng, 3 ),
+      Ting::Syllable.new( Ting::Initial::Le,  Ting::Final::Ue,  4 ),
+    ])
+  end
+
+  it 'should parse regardless of apostrophes and weird whitespace' do
+    pinyin = "Xī'ān\thǎowánr\tma?\nHǎowánr!"
+    expect(parser.parse(pinyin).map(&:tone)).to eq([1, 1, 3, 2, 5, 5, 3, 2, 5])
+  end
+
+  it 'should parse ambiguous syllables based on context' do
+    pinyin = 'gūnánguǎnǚ'
+    expect(parser.parse(pinyin)).to eq([
+      Ting::Syllable.new( Ting::Initial::Ge, Ting::Final::U,  1 ),
+      Ting::Syllable.new( Ting::Initial::Ne, Ting::Final::An, 2 ),
+      Ting::Syllable.new( Ting::Initial::Ge, Ting::Final::Ua, 3 ),
+      Ting::Syllable.new( Ting::Initial::Ne, Ting::Final::V,  3 ),
+    ])
+
+    pinyin = 'yángròu'
+    expect(parser.parse(pinyin)).to eq([
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Iang, 2 ),
+      Ting::Syllable.new( Ting::Initial::Ri,    Ting::Final::Ou,   4 ),
+    ])
+  end
+  
+  it 'should parse some invalid pinyin (missing apostrophe)' do
+    # Syllables that begin with [aeo] must be prefixed with an apostrophe in the middle of the word.
+    # Ref.: https://en.wikipedia.org/wiki/Pinyin#Pronunciation_of_initials, "Note on the apostrophe"
+    # Still, Ting should be able to parse these syllables if they follow unambiguous characters.
+
+    expect(parser.parse('hǎiàn')).to eq([
+      Ting::Syllable.new( Ting::Initial::He,    Ting::Final::Ai, 3 ),
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::An, 4 ),
+    ])
+
+    expect(parser.parse('mòshuǐer')).to eq([
+      Ting::Syllable.new( Ting::Initial::Mo,    Ting::Final::O,  4 ),
+      Ting::Syllable.new( Ting::Initial::Shi,   Ting::Final::Ui, 3 ),
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::Er, 5 ),
+    ])
+
+    expect(parser.parse('gōngānjú')).to eq([
+      Ting::Syllable.new( Ting::Initial::Ge,    Ting::Final::Ong, 1 ),
+      Ting::Syllable.new( Ting::Initial::Empty, Ting::Final::An,  1 ),
+      Ting::Syllable.new( Ting::Initial::Ji,    Ting::Final::V,   2 ),
+    ])
+  end
+end

data/spec/ting_spec.rb

@@ -18,7 +18,18 @@ describe Ting do
   end
   
   it 'should parse syllables correctly' do
-    expect(Ting.pretty_tones('wo3 de peng2you3 hen3 zhuang4')).to eq('wǒ de péngyǒu hěn zhuàng')
-    expect(Ting.bpmf('wo3 de peng2you3 hen3 zhuang4')).to eq('ㄨㄛˇ ㄉㄜ˙ ㄆㄥˊ ㄧㄡˇ ㄏㄣˇ ㄓㄨㄤˋ')
+    expect(Ting.pretty_tones('Wo3 de Ou1zhou1 peng2you3 hen3 zhuang4')).to eq('wǒ de ōuzhōu péngyǒu hěn zhuàng')
+    expect(Ting.bpmf('Wo3 de peng2you3 hen3 zhuang4')).to eq('ㄨㄛˇ ㄉㄜ˙ ㄆㄥˊ ㄧㄡˇ ㄏㄣˇ ㄓㄨㄤˋ')
+  end
+
+  it 'should be able to pretty-print simple strings' do
+    expect(Ting.pretty_tones('wo3 ai4 ni3')).to eq('wǒ ài nǐ')
+    expect(Ting.pretty_tones('you3dian3r hao3xiao4')).to eq('yǒudiǎnr hǎoxiào')
+  end
+
+  it 'should insert apostrophes when appropriate' do
+    expect(Ting.pretty_tones('hai3an4')).to eq("hǎi'àn")
+    expect(Ting.pretty_tones('ding4e2')).to eq("dìng'é")
+    expect(Ting.pretty_tones('an5an5an5an5an')).to eq("an'an'an'an'an")
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ting
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Arne Brasseur
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-06 00:00:00.000000000 Z
+date: 2018-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -104,6 +104,7 @@ files:
 - lib/ting/tones/supernum.rb
 - lib/ting/version.rb
 - lib/ting/writer.rb
+- spec/hanyu_pinyin_parser_spec.rb
 - spec/jruby_csv_spec.rb
 - spec/palladius_spec.rb
 - spec/spec_helper.rb
@@ -132,12 +133,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A conversion library for Chinese transcription methods like Hanyu Pinyin,
   Bopomofo and Wade-Giles.
 test_files:
+- spec/hanyu_pinyin_parser_spec.rb
 - spec/jruby_csv_spec.rb
 - spec/palladius_spec.rb
 - spec/spec_helper.rb