formosa 0.0.1

data/lib/ext/native_syllable_composer/LibHolo.h

@@ -0,0 +1,1223 @@
+// LibHolo.h: LibFormosa's Holo Processing Library
+//
+// Copyright (c) 2007 The OpenVanilla Project (http://openvanilla.org)
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 
+// 1. Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+// 3. Neither the name of OpenVanilla nor the names of its contributors
+//    may be used to endorse or promote products derived from this software
+//    without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include <vector>
+#include <string>
+#include <iostream>
+#include <ctype.h>
+
+#include "HoloVowels.h"
+
+namespace LibHolo
+{
+    using namespace std;
+    
+    class Composable
+    {
+    public:
+        virtual ~Composable() {}
+        virtual void clear()=0;
+        virtual bool empty()=0;
+        virtual unsigned int numberOfCodepoints()=0;
+        virtual string composedForm()=0;
+    }; 
+
+    class ComposableStringBuffer : Composable {
+    public:
+        ComposableStringBuffer() : _cursor(0)
+        {
+        }
+        
+        virtual unsigned int cursor()
+        {
+            return _cursor;
+        }
+        
+        virtual unsigned int setCursor(unsigned int c)
+        {
+            if (c <= numberOfCodepoints()) _cursor = c;
+            return _cursor;
+        }
+        
+        virtual void clear()
+        {
+            _cursor = 0;
+            strvec.clear();
+        }
+        
+        virtual bool empty()
+        {
+            return strvec.empty();
+        }
+            
+        virtual unsigned int numberOfCodepoints()
+        {
+            return strvec.size();
+        }
+     
+        virtual bool insertCharacterAt(unsigned int i, char c)
+        {
+            if (i > numberOfCodepoints()) return false;
+            strvec.insert(strvec.begin()+i, string(1, c));
+            return true;
+        }
+        
+        virtual bool removeCodepointAt(unsigned int i)
+        {
+            if (i >= numberOfCodepoints()) return false;
+            strvec.erase(strvec.begin() + i);
+            return true;
+        }
+        
+        virtual string composedForm()
+        {
+            return internalForm();
+        }
+        
+        virtual string internalForm()
+        {
+            string newstr;
+            unsigned int s=numberOfCodepoints();
+            for (unsigned int i=0; i<s; i++) newstr+=strvec[i];
+            return newstr;
+        }
+        
+    protected:
+        vector<string> strvec;
+        unsigned int _cursor;
+    };
+    
+    
+    enum SyllableType
+    {
+        POJSyllable = 0,
+        TLSyllable = 1,
+        TLPASyllable = 2,
+        DTSyllable = 3
+    };
+    
+    enum DiacriticInputOption
+    {
+        DiacriticGivenBeforeVowel = 0,
+        DiacriticGivenAfterVowel = 1
+    };
+
+	class HoloSymbol
+	{
+	public:
+		HoloSymbol() : _tone(0), _type(POJSyllable)
+		{
+		}
+		
+		HoloSymbol(const string &s, SyllableType t) : _tone(0), _type(t), _symbol(s)
+		{
+		}
+		
+		HoloSymbol(const HoloSymbol &s) : _tone(s._tone), _type(s._type), _symbol(s._symbol)
+		{
+		}
+		
+		void setType(SyllableType t)
+		{
+			_type = t;
+		}
+		
+		const HoloSymbol& operator=(const HoloSymbol &s)
+		{
+			_symbol = s._symbol;
+			_tone = s._tone;
+			_type = s._type;
+			return *this;
+		}
+		
+		string symbol()
+		{
+			return string(_symbol);
+		}
+		
+		string symbolInLowerCase()
+		{
+			string lower;
+			unsigned int s=_symbol.length();
+			for (unsigned int i=0; i<s; i++) lower+=tolower(_symbol[i]);
+			return lower;
+		}
+		
+		string setSymbol(const string& s)
+		{
+			return (_symbol = s);
+		}
+		
+		string composedForm(bool forcePOJStyle=false)
+		{
+			string composed = ComposeHoloVowel(_symbol, _tone, ((_type==POJSyllable) || forcePOJStyle) ? true : false);
+			if (!composed.length()) return _symbol;
+			return composed;
+		}
+		
+		unsigned int composedLength()
+		{
+			string composed = composedForm();
+			unsigned int len = 0, clen = composed.length();
+			for (unsigned int i=0; i<clen; )
+			{
+				if (!(composed[i] & 0x80)) {
+					len++;
+					i++;
+				}
+				else if ((composed[i] & 0xe0) == 0xc0) {
+					len++;
+					i+=2;
+				}
+				else if ((composed[i] & 0xf0) == 0xe0) {
+					len++;
+					i+=3;
+				}
+				else {
+					len++;
+					i+=4;
+				}
+			}
+			
+			// fprintf (stderr, "composed=%s, strlen=%d, calculated len=%d\n", composed.c_str(), clen, len);
+			
+			return len;
+		}
+		
+		unsigned int tone()
+		{
+			return _tone;
+		}
+		
+		unsigned int setTone(unsigned int t)
+		{
+			_tone = t > 9 ? _tone : t;
+			return _tone;
+		}
+		
+		bool isUpperCase()
+		{
+		  if (!_symbol.length()) return false;
+		  return toupper(_symbol[0]) == _symbol[0];
+		}
+		
+		
+	protected:
+		unsigned int _tone;
+		SyllableType _type;
+		string _symbol;
+	};
+
+
+	// the _cursor in HoloSyllable really means "the internal symbol cursor",
+	// as for the display cursor (called by the input method context), 
+	// we need to recalculate
+	class HoloSyllable : public Composable
+    {
+    public:
+		HoloSyllable() : _inputType(POJSyllable), _inputOption(DiacriticGivenBeforeVowel), 
+						 _forcePOJStyle(false),
+					     _cursor(0), _preparedTone(0)
+		{
+		}
+
+		HoloSyllable(const HoloSyllable &s) : _inputType(s._inputType),
+						 _inputOption(s._inputOption),
+						 _forcePOJStyle(s._forcePOJStyle),
+						 _symvec(s._symvec),
+					     _cursor(s._cursor), _preparedTone(s._preparedTone) 
+		{
+		}
+		
+		const HoloSyllable& operator=(const HoloSyllable &s)
+		{
+			_inputType = s._inputType;
+			_inputOption = s._inputOption;
+			_forcePOJStyle = s._forcePOJStyle;
+			_symvec = s._symvec;
+			_cursor = s._cursor;
+			_preparedTone = s._preparedTone;
+			return *this;
+		}		
+		
+		virtual void setInputType(SyllableType t)
+		{
+			_inputType = t;
+		}
+		
+		virtual void setInputOption(DiacriticInputOption o)
+		{
+			if (o != _inputOption) clearPreparedTone();
+			_inputOption = o;
+		}
+		
+		virtual void setForcePOJStyle(bool p)
+		{
+			_forcePOJStyle = p;
+		}
+		
+        virtual void clear()
+		{
+			_symvec.clear();
+			_cursor = 0;
+			_preparedTone = 0;
+		}
+		
+        virtual bool empty()
+		{
+			return _symvec.empty();
+		}
+		
+        virtual unsigned int numberOfCodepoints()
+		{
+			return _symvec.size();
+		}
+		
+        virtual string composedForm()
+		{
+			unsigned int s = _symvec.size();
+			string composed;
+			unsigned int i;
+			
+			if (_preparedTone) _cursor--;
+			
+			for (i=0; i<_cursor; i++) 
+			{
+				composed += _symvec[i].composedForm(_forcePOJStyle);
+				// fprintf(stderr, "%d, symbol=%s, composed=%s, composd form=%s\n", i, _symvec[i].symbol().c_str(), _symvec[i].composedForm().c_str(), composed.c_str());
+			}
+			
+			composed += GetToneASCIIRepresentation(_preparedTone);
+			// fprintf(stderr, "composd form=%s\n", composed.c_str());
+			
+			for (; i<s; i++)
+			{
+				composed += _symvec[i].composedForm(_forcePOJStyle);
+				// fprintf(stderr, "composd form=%s\n", composed.c_str());
+			}
+
+			if (_preparedTone) _cursor++;
+
+			return composed;
+		}
+
+		void setCursor(unsigned int c)
+		{
+			clearPreparedTone();
+			_cursor = c;
+		}
+		
+		unsigned int cursor()
+		{
+			unsigned int realcursor = _preparedTone ? _cursor-1 : _cursor;			
+			unsigned codepointCursor=0;
+			for (unsigned int i=0; i<realcursor; i++) codepointCursor+=_symvec[i].composedLength();
+			
+			if (_preparedTone) codepointCursor++;
+			
+			return codepointCursor;
+		}
+		
+		bool cursorHome()
+		{
+			clearPreparedTone();
+			if (_cursor==0) return false;
+			_cursor=0;
+			return true;
+		}
+		
+		bool cursorEnd()
+		{
+			clearPreparedTone();
+			unsigned int len = numberOfCodepoints();
+			if (_cursor == len) return false;
+			_cursor = len;
+			return true;
+		}
+		
+		bool cursorLeft()
+		{
+			clearPreparedTone();
+			if (_cursor==0) return false;
+			_cursor--;
+			return true;
+		}
+		
+		bool cursorRight()
+		{
+			clearPreparedTone();
+			if (_cursor == numberOfCodepoints()) return false;
+			_cursor++;
+			return true;
+		}
+		
+		bool insertSymbolAtCursor(const HoloSymbol &s)
+		{
+			clearPreparedTone();
+			HoloSymbol newsym(s);
+			newsym.setType(_inputType);
+			_symvec.insert(_symvec.begin() + _cursor, newsym);
+			_cursor++;			
+			
+			return true;			
+		}
+		
+		// if there is a prepared tone, the given tone parameter will be ignored
+		bool insertCharacterAtCursor(char c, unsigned int tone=0)
+		{
+			// fprintf(stderr, "insert char %d ('%c'), cursor=%d\n", c, c, _cursor);
+			if (IsDiacriticSymbol(c))
+			{
+				unsigned int tone = ToneFromDiacriticSymbol(c);
+				
+				// if there's already a prepared tone, we replace it with the current one
+				if (_preparedTone) {
+					_preparedTone = tone;
+					return true;
+				}
+								
+				if (_inputOption==DiacriticGivenBeforeVowel) {
+					_preparedTone = tone;
+					_cursor++;					
+				}
+				else {
+					// diacritic given after vowel
+					if (hasPreviousSymbolAtCursor()) previousSymbolAtCursor().setTone(tone);
+				}
+				return true;
+			}
+			
+			// if it's not a diacritic symbol, it's POJ^W^W^W, and it's n or u or g,
+			// (and if there's no prepared tone!)
+			// we need to do something special...
+			if (!IsDiacriticSymbol(c) && !_preparedTone /* && _inputType==POJSyllable */)
+			{
+				if (hasPreviousSymbolAtCursor())
+				{
+					string prev = previousSymbolAtCursor().symbolInLowerCase();
+					
+					// N -> nn only works if the first character of the syllable is not an
+					// all uppercase symbol
+					if (c=='N' && ((prev != "n") && (prev != "nn")) && _inputType == POJSyllable
+					   && (_symvec.size() > 0 && !_symvec[0].isUpperCase()))
+					{
+						// insert two n's in a row
+						_symvec.insert(_symvec.begin() + _cursor, HoloSymbol(string("nn"), _inputType));
+						_cursor++;
+						return true;
+					}
+					else if (tolower(c)=='n' && prev=="n") {
+						previousSymbolAtCursor().setSymbol(previousSymbolAtCursor().symbol() + string(1, c));
+						return true;
+					}
+					else if (_inputType == POJSyllable && tolower(c)=='u' && prev=="o") {
+						previousSymbolAtCursor().setSymbol(previousSymbolAtCursor().symbol() + string(1, c));
+						return true;
+					}
+					else if (_inputType == TLSyllable && tolower(c)=='o' && prev=="o") {
+						previousSymbolAtCursor().setSymbol(previousSymbolAtCursor().symbol() + string(1, c));
+						return true;
+					}
+					else if (tolower(c)=='g' && prev=="nn") {
+						// we need to break them up!
+						string before = previousSymbolAtCursor().symbol();
+						
+						// and the tone of the previous symbol (when it's combined into nn) will be retained
+						previousSymbolAtCursor().setSymbol(before.substr(0, 1));
+						
+						// insert one n and one g
+						_symvec.insert(_symvec.begin() + _cursor, HoloSymbol(before.substr(1,1), _inputType));
+						_cursor++;
+						_symvec.insert(_symvec.begin() + _cursor, HoloSymbol(string(1, c), _inputType));
+						_cursor++;
+						return true;
+					}
+				}
+			}
+
+			
+			HoloSymbol s(string(1, c), _inputType);
+			if (_preparedTone)
+			{
+				_cursor--;
+				s.setTone(_preparedTone);
+				_preparedTone = 0;
+			}
+			else if (tone > 1)
+			{
+				s.setTone(tone);
+			}
+			
+			_symvec.insert(_symvec.begin() + _cursor, s);
+			_cursor++;			
+			
+			return true;
+		}
+		
+		bool removeCharacterAtRightOfCursor()	// backspace
+		{
+			if (_preparedTone)
+			{
+				clearPreparedTone();
+				return true;
+			}
+			
+			if (atBeginning()) return false;
+			_cursor--;
+			_symvec.erase(_symvec.begin() + _cursor);
+			return true;
+		}
+		
+		bool removeCharacterAtLeftOfCursor()	// delete
+		{
+			// we do some tightrope trick here: if we have _preparedTone ready,
+			// we "push back" the real _cursor position, do the delete thing,
+			// then push it back
+			bool retval=true;
+			if (_preparedTone) _cursor--;
+			if (atEnd()) retval=false; else _symvec.erase(_symvec.begin() + _cursor);
+			if (_preparedTone) _cursor++;
+			return retval;
+		}
+	
+		// returns a normalized string that represents the "internal form" for querying the database
+		// implies normalization
+		string normalizedQueryData(unsigned int finalTone=0)
+		{
+			HoloSyllable s(*this);
+			s.normalize(finalTone);
+			string query;
+			
+			unsigned int size=s._symvec.size();
+			unsigned int loudest = 0;
+			
+			for (unsigned int i=0; i<size; i++) {
+				query = query + s._symvec[i].symbol();
+				if (s._symvec[i].tone() > 1) loudest = s._symvec[i].tone();
+				// fprintf (stderr, "combining query data %s, tone %d\n", s._symvec[i].symbol().c_str(), s._symvec[i].tone());
+			}
+			
+			if (loudest > 1) query = query + string(1, loudest+'0');
+			return query;
+		}
+	
+		// normalization is an "identpotent" function, ie. the result should
+		// be the same no matter how many times you call it--this being a very
+		// important linguistic characteristic of this function	
+		void normalize(unsigned int finalTone=0)
+		{
+			// fprintf (stderr, "input finalTone=%d\n", finalTone);
+			unsigned int end = _symvec.size();
+			
+			// if it's empty, just return
+			if (!end) return;
+			
+			unsigned int loudestVowel = end;
+			unsigned int loudestTone = 0;
+			unsigned int p;
+			
+			// find the loudest vowel
+			#define FLV(x) ((p=findSymbol(x)) != end)
+			#define SETLOUDEST(v) do { loudestVowel = v; if (_symvec[loudestVowel].tone()>1) { loudestTone = _symvec[loudestVowel].tone(); } } while(0)
+
+	
+			if (end==1 && _symvec[0].symbolInLowerCase()=="m") SETLOUDEST(0);
+			if (FLV("n")) SETLOUDEST(p);
+			if (FLV("m")) SETLOUDEST(p);
+
+			// see if it's ng
+			if ((p=findSymbolPair("n", "g")) != end)
+				SETLOUDEST(p);
+
+			if (FLV("u")) SETLOUDEST(p);
+			if (FLV("i")) SETLOUDEST(p);
+			if (FLV("o")) SETLOUDEST(p);
+			if (FLV("e")) SETLOUDEST(p);
+			if (FLV("ou")) SETLOUDEST(p);
+			if (FLV("oo")) SETLOUDEST(p);
+			if (FLV("a")) SETLOUDEST(p);
+			
+			// the last "ere" override
+			if (end >= 3) {
+                if (_symvec[end-1].symbolInLowerCase() == "e" && _symvec[end-2].symbolInLowerCase() == "r" && _symvec[end-3].symbolInLowerCase() == "e")
+                {
+                    SETLOUDEST(end-1);
+                }
+			}
+			
+			if (loudestVowel==end) return;
+			// fprintf(stderr, "found loudest vowel=%d (%s), loudest tone=%d\n", loudestVowel, _symvec[loudestVowel].symbol().c_str(), loudestTone);
+			
+			// finalTone overrides
+			if (finalTone > 1) loudestTone = finalTone;
+			
+			for (unsigned int i=0; i<end; i++) _symvec[i].setTone(0);
+			
+			string lastSymbolStr = _symvec[end-1].symbolInLowerCase();
+			
+			// if the symbol is "i", and there's a next "u", we shift
+			// the vowel to "u"
+			
+			if (_symvec[loudestVowel].symbolInLowerCase()=="i")
+			{
+				if (loudestVowel+1 < end)
+				{
+					if (_symvec[loudestVowel+1].symbolInLowerCase()=="u") loudestVowel++;
+				}
+			}
+
+			if (loudestTone==4 || /* loudestTone==6 || */ loudestTone <= 1) {
+				// ignore the 4th, 6th and 1th (or no tone), so everything is set to 0 now
+				return;
+			}
+			
+			if (lastSymbolStr=="t" || lastSymbolStr=="p" || lastSymbolStr=="k" || lastSymbolStr=="h") {
+				// only when the ending is t, p, k, h is the tone set -- and only when the tone is 8
+				if (loudestTone==8) _symvec[loudestVowel].setTone(loudestTone);
+				return;
+			}
+			else {
+			     // if not t,p,k,h, we need to override the loudest tone--back to tone 1 !
+				if (loudestTone==8) {
+				    _symvec[loudestVowel].setTone(0);
+    				return;
+                }
+			}
+			
+			_symvec[loudestVowel].setTone(loudestTone);
+			
+			#undef FLV
+			#undef SETTONE
+		}
+		
+		HoloSyllable convertToPOJSyllable()
+		{
+			HoloSyllable syl = *this;
+			syl.clearPreparedTone();
+			syl.setCursor(0);
+			if (_inputType==POJSyllable) return syl;
+			
+			syl.setInputType(POJSyllable);
+			syl.clear();
+			
+			// begin TL->POJ conversion
+			unsigned int size=_symvec.size();
+			unsigned int i;
+			
+			for (i=0; i<size; i++)
+			{
+				HoloSymbol sym1 = _symvec[i];
+				string str1 = sym1.symbol();
+				
+				// fprintf (stderr, "converting to POJ: %s\n", str1.c_str());
+				
+				string lowstr1 = sym1.symbolInLowerCase();
+				
+				// oo -> ou
+				if (lowstr1=="oo")
+				{
+					// detect case
+					if (str1[0] == tolower(str1[0])) {
+						syl.insertCharacterAtCursor('o', sym1.tone());
+						syl.insertCharacterAtCursor('u');
+					}
+					else
+					{
+						syl.insertCharacterAtCursor('O', sym1.tone());
+						syl.insertCharacterAtCursor('U');						
+					}
+					continue;
+				}
+				
+				
+				if (hasNextSymbol(i)) {
+					HoloSymbol sym2 = _symvec[i+1];
+					string str2 = sym2.symbol();
+					string lowstr2 = sym2.symbolInLowerCase();
+					
+					// ts -> ch with case detection
+					if (lowstr1=="t" && lowstr2=="s") {
+						// detect case
+						if (str1[0] == tolower(str1[0])) {
+							syl.insertCharacterAtCursor('c');
+							syl.insertCharacterAtCursor('h');
+						}
+						else {
+							syl.insertCharacterAtCursor('C');
+							syl.insertCharacterAtCursor('H');							
+						}
+						
+						i++;
+						continue;
+					}
+										
+					// ue -> oe
+					if (lowstr1=="u" && lowstr2=="e") {
+						// detect case
+						if (str1[0] == tolower(str1[0])) {
+							syl.insertCharacterAtCursor('o', sym1.tone());
+							syl.insertCharacterAtCursor('e', sym2.tone());
+						}
+						else {
+							syl.insertCharacterAtCursor('O', sym1.tone());
+							syl.insertCharacterAtCursor('E', sym2.tone());							
+						}
+						
+						i++;
+						continue;
+					}
+								
+					// ua -> oa
+					if (lowstr1=="u" && lowstr2=="a") {
+						// detect case
+						if (str1[0] == tolower(str1[0])) {
+							syl.insertCharacterAtCursor('o', sym1.tone());
+							syl.insertCharacterAtCursor('a', sym2.tone());
+						}
+						else {
+							syl.insertCharacterAtCursor('O', sym1.tone());
+							syl.insertCharacterAtCursor('A', sym2.tone());
+						}
+						
+						i++;
+						continue;
+					}
+
+					// ik -> ek (at ending)
+					if (lowstr1=="i" && lowstr2=="k" && (i+2)==size) {
+						// detect case
+						if (str1[0] == tolower(str1[0])) {
+							syl.insertCharacterAtCursor('e', sym1.tone());
+							syl.insertCharacterAtCursor('k', sym2.tone());
+						}
+						else {
+							syl.insertCharacterAtCursor('E', sym1.tone());
+							syl.insertCharacterAtCursor('K', sym2.tone());
+						}
+						
+						i++;
+						continue;
+					}
+					
+					if (hasNextNextSymbol(i) && (i+3)==size) {
+						HoloSymbol sym3 = _symvec[i+2];
+						string str3 = sym3.symbol();
+						string lowstr3 = sym3.symbolInLowerCase();
+
+						// ing -> eng (must be ending)					
+						if (lowstr1=="i" && lowstr2=="n" && lowstr3=="g") {
+							// detect case
+							if (str1[0] == tolower(str1[0])) {
+								syl.insertCharacterAtCursor('e', sym1.tone());
+								syl.insertCharacterAtCursor('n', sym2.tone());
+								syl.insertCharacterAtCursor('g', sym3.tone());
+							}
+							else {
+								syl.insertCharacterAtCursor('E', sym1.tone());
+								syl.insertCharacterAtCursor('N', sym2.tone());
+								syl.insertCharacterAtCursor('G', sym3.tone());
+							}
+
+							i+=2;
+							continue;
+						}
+						
+						// ouh -> oh (ending)
+						if (lowstr1=="o" && lowstr2=="u" && lowstr3=="h") {
+							// detect case
+							if (str2[0] == tolower(str2[0])) {
+								syl.insertCharacterAtCursor('o', sym1.tone());
+								syl.insertCharacterAtCursor('h', sym2.tone());
+							}
+							else {
+								syl.insertCharacterAtCursor('O', sym1.tone());
+								syl.insertCharacterAtCursor('H', sym2.tone());
+							}
+
+							i+=2;
+							continue;
+						}
+					}			
+				}
+
+				syl.insertSymbolAtCursor(sym1);
+			}			
+			
+			return syl;
+		}
+		
+		HoloSyllable convertToTLSyllable()
+		{
+			HoloSyllable syl = *this;
+			syl.clearPreparedTone();
+			syl.setCursor(0);
+			if (_inputType==TLSyllable) return syl;
+
+			syl.setInputType(TLSyllable);			
+			syl.clear();
+			
+			// begin POJ->TL conversion
+			unsigned int size=_symvec.size();
+			unsigned int i;
+			
+			for (i=0; i<size; i++)
+			{
+				HoloSymbol sym1 = _symvec[i];
+				string str1 = sym1.symbol();
+				string lowstr1 = sym1.symbolInLowerCase();
+				
+				// ou -> oo
+				if (lowstr1=="ou")
+				{
+					// detect case
+					if (str1[0] == tolower(str1[0])) {
+						syl.insertCharacterAtCursor('o', sym1.tone());
+						syl.insertCharacterAtCursor('o');
+					}
+					else
+					{
+						syl.insertCharacterAtCursor('O', sym1.tone());
+						syl.insertCharacterAtCursor('O');						
+					}
+					continue;
+				}
+				
+				
+				if (hasNextSymbol(i)) {
+					HoloSymbol sym2 = _symvec[i+1];
+					string str2 = sym2.symbol();
+					string lowstr2 = sym2.symbolInLowerCase();
+					
+					// ch -> ts with case detection
+					if (lowstr1=="c" && lowstr2=="h") {
+						// detect case
+						if (str1[0] == tolower(str1[0])) {
+							syl.insertCharacterAtCursor('t');
+							syl.insertCharacterAtCursor('s');
+						}
+						else {
+							syl.insertCharacterAtCursor('T');
+							syl.insertCharacterAtCursor('S');							
+						}
+						
+						i++;
+						continue;
+					}
+										
+					// oe -> ue
+					if (lowstr1=="o" && lowstr2=="e") {
+						// detect case
+						if (str1[0] == tolower(str1[0])) {
+							syl.insertCharacterAtCursor('u', sym1.tone());
+							syl.insertCharacterAtCursor('e', sym2.tone());
+						}
+						else {
+							syl.insertCharacterAtCursor('U', sym1.tone());
+							syl.insertCharacterAtCursor('E', sym2.tone());							
+						}
+						
+						i++;
+						continue;
+					}
+								
+					// oa -> ua
+					if (lowstr1=="o" && lowstr2=="a") {
+						// detect case
+						if (str1[0] == tolower(str1[0])) {
+							syl.insertCharacterAtCursor('u', sym1.tone());
+							syl.insertCharacterAtCursor('a', sym2.tone());
+						}
+						else {
+							syl.insertCharacterAtCursor('U', sym1.tone());
+							syl.insertCharacterAtCursor('A', sym2.tone());
+						}
+						
+						i++;
+						continue;
+					}
+
+					// ek -> ik (at ending)
+					if (lowstr1=="e" && lowstr2=="k" && (i+2)==size) {
+						// detect case
+						if (str1[0] == tolower(str1[0])) {
+							syl.insertCharacterAtCursor('i', sym1.tone());
+							syl.insertCharacterAtCursor('k', sym2.tone());
+						}
+						else {
+							syl.insertCharacterAtCursor('I', sym1.tone());
+							syl.insertCharacterAtCursor('K', sym2.tone());
+						}
+						
+						i++;
+						continue;
+					}
+					
+					if (hasNextNextSymbol(i) && (i+3)==size) {
+						HoloSymbol sym3 = _symvec[i+2];
+						string str3 = sym3.symbol();
+						string lowstr3 = sym3.symbolInLowerCase();
+
+						// ing -> eng (must be ending)					
+						if (lowstr1=="e" && lowstr2=="n" && lowstr3=="g") {
+							// detect case
+							if (str1[0] == tolower(str1[0])) {
+								syl.insertCharacterAtCursor('i', sym1.tone());
+								syl.insertCharacterAtCursor('n', sym2.tone());
+								syl.insertCharacterAtCursor('g', sym3.tone());
+							}
+							else {
+								syl.insertCharacterAtCursor('I', sym1.tone());
+								syl.insertCharacterAtCursor('N', sym2.tone());
+								syl.insertCharacterAtCursor('G', sym3.tone());
+							}
+
+							i+=2;
+							continue;
+						}						
+					}			
+				}
+
+				syl.insertSymbolAtCursor(sym1);
+			}			
+			return syl;
+		}
+		
+		
+	protected:
+		bool atBeginning()
+		{
+			return _cursor == 0;
+		}
+		
+		bool atEnd()
+		{
+			return _cursor == numberOfCodepoints();
+		}
+
+		void clearPreparedTone()
+		{
+			if (!_preparedTone) return;
+			_preparedTone = 0;
+			_cursor--;
+		}
+		
+		bool hasPreviousSymbolAtCursor()
+		{
+			unsigned int realcursor = _preparedTone ? _cursor-1 : _cursor;
+			return realcursor > 0;
+		}
+		
+		bool hasNextSymbol(unsigned int pos)
+		{
+			if (pos+1 >= _symvec.size()) return false;
+			return true;
+		}
+
+		bool hasNextNextSymbol(unsigned int pos)
+		{
+			if (pos+2 >= _symvec.size()) return false;
+			return true;
+		}
+
+		
+		// the result of this function is unpredictable if there's no
+		// previous symbol--always check with hasPreviousSymbolAtCursor() !
+		HoloSymbol& previousSymbolAtCursor()
+		{
+			unsigned int realcursor = _preparedTone ? _cursor-1 : _cursor;
+			return _symvec[realcursor-1];
+		}
+		
+		// always assumes that the given input is in all lower case
+		unsigned int findSymbol(const char *s)
+		{
+			string cpps(s);
+			unsigned int size = _symvec.size();
+			unsigned int i;
+			for (i = 0; i < size; i++) {
+				if (_symvec[i].symbolInLowerCase() == cpps) break;
+			}
+			return i;
+		}
+		
+		unsigned int findSymbolPair(const char *s1, const char *s2)
+		{
+			string cpps1(s1), cpps2(s2);
+
+			unsigned int size = _symvec.size();
+			if (size < 2) return size;
+			
+			unsigned int i;			
+			for (i = 0; i < size-1; i++) {
+				if (_symvec[i].symbolInLowerCase()==cpps1 && _symvec[i+1].symbolInLowerCase()==cpps2) return i;
+			}
+			
+			return size;
+		}
+		
+		SyllableType _inputType;
+		DiacriticInputOption _inputOption;
+		bool _forcePOJStyle;
+		
+		vector<HoloSymbol> _symvec;
+		unsigned int _cursor;
+		unsigned int _preparedTone;
+	};
+	
+	class FreeFormSyllable : public ComposableStringBuffer
+	{
+	public:
+		HoloSyllable convertToTLFromTLPA(unsigned int finalTone=0)
+		{
+			string rep=internalForm();
+			HoloSyllable syl;
+			syl.setInputType(TLSyllable);
+			
+			unsigned int size=rep.length();
+			for (unsigned int i=0; i<size; i++)
+			{
+				if (rep[i]=='c') {
+					syl.insertCharacterAtCursor('t');
+					syl.insertCharacterAtCursor('s');
+				}
+				else if (rep[i]=='C')
+				{
+					syl.insertCharacterAtCursor('T');
+					syl.insertCharacterAtCursor('S');					
+				}
+				else syl.insertCharacterAtCursor(rep[i]);
+			}
+			
+			syl.normalize(finalTone);
+			return syl;
+		}
+		
+		
+		HoloSyllable convertToTLFromDT(unsigned int finalTone=0)
+		{
+			string rep=internalForm();
+			HoloSyllable syl;
+			syl.setInputType(TLSyllable);
+			
+			unsigned int size=rep.length();
+			for (unsigned int i=0; i<size; i++)	
+			{
+				char dt1 = rep[i];
+				char lowdt1 = tolower(dt1);
+				
+				// r -> j (beginning)
+				if (i==0 && lowdt1=='r') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('j', dt1));
+					continue;
+				}
+
+				// replaces the two-character combinations
+				if (i+1 < size) {
+					string part=rep.substr(i, 2);
+					string lower=toLowerString(part);
+					
+					// or -> o
+					if (lower=="or") {
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('o', part));
+						i++;
+						continue;
+					}
+					
+					// en -> ian
+					if (lower=="en") {                                          
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('i', part));
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('a', part));
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('n', part));
+						i++;                                                    
+						continue;
+					}
+					
+					// et -> iat
+					if (lower=="et") {                                          
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('i', part));
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('a', part));
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('t', part));
+						i++;                                                    
+						continue;
+					}
+
+					
+					// bh -> b (beginning)
+					if (i==0 && lower=="bh") {
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('b', part));
+						i++;
+						continue;
+					}
+
+					// gh -> g (beginning)
+					if (i==0 && lower=="gh") {
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('g', part));
+						i++;
+						continue;
+					}
+					
+					// wa -> ua (beginning)
+					if (lower=="wa" && i==0) {
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('u', part));
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('a', part));
+						i++;
+						continue;
+					}
+
+					// we -> ue (beginning)
+					if (lower=="we" && i==0) {
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('u', part));
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('e', part));
+						i++;
+						continue;
+					}
+
+					// wi -> ui (beginning)
+					if (lower=="wi" && i==0) {
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('u', part));
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('i', part));
+						i++;
+						continue;
+					}
+
+					// yo -> io (beginning)
+					if (lower=="yo" && i==0) {
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('i', part));
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('o', part));
+						i++;
+						continue;
+					}
+
+					// yi -> i (beginning)
+					if (lower=="yi" && i==0) {
+						syl.insertCharacterAtCursor(charWithCaseAccordingTo('i', part));
+						i++;
+						continue;
+					}
+				}
+
+				// o -> oo
+				if (lowdt1=='o') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('o', dt1));
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('o', dt1));
+					continue;
+				}
+				
+				// b -> p (beginning)
+				if (i==0 && lowdt1=='b') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('p', dt1));
+					continue;
+				}
+
+				// p -> ph (beginning)
+				if (i==0 && lowdt1=='p') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('p', dt1));
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('h', dt1));
+					continue;
+				}
+
+				// k -> kh (beginning)
+				if (i==0 && lowdt1=='k') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('k', dt1));
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('h', dt1));
+					continue;
+				}
+
+				// g -> k (beginning)
+				if (i==0 && lowdt1=='g') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('k', dt1));
+					continue;
+				}
+
+				// d -> t (beginning)
+				if (i==0 && lowdt1=='d') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('t', dt1));
+					continue;
+				}
+				
+				// t -> th (beginning)
+				if (i==0 && lowdt1=='t') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('t', dt1));
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('h', dt1));
+					continue;
+				}
+
+				// z -> ts (beginning)
+				if (i==0 && lowdt1=='z') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('t', dt1));
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('s', dt1));
+					continue;
+				}
+
+				// c -> tsh (beginning)
+				if (i==0 && lowdt1=='c') {
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('t', dt1));
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('s', dt1));
+					syl.insertCharacterAtCursor(charWithCaseAccordingTo('h', dt1));
+					continue;
+				}
+
+				// else ...
+				syl.insertCharacterAtCursor(dt1);
+			}
+			
+			// remap the final tone
+			unsigned int tltone=finalTone;
+
+			syl.normalize(tltone);
+			return syl;
+			
+		}
+
+	protected:
+		char charWithCaseAccordingTo(char c, char ref)
+		{
+			if (tolower(ref) == ref) return tolower(c);
+			return toupper(c);
+		}
+		
+		char charWithCaseAccordingTo(char c, const string &r)
+		{
+			if (tolower(r[0]) == r[0]) return tolower(c);
+			return toupper(c);
+		}
+		
+		string toLowerString(const string &s)
+		{
+			unsigned int size=s.length();
+			string lower;
+			unsigned int i;
+			for (i=0;i<size;i++) lower+=string(1, tolower(s[i]));
+			return lower;
+		}
+	};
+};
+
+