ruby-sfst 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19ac73a1ab1fdac44819260e4e522ad822cb99bf
-  data.tar.gz: 16a8b37a117b2367a0701202e1e6ef178ef6907a
+  metadata.gz: 4892aa62071212d5f65b05afb0476d59852c75c2
+  data.tar.gz: a00ef4a9b22a7b1fda9c0c6e8f89df2a1fe24205
 SHA512:
-  metadata.gz: bf06fe247b61f4d4d1f7cc516a34d9b706bfab7116f7859b419548956a16e160313ac6523a85979241d644abaa216cb79087be092b6ddb7f30fa62763e8d58b5
-  data.tar.gz: 35e93cf5b6f8f88656a19f11dd68561a3541a4c5faf1d0f7bf15b9b78d91bffde33767e4f683bdfd46ffc51869678d83b461a4fcd651ef91bc8af05f4768681f
+  metadata.gz: 194dbde4538a06b21367d1083c7f32d75558539b31e4df0bdcb6a53793ec619e3ace0243e41253117a95a0ec36b26dbe4c81f65ad23e33fe366e4c0d7758a82e
+  data.tar.gz: 2b6074dc37444cd3bdce4c75db8b3dfb4574841c16cc67d82bc39ecc5539f2c1af36b2c0f456e0bf021ac89898efdb8898a4ae4b82dd5221ec6e9724b7ec665e

data/CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+* Update to SFST 1.4.7c
 * Dropped support for compiling FSTs from Ruby; use fst-compiler-utf8 instead
 
 ## 0.4.1

data/COPYING

@@ -0,0 +1,280 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+     59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,54 @@
+PATH
+  remote: .
+  specs:
+    ruby-sfst (0.4.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    diff-lcs (1.3)
+    docile (1.1.5)
+    json (2.1.0)
+    method_source (0.9.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (12.3.0)
+    rake-compiler (1.0.4)
+      rake
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+    simplecov (0.15.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    yard (0.9.12)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  pry (~> 0.11)
+  rake (~> 12.3)
+  rake-compiler (~> 1.0)
+  rspec (~> 3.7)
+  ruby-sfst!
+  simplecov (~> 0.15)
+  yard (~> 0.9)
+
+BUNDLED WITH
+   1.16.1

data/README.md

@@ -3,7 +3,7 @@
 `ruby-sfst` is a wrapper for the Stuttgart Finite State Transducer Tools
 (SFST).
 
-The wrapper is based on SFST 1.3. See
+The wrapper is based on SFST 1.4.7c. See
 http://www.cis.uni-muenchen.de/~schmid/tools/SFST/ for details on how to obtain
 SFST and how to write SFST transducers.
 

data/Rakefile

@@ -1,23 +1,14 @@
-# coding: utf-8
-require 'bundler'
-Bundler::GemHelper.install_tasks
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
 
-namespace :doc do
-  require 'yard'
-  YARD::Rake::YardocTask.new do |task|
-  task.files   = ['README.md', 'lib/**/*.rb']
-  task.options = [
-    '--output-dir', 'doc/yard',
-    '--markup', 'markdown',
-    ]
-  end
-end
+RSpec::Core::RakeTask.new(:spec)
+
+require "rake/extensiontask"
 
-require 'rake/testtask'
+task :build => :compile
 
-Rake::TestTask.new do |t|
-    t.libs << 'test'
+Rake::ExtensionTask.new("sfst") do |ext|
+  ext.lib_dir = "lib/sfst"
 end
 
-desc "Run tests"
-task :default => :test
+task :default => [:clobber, :compile, :spec]

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "sfst"
+
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/ext/sfst/alphabet.cc

@@ -0,0 +1,879 @@
+
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     alphabet.C                                            */
+/*  MODULE   alphabet                                              */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*  PURPOSE  basic FST functions                                   */
+/*                                                                 */
+/*******************************************************************/
+
+#include <climits>
+#include <cstring>
+
+#include "utf8.h"
+#include "alphabet.h"
+
+#include <map>
+using std::map;
+
+namespace SFST {
+
+  using std::vector;
+  using std::ostream;
+
+  const int BUFFER_SIZE=100000;
+
+  char EpsilonString[]="<>";
+
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::add                                                  */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::add( const char *symbol, Character c )
+
+  {
+    char *s = fst_strdup(symbol);
+    cm[c] = s;
+    sm[s] = c;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::Alphabet                                             */
+  /*                                                                 */
+  /*******************************************************************/
+
+  Alphabet::Alphabet()
+
+  { 
+    utf8 = false;
+    add(EpsilonString, Label::epsilon);
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::clear                                                */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::clear()
+
+  {
+    char **s=new char*[cm.size()];
+    ls.clear();
+    sm.clear();
+
+    size_t i, n=0;
+    for( CharMap::iterator it=cm.begin(); it!=cm.end(); it++ )
+      s[n++] = it->second;
+    cm.clear();
+
+    for( i=0; i<n; i++ )
+      free(s[i]);
+    delete[] s;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::print                                                */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::print(void)
+
+  {
+    for( LabelSet::const_iterator it=begin(); it!=end(); it++ ) {
+      Label l = *it;
+      fprintf(stderr, "%s\n", write_label(l));
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::new_marker                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  Character Alphabet::new_marker()
+
+  {
+    // find some unused character code
+    for(Character i=1; i!=0; i++)
+      if (cm.find(i) == cm.end()) {
+	// create a unique identifier string
+	char symbol[100];
+	sprintf(symbol,">%ld<",(long)i);
+	add(symbol, i);
+	return i;
+      }
+  
+    throw "Error: too many symbols in transducer definition";
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  is_marker_symbol                                               */
+  /*                                                                 */
+  /*******************************************************************/
+
+  static bool is_marker_symbol( const char *s )
+
+  {
+    // recogize strings matching the expression ">[0-9]+<"
+    if (s != NULL && *s == '>') {
+      do { s++; } while (*s >= '0' && *s <= '9');
+      if (*s=='<' && *(s+1) == 0 && *(s-1) != '>')
+	return true;
+    }
+    return false;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::delete_markers                                       */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::delete_markers()
+
+  {
+    vector<char*> sym;
+    vector<Character> code;
+    vector<Label> label;
+
+    for( CharMap::const_iterator it=cm.begin(); it!=cm.end(); it++ ) {
+      Character c=it->first;
+      char *s=it->second;
+      if (!is_marker_symbol(s)) {
+	sym.push_back(fst_strdup(s));
+	code.push_back(c);
+      }
+    }
+    
+    for( LabelSet::const_iterator it=begin(); it!=end(); it++ ) {
+      Label l=*it;
+      if (!is_marker_symbol(code2symbol(l.upper_char())) &&
+	  !is_marker_symbol(code2symbol(l.lower_char())))
+	label.push_back(l);
+    }
+
+    clear();
+
+    for( size_t i=0; i<sym.size(); i++ ) {
+      add_symbol(sym[i], code[i]);
+      free(sym[i]);
+    }
+    for( size_t i=0; i<label.size(); i++ )
+      insert( label[i] );
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::add_symbol                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  Character Alphabet::add_symbol(const char *symbol)
+
+  {
+    if (sm.find(symbol) != sm.end())
+      return sm[symbol];
+
+    // assign the symbol to some unused character
+    for(Character i=1; i!=0; i++)
+      if (cm.find(i) == cm.end()) {
+	add(symbol, i);
+	return i;
+      }
+  
+    throw "Error: too many symbols in transducer definition";
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::add_symbol                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::add_symbol( const char *symbol, Character c )
+
+  {
+    // check whether the symbol was previously defined
+    int sc=symbol2code(symbol);
+    if (sc != EOF) {
+      if ((Character)sc == c)
+	return;
+
+      if (strlen(symbol) < 60) {
+	static char message[100];
+	sprintf(message, "Error: reinserting symbol '%s' in alphabet with incompatible character value %u %u", symbol, (unsigned)sc, (unsigned)c);
+	throw message;
+      }
+      else
+	throw "reinserting symbol in alphabet with incompatible character value";
+    }
+
+    // check whether the character is already in use
+    const char *s=code2symbol(c);
+    if (s == NULL)
+      add(symbol, c);
+    else {
+      if (strcmp(s, symbol) != 0) {
+	static char message[100];
+	if (strlen(symbol) < 70)
+	  sprintf(message,"Error: defining symbol %s as character %d (previously defined as %s)", symbol, (unsigned)c, s);
+	else
+	  sprintf(message,"Error: defining a (very long) symbol with previously used character");
+	throw message;
+      }
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::write_char                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::write_char( Character c, char *buffer, int *pos, 
+			     bool with_brackets) const
+  {
+    const char *s = code2symbol(c);
+
+    // quote colons
+    if (strcmp(s,":") == 0 || strcmp(s,"\\") == 0) {
+      buffer[(*pos)++] = '\\';
+      buffer[(*pos)++] = s[0];
+    }
+    else if (s) {
+      int i = 0;
+      int l=(int)strlen(s)-1;
+      if (!with_brackets && s[i] == '<' && s[l] == '>') { i++; l--; }
+      while (i <= l)
+	buffer[(*pos)++] = s[i++];
+    }
+    else {
+      unsigned int uc = c;
+      if (uc>=32 && uc<256)
+	buffer[(*pos)++] = (char)c;
+      else {
+	sprintf(buffer+(*pos),"\\%u", uc);
+	*pos += (int)strlen(buffer+(*pos));
+      }
+    }
+    buffer[*pos] = '\0';
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::write_char                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  const char *Alphabet::write_char( Character c, bool with_brackets ) const
+
+  {
+    static char buffer[1000];
+    int n=0;
+
+    write_char( c, buffer, &n, with_brackets );
+    return buffer;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::write_label                                          */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::write_label( Label l, char *buffer, int *pos, 
+			      bool with_brackets ) const
+  {
+    Character lc=l.lower_char();
+    Character uc=l.upper_char();
+    write_char( lc, buffer, pos, with_brackets );
+    if (lc != uc) {
+      buffer[(*pos)++] = ':';
+      write_char( uc, buffer, pos, with_brackets );
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::write_label                                          */
+  /*                                                                 */
+  /*******************************************************************/
+
+  const char *Alphabet::write_label( Label l, bool with_brackets  ) const
+
+  {
+    static char buffer[1000];
+    int n=0;
+    write_label( l, buffer, &n, with_brackets );
+    return buffer;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::insert_symbols                                       */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::insert_symbols( const Alphabet &a )
+
+  {
+    for( CharMap::const_iterator it=a.cm.begin(); it!=a.cm.end(); it++ )
+      add_symbol(it->second, it->first);
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::complement                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::complement( vector<Character> &sym )
+
+  {
+    vector<Character> result;
+    for( CharMap::const_iterator it=cm.begin(); it!=cm.end(); it++ ) {
+      Character c = it->first;
+      if (c != Label::epsilon) {
+	size_t i;
+	for( i=0; i<sym.size(); i++ )
+	  if (sym[i] == c)
+	    break;
+	if (i == sym.size())
+	  result.push_back(c);
+      }
+    }
+    sym.swap(result);
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::copy                                                 */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::copy( const Alphabet &a, Level level )
+
+  {
+    utf8 = a.utf8;
+    sm.resize(a.sm.size());
+    cm.resize(a.sm.size());
+    insert_symbols( a );
+    for( LabelSet::const_iterator it=a.begin(); it!=a.end(); it++ ) {
+      Label l = *it;
+      if (level == lower)
+	ls.insert( Label(l.lower_char()) );
+      else if (level == upper)
+	ls.insert( Label(l.upper_char()) );
+      else
+	ls.insert( l );
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::compose                                              */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::compose( const Alphabet &la, const Alphabet &ua )
+
+  {
+    // insert the symbols
+    insert_symbols(la);
+    insert_symbols(ua);
+    utf8 = la.utf8;
+
+    map<Character, set<Character> > cs;
+
+    // create a table for a quick lookup of the target characters
+    for( iterator it=ua.begin(); it!=ua.end(); it++ ) {
+      Character lc=it->lower_char();
+      if (lc == Label::epsilon)
+	insert(*it);
+      else
+	cs[lc].insert(it->upper_char());
+    }
+
+    for( iterator it=la.begin(); it!=la.end(); it++ ) {
+      Character uc=it->upper_char();
+      if (uc == Label::epsilon)
+	insert(*it);
+      else {
+	if (cs.find(uc) != cs.end()) {
+	  set<Character> s=cs[uc];
+	  Character lc=it->lower_char();
+	  for( set<Character>::iterator it=s.begin(); it!=s.end(); it++)
+	    insert(Label(lc, *it));
+	}
+      }
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  operator<<(Alphabet)                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  ostream &operator<<( ostream &s, const Alphabet &a )
+
+  {
+    for( Alphabet::CharMap::const_iterator it=a.cm.begin(); it!=a.cm.end(); it++ )
+      s << it->first << " -> " << it->second << "\n";
+    for( Alphabet::iterator it=a.begin(); it!=a.end(); it++ )
+      s << a.write_label(*it) << " ";
+    s << "\n";
+    return s;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::next_mcsym                                           */
+  /*                                                                 */
+  /*  recognizes multi-character symbols which are enclosed with     */
+  /*  angle brackets <...>. If the argument flag insert is false,    */
+  /*  the multi-character symbol must already be in the lexicon in   */
+  /*  order to be recognized.                                        */
+  /*                                                                 */
+  /*******************************************************************/
+
+  int Alphabet::next_mcsym( char* &string, bool insert )
+
+  {
+    char *start=string;
+
+    if (*start == '<')
+      // symbol might start here
+      for( char *end=start+1; *end; end++ )
+	if (*end == '>') {
+	  // matching pair of angle brackets found
+	  // mark the end of the substring with \0
+	  char lastc = *(++end);
+	  *end = 0;
+
+	  int c;
+	  if (insert)
+	    c = add_symbol( start );
+	  else
+	    c = symbol2code(start);
+	  // restore the original string
+	  *end = lastc;
+
+	  if (c != EOF) {
+	    // symbol found
+	    // return its code
+	    string = end;
+	    return (Character)c;
+	  }
+	  else
+	    // not a complex character
+	    break;
+	}
+    return EOF;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::next_code                                            */
+  /*                                                                 */
+  /*******************************************************************/
+
+  int Alphabet::next_code( char* &string, bool extended, bool insert )
+
+  {
+    if (*string == 0)
+      return EOF; // finished
+
+    int c = next_mcsym(string, insert);
+    if (c != EOF)
+      return c;
+
+    if (extended && *string == '\\')
+      string++; // remove quotation
+
+    if (utf8) {
+      unsigned int c = utf8toint( &string );
+      if (c == 0) {
+	fprintf(stderr, "Error in UTF-8 encoding at: <%s>\n", string);
+	return EOF; // error encountered in utf8 character
+      }
+      return (int)add_symbol(int2utf8(c));
+    }
+    else {
+      char buffer[2];
+      buffer[0] = *string;
+      buffer[1] = 0;
+      string++;
+      return (int)add_symbol(buffer);
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::next_label                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  Label Alphabet::next_label( char* &string, bool extended )
+
+  {
+    // read first character
+    int c = next_code( string, extended );
+    if (c == EOF)
+      return Label(); // end of string reached
+
+    Character lc=(Character)c;
+    if (!extended || *string != ':') { // single character?
+      if (lc == Label::epsilon)
+	return next_label(string, extended); // ignore epsilon
+      return Label(lc);
+    }
+
+    // read second character
+    string++; // jump over ':'
+    c = next_code( string, extended );
+    if (c == EOF) {
+      static char buffer[1000];
+      sprintf(buffer,"Error: incomplete symbol in input file: %s", string);
+      throw buffer;
+    }
+
+    Label l(lc, (Character)c);
+    if (l.is_epsilon())
+      return next_label(string, extended); // ignore epsilon transitions
+    return l;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::string2symseq                                        */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::string2symseq( char *s, vector<Character> &ch )
+
+  {
+    int c;
+    while ((c = next_code(s, false, false)) != EOF)
+      ch.push_back((Character)c);
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::string2labelseq                                      */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::string2labelseq( char *s, vector<Label> &labels )
+
+  {
+    Label l;
+    while ((l = next_label(s)) != Label::epsilon)
+      labels.push_back(l);
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::store                                                */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::store( FILE *file ) const
+
+  {
+    char c=(utf8)? (char)1: (char)0;
+    fputc(c, file);
+
+    // write the symbol mapping
+    Character n=(Character)cm.size();
+    fwrite(&n, sizeof(n), 1, file);
+    for( CharMap::const_iterator it=cm.begin(); it!=cm.end(); it++ ) {
+      Character c=it->first;
+      char *s=it->second;
+      fwrite(&c, sizeof(c), 1, file);
+      fwrite(s, sizeof(char), strlen(s)+1, file);
+    }
+
+    // write the character pairs
+    n = (Character)size();
+    fwrite(&n, sizeof(n), 1, file);
+    for( LabelSet::const_iterator p=ls.begin(); p!=ls.end(); p++ ) {
+      Character c=p->lower_char();
+      fwrite(&c, sizeof(c), 1, file);
+      c = p->upper_char();
+      fwrite(&c, sizeof(c), 1, file);
+    }
+
+    if (ferror(file))
+      throw "Error encountered while writing alphabet to file\n";
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::read                                                 */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::read( FILE *file )
+
+  {
+    utf8 = (fgetc(file) != 0);
+
+    // read the symbol mapping
+    Character n=0;
+    read_num(&n, sizeof(n), file);
+    for( unsigned i=0; i<n; i++) {
+      char buffer[BUFFER_SIZE];
+      Character c;
+      read_num(&c, sizeof(c), file);
+      if (!read_string(buffer, BUFFER_SIZE, file) || 
+	  feof(file) || ferror(file))
+	throw "Error1 occurred while reading alphabet!\n";
+      add_symbol(buffer, c);
+    }
+
+    // read the character pairs
+    read_num(&n, sizeof(n), file);
+    if (ferror(file))
+      throw "Error2 occurred while reading alphabet!\n";
+    for( unsigned i=0; i<n; i++) {
+      Character lc, uc;
+      read_num(&lc, sizeof(lc), file);
+      read_num(&uc, sizeof(uc), file);
+      insert(Label(lc, uc));
+    }
+    if (ferror(file))
+      throw "Error3 occurred while reading alphabet!\n";
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::compute_score                                        */
+  /*                                                                 */
+  /*******************************************************************/
+
+  int Alphabet::compute_score( Analysis &ana )
+
+  {
+    // check whether the morpheme boundaries are explicitly marked
+    // with <X> tags
+    int score=0;
+    for( size_t i=0; i<ana.size(); i++ ) {
+
+      // get next symbol
+      const char *sym=write_char(ana[i].lower_char());
+
+      if (strcmp(sym,"<X>") == 0)
+	score--;
+    }
+    if (score <  0)
+      return score;
+
+    // No explicit morpheme boundary markers have been found.
+    // Count the number of part-of-speech and PREF tags.
+    for( size_t i=0; i<ana.size(); i++ ) {
+
+      // get next symbol
+      const char *sym=write_char(ana[i].lower_char());
+
+      // Is it not a multi-character symbol
+      if (sym[0] != '<' || sym[1] == 0)
+	continue;
+
+      // Is it a POS tag starting with "+" like <+NN>?
+      if (sym[1] == '+') {
+	const char *t=sym+2;
+	for( ; *t >= 'A' && *t <= 'Z'; t++) ;
+	if (t > sym+2 && *t == '>')
+	  return score;
+      }
+
+      // Is it a potential POS tag (i.e. all uppercase)?
+      const char *t = sym+1;
+      for( ; *t >= 'A' && *t <= 'Z'; t++) ;
+      if (t == sym+1 || *t != '>')
+	continue;
+
+      // uppercase symbol found
+      if (strcmp(sym,"<SUFF>") == 0 ||
+	  strcmp(sym,"<OLDORTH>") == 0 ||
+	  strcmp(sym,"<NEWORTH>") == 0)
+	continue; // not what we are looking for
+
+      // disprefer nouns with prefixes
+      if (strcmp(sym,"<PREF>") == 0)
+	score-=2;
+
+      if (strcmp(sym,"<V>") == 0 || strcmp(sym,"<ADJ>") == 0) {
+	bool is_verb=(strcmp(sym,"<V>")==0);
+	// get the next non-empty symbol
+	Character c=Label::epsilon;
+	size_t k;
+	for( k=i+1; k<ana.size(); k++ )
+	  if ((c = ana[k].lower_char()) != Label::epsilon)
+	    break;
+	// Is it a participle
+	if (c != Label::epsilon) {
+	  sym = write_char(c);
+	  if (strcmp(sym,"<OLDORTH>") == 0 || 
+	      strcmp(sym,"<NEWORTH>") == 0 || 
+	      strcmp(sym,"<SUFF>") == 0) {
+	    for( k++; k<ana.size(); k++ )
+	      if ((c = ana[k].lower_char()) != Label::epsilon)
+		break;
+	    if (c != Label::epsilon)
+	      sym = write_char(c);
+	  }
+	  if (is_verb &&
+	      (strcmp(sym,"<PPres>") == 0 || strcmp(sym,"<PPast>") == 0))
+	    continue; // don't consider participles as complex
+	  if (!is_verb &&
+	      (strcmp(sym,"<Sup>") == 0 || strcmp(sym,"<Comp>") == 0))
+	    continue;
+	}
+      }
+      score--;
+    }
+    return score;
+  }
+
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::disambiguate                                         */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void Alphabet::disambiguate( vector<Analysis> &analyses )
+
+  {
+    // compute the scores
+    int bestscore=INT_MIN;
+    vector<int> score;
+
+    for( size_t i=0; i<analyses.size(); i++ ) {
+      score.push_back(compute_score(analyses[i]));
+      if (bestscore < score[i])
+	bestscore = score[i];
+    }
+
+    // delete suboptimal analyses
+    size_t k=0;
+    for( size_t i=0; i<analyses.size(); i++ )
+      if (score[i] == bestscore)
+	analyses[k++] = analyses[i];
+    analyses.resize(k);
+  }
+
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::print_analysis                                       */
+  /*                                                                 */
+  /*******************************************************************/
+
+  char *Alphabet::print_analysis( Analysis &ana, bool both_layers )
+
+  {
+    vector<char> ch;
+
+    // for each transition
+    for( size_t i=0; i<ana.size(); i++ ) {
+
+      // get the transition label
+      Label l=ana[i];
+      const char *s;
+
+      // either print the analysis symbol or the whole label
+      if (both_layers)
+	s = write_label(l);
+      else if (l.lower_char() != Label::epsilon)
+	s = write_char(l.lower_char());
+      else
+	continue;
+
+      // copy the characters to the character array
+      while (*s)
+	ch.push_back(*(s++));
+    }
+    ch.push_back(0); // terminate the string
+
+    static char *result=NULL;
+    if (result != NULL)
+      delete[] result;
+    result = new char[ch.size()];
+    for( size_t i=0; i<ch.size(); i++ )
+      result[i] = ch[i];
+  
+    return result;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  Alphabet::operator==                                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  bool Alphabet::operator==(const Alphabet &alpha) const
+
+  {
+    for ( SymbolMap::const_iterator it = this->sm.begin(); it != this->sm.end(); it++ )
+      {
+	SymbolMap::const_iterator alpha_it = alpha.sm.find(it->first);
+	if ( alpha_it == alpha.sm.end() )
+	  return false;
+	if ( alpha_it->second == it->second )
+	  return false;
+      }
+    for ( SymbolMap::const_iterator alpha_it = alpha.sm.begin(); alpha_it != alpha.sm.end(); alpha_it++ )
+      {
+	SymbolMap::const_iterator it = this->sm.find(alpha_it->first);
+	if ( it == this->sm.end() )
+	  return false;
+	if ( it->second == alpha_it->second )
+	  return false;
+      }
+    return true;
+  }
+
+}