ruby-sfst 0.4.3 → 0.4.4

data/ext/sfst/alphabet.h

@@ -0,0 +1,302 @@
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     alphabet.h                                            */
+/*  MODULE   alphabet                                              */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*  PURPOSE  finite state tools                                    */
+/*                                                                 */
+/*******************************************************************/
+
+#ifndef _ALPHABET_H_
+#define _ALPHABET_H_
+
+#include <stdio.h>
+
+#include "basic.h"
+
+#include <set>
+using std::set;
+
+#include <vector>
+using std::vector;
+
+#include <iostream>
+using std::ostream;
+
+#include <cstring>
+
+#include "sgi.h"
+
+#define SFSTVersion "1.4.7a"
+
+namespace SFST {
+
+#ifndef CODE_DATA_TYPE
+  typedef unsigned short Character;  // data type of the symbol codes
+#else
+  typedef unsigned CODE_DATA_TYPE Character;
+#endif
+
+  // data type used to indicate whether some action is to be performed
+  // on the analysis level (lower) or the surface level (upper)
+  typedef enum {upper, lower, both} Level;
+
+
+  /*****************  class Label  ***********************************/
+
+  class Label {
+
+  private:
+    // data structure where the two symbols are stored
+    struct {
+      Character lower;
+      Character upper;
+    } label;
+
+  public:
+    static const Character epsilon=0; // code of the empty symbol
+
+    // new label with two identical symbols
+    Label( Character c=epsilon ) { label.lower = label.upper = c; };
+
+    // new label with two different symbols
+    Label( Character c1, Character c2 )
+      { label.lower = c1; label.upper = c2; };
+
+    // returns the indicated symbol of the label
+    Character get_char( Level l ) const
+    { return ((l==upper)? label.upper: label.lower); };
+
+    // returns the "upper" symbol of the label (i.e. the surface symbol)
+    Character upper_char() const {  return label.upper; };
+
+    // returns the "lower" symbol of the label (i.e. the analysis symbol)
+    Character lower_char() const {  return label.lower; };
+
+    // replaces symbols in a label
+    Label replace_char( Character c, Character nc ) const {
+      Label l = *this;
+      if (l.label.lower == c)
+	l.label.lower = nc;
+      if (l.label.upper == c)
+	l.label.upper = nc;
+      return l;
+    };
+
+    // operators checking the equality of labels
+    int operator==( Label l ) const
+    { return (label.lower==l.label.lower && label.upper==l.label.upper); };
+    int operator!=( Label l ) const
+    { return !(l == *this); };
+
+    // comparison operator needed for sorting labels in compact.C
+    int operator<( Label l ) const { 
+      if (upper_char() < l.upper_char())
+	return true;
+      if (upper_char() > l.upper_char())
+	return false;
+      if (lower_char() < l.lower_char())
+	return true;
+      return false;
+    };
+    int operator>( Label l ) const { 
+      if (upper_char() > l.upper_char())
+	return true;
+      if (upper_char() < l.upper_char())
+	return false;
+      if (lower_char() > l.lower_char())
+	return true;
+      return false;
+    };
+
+    // check whether the label is epsilon (i.e. both symbols are epsilon)
+    // transitions with epsilon labels are epsilon transitions
+    int is_epsilon() const
+    { return (label.upper == epsilon && label.lower == epsilon); };
+
+    // check whether the "upper" symbol is epsilon
+    int upper_is_epsilon() const
+    { return (label.upper == epsilon); };
+
+    // check whether the "lower" symbol is epsilon
+    int lower_is_epsilon() const
+    { return (label.lower == epsilon); };
+
+    // hash function needed to store labels in a hash table
+    struct label_hash {
+      size_t operator() ( const Label l ) const {
+	return (size_t)l.lower_char() ^ 
+	  ((size_t)l.upper_char() << 16) ^
+	  ((size_t)l.upper_char() >> 16);
+      }
+    };
+
+    // comparison function needed to store labels in a map table
+    struct label_cmp {
+      bool operator() ( const Label l1, const Label l2 ) const {
+	return (l1.lower_char() < l2.lower_char() ||
+		(l1.lower_char() == l2.lower_char() && 
+		 l1.upper_char() < l2.upper_char()));
+      }
+    };
+
+    // comparison operator needed to store labels in a hash table
+    struct label_eq {
+      bool operator() ( const Label l1, const Label l2 ) const {
+	return (l1.lower_char() == l2.lower_char() &&
+		l1.upper_char() == l2.upper_char());
+      }
+    };
+  };
+
+  typedef vector<Label> Analysis;
+
+
+  /*****************  class Alphabet  *******************************/
+
+  class Alphabet {
+
+    // string comparison operators needed to stored strings in a hash table
+    struct eqstr {
+      bool operator()(const char* s1, const char* s2) const {
+	return strcmp(s1, s2) == 0;
+      }
+    };
+
+    // data structure storing labels without repetitions (i.e. as a set)
+    typedef set<Label, Label::label_cmp> LabelSet;
+
+    // hash table used to map the symbols to their codes
+    typedef hash_map<const char*, Character, hash<const char*>,eqstr> SymbolMap;
+
+  public: // HFST addition
+    // hash table used to map the codes back to the symbols
+    typedef hash_map<Character, char*> CharMap;
+
+    // HFST addition
+    bool operator==(const Alphabet &alpha) const;
+
+  private:
+    SymbolMap sm; // maps symbols to codes
+    CharMap  cm; // maps codes to symbols
+    LabelSet ls; // set of labels known to the alphabet
+
+    // add a new symbol with symbol code c
+    void add( const char *symbol, Character c );
+
+  public:
+    bool utf8;
+
+    // iterators over the set of known labels
+    typedef LabelSet::iterator iterator;
+    typedef LabelSet::const_iterator const_iterator;
+    Alphabet();
+    ~Alphabet() { clear(); };
+    const_iterator begin() const { return ls.begin(); };
+    const_iterator end() const { return ls.end(); };
+    size_t size() const { return ls.size(); };
+
+    // HFST additions
+    CharMap get_char_map(void) { return cm; };
+    void print(void);
+
+
+    void clear();
+    void clear_char_pairs() { ls.clear(); };
+
+    // lookup a label in the alphabet
+    iterator find( Label l ) { return ls.find(l); };
+
+    // insert a label in the alphabet
+    void insert( Label l ) { if (!l.is_epsilon()) ls.insert(l); };
+
+    // insert the known symbols from another alphabet
+    void insert_symbols( const Alphabet& );
+
+    // insert the labels and known symbols from another alphabet
+    void copy( const Alphabet &a, Level level=both );
+
+    // create the alphabet of a transducer obtained by a composition operation
+    void compose( const Alphabet &la, const Alphabet &ua );
+
+    // add a symbol to the alphabet and return its code
+    Character add_symbol(const char *symbol);
+
+    // add a symbol to the alphabet with a given code
+    void add_symbol(const char *symbol, Character c );
+
+    // create a new marker symbol and return its code
+    Character new_marker( void );
+    void delete_markers();
+
+    // compute the complement of a symbol set
+    void complement( vector<Character> &sym );
+  
+    // return the code of the argument symbol
+    int symbol2code( const char *s ) const { 
+      SymbolMap::const_iterator p = sm.find(s);
+      if (p != sm.end()) return p->second;
+      return EOF;
+    };
+
+    // return the symbol for the given symbol code
+    const char *code2symbol( Character c ) const {
+      CharMap::const_iterator p=cm.find(c);
+      if (p == cm.end())
+	return NULL;
+      else
+	return p->second;
+    };
+
+    // write the symbol for the given symbol code into a string
+    void write_char( Character c, char *buffer, int *pos,
+		     bool with_brackets=true ) const;
+
+    // write the symbol pair of a given label into a string
+    void write_label( Label l, char *buffer, int *pos,
+		      bool with_brackets=true ) const;
+
+    // write the symbol for the given symbol code into a buffer and return
+    // a pointer to it
+    // the flag "with_brackets" indicates whether the angle brackets
+    // surrounding multi-character symbols are to be printed or not
+    const char *write_char( Character c, bool with_brackets=true ) const;
+
+    // write the symbol pair of a given label into a string
+    // and return a pointer to it
+    const char *write_label( Label l, bool with_brackets=true ) const;
+
+    // scan the next multi-character symbol in the argument string
+    int next_mcsym( char*&, bool insert=true );
+
+    // scan the next symbol in the argument string
+    int next_code( char*&, bool extended=true, bool insert=true );
+
+    // convert a character string into a symbol or label sequence
+    void string2symseq( char*, vector<Character>& );
+    void string2labelseq( char*, vector<Label>& );
+
+    // scan the next label in the argument string
+    Label next_label( char*&, bool extended=true );
+
+    // store the alphabet in the argument file (in binary form)
+    void store( FILE* ) const;
+
+    // read the alphabet from the argument file
+    void read( FILE* );
+
+    // disambiguation and printing of analyses
+    int compute_score( Analysis &ana );
+    void disambiguate( vector<Analysis> &analyses );
+    char *print_analysis( Analysis &ana, bool both_layers );
+
+    friend ostream &operator<<(ostream&, const Alphabet&);
+  };
+
+  // write the alphabet to the output stream (in readable form)
+  ostream &operator<<(ostream&, const Alphabet&);
+}
+
+#endif

data/ext/sfst/basic.cc

@@ -0,0 +1,85 @@
+
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     basic.C                                               */
+/*  MODULE   basic                                                 */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*  PURPOSE                                                        */
+/*                                                                 */
+/*******************************************************************/
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "basic.h"
+
+namespace SFST {
+
+  bool Switch_Bytes=false;
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  fst_strdup                                                     */
+  /*                                                                 */
+  /*******************************************************************/
+
+  char* fst_strdup(const char* pString)
+
+  {
+    char* pStringCopy = (char*)malloc(strlen(pString) + 1);
+    if (pStringCopy == NULL) {
+      fprintf(stderr, "\nError: out of memory (malloc failed)\naborted.\n");
+      exit(1);
+    }
+    strcpy(pStringCopy, pString);
+    return pStringCopy;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  read_string                                                    */
+  /*                                                                 */
+  /*******************************************************************/
+
+  int read_string( char *buffer, int size, FILE *file )
+
+  {
+    for( int i=0; i<size; i++ ) {
+      int c=fgetc(file);
+      if (c == EOF || c == 0) {
+	buffer[i] = 0;
+	return (c==0);
+      }
+      buffer[i] = (char)c;
+    }
+    buffer[size-1] = 0;
+    return 0;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  read_num                                                       */
+  /*                                                                 */
+  /*******************************************************************/
+
+  size_t read_num( void *p, size_t n, FILE *file )
+
+  {
+    char *pp=(char*)p;
+    size_t result=fread( pp, 1, n, file );
+    if (Switch_Bytes) {
+      size_t e=n/2;
+      for( size_t i=0; i<e; i++ ) {
+	char tmp=pp[i];
+	pp[i] = pp[--n];
+	pp[n] = tmp;
+      }
+    }
+    return result;
+  }
+}

data/ext/{sfst_machine → sfst}/basic.h

@@ -15,10 +15,13 @@
 
 #include <stdio.h>
 
-extern bool Switch_Bytes;
+namespace SFST {
 
-char* fst_strdup(const char* pString);
-int read_string( char *buffer, int size, FILE *file );
-size_t read_num( void *p, size_t size, FILE *file );
+  extern bool Switch_Bytes;
 
+  char* fst_strdup(const char* pString);
+  int read_string( char *buffer, int size, FILE *file );
+  size_t read_num( void *p, size_t size, FILE *file );
+
+}
 #endif

data/ext/sfst/compact.cc

@@ -0,0 +1,629 @@
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     compact.C                                             */
+/*  MODULE   compact                                               */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*  PURPOSE  Code needed for analysing data                        */
+/*                                                                 */
+/*******************************************************************/
+
+#include <stdio.h>
+#include <math.h>
+
+#include <limits.h>
+
+#include "compact.h"
+
+namespace SFST {
+
+  using std::equal_range;
+  using std::vector;
+  using std::pair;
+
+  class label_less {
+  public:
+    bool operator()(const Label l1, const Label l2) const {
+      return l1.upper_char() < l2.upper_char();
+    }
+  };
+
+  const int BUFFER_SIZE=1000;
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::convert                                     */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::convert( CAnalysis &cana, Analysis &ana )
+
+  {
+    ana.resize(cana.size());
+    for( size_t i=0; i<cana.size(); i++ )
+      ana[i] = label[cana[i]];
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::analyze                                     */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::analyze(unsigned int n, vector<Character> &input,
+				  size_t ipos, CAnalysis &ca, 
+				  vector<CAnalysis> &analyses )
+  {
+    // "n" is the number of the current transducer node/state
+    // "input" is the sequence of input symbols
+    // "ipos" is the input position currently analysed
+    // "ca" stores the incomplete analysis string
+    // "analyses" stores the analyses found so far
+
+    if (analyses.size() > 10000)
+      return; // limit the maximal number of analyses
+
+    // Is the input string fully analyzed and the current node a final node?
+    if (finalp[n] && ipos == input.size())
+      // store the new analysis
+      analyses.push_back(ca);
+
+    // follow the epsilon transitions
+    // first_arc[n] is the number of the first outgoing transition of node n
+    // first_arc[n+1]-1 is the number of the last outgoing transition of node n
+    // first_arc[n+1] is the number of the first outgoing transition of node n+1
+    unsigned int i;
+    for( i=first_arc[n]; 
+	 i<first_arc[n+1] && label[i].upper_char() == Label::epsilon; 
+	 i++)
+      {
+	ca.push_back(i);
+	analyze(target_node[i], input, ipos, ca, analyses);
+	ca.pop_back();
+      }
+
+    // follow the non-epsilon transitions
+
+    // scan the next input symbol
+    if (ipos < input.size()) {
+      // find the set of arcs with matching upper character in the sorted list
+      pair<Label*,Label*>range = 
+	equal_range(label+i, label+first_arc[n+1], Label(input[ipos]), 
+		    label_less());
+      unsigned int to = (unsigned int)(range.second - label);
+
+      // follow the non-epsilon transitions
+      for( i=(unsigned)(range.first-label); i<to; i++) {
+	ca.push_back(i);
+	analyze(target_node[i], input, ipos+1, ca, analyses);
+	ca.pop_back();
+      }
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::analyze_string                              */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::analyze_string( char *s, vector<CAnalysis> &analyses )
+
+  {
+    // "s" input string to be analyzed
+    // "analyses" is the data structure in which the results are stored
+    // and returned
+
+    vector<Character> input;
+
+    alphabet.string2symseq( s, input );
+
+    analyses.clear();
+    CAnalysis ca; // data structure where the current incomplete analysis
+    // is stored
+    analyze(0, input, 0, ca, analyses); // start the analysis
+
+    if (analyses.size() > 10000)
+      fprintf(stderr,"Warning: Only the first 10000 analyses considered for \"%s\"!\n", s);
+  
+    if (simplest_only && analyses.size() > 1)
+      disambiguate( analyses ); // select the simplest analyses
+  }
+
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::~CompactTransducer                          */
+  /*                                                                 */
+  /*******************************************************************/
+
+  CompactTransducer::~CompactTransducer()
+
+  {
+    delete[] finalp;
+    delete[] first_arc;
+    delete[] label;
+    delete[] target_node;
+    delete[] final_logprob;
+    delete[] arc_logprob;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::CompactTransducer                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  CompactTransducer::CompactTransducer()
+
+  {
+    both_layers = false;
+    simplest_only = false;
+    number_of_nodes = 0;
+    number_of_arcs = 0;
+    finalp = NULL;
+    first_arc = NULL;
+    label = NULL;
+    target_node = NULL;
+    arc_logprob = final_logprob = (float*)NULL;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::read_finalp                                 */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::read_finalp( FILE *file )
+
+  {
+    int k=0;
+    unsigned char n=0;
+    for( size_t i=0; i<number_of_nodes; i++ ) {
+      if (k == 0) {
+	n = (unsigned char)fgetc(file);
+	k = 8;
+      }
+      k--;
+      if (n & (1 << k))
+	finalp[i] = 1;
+      else
+	finalp[i] = 0;
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::read_first_arcs                             */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::read_first_arcs( FILE *file )
+
+  {
+    int k=0;
+    unsigned int n=0;
+    int bits=(int)ceil(log(number_of_arcs+1)/log(2));
+
+    for( size_t i=0; i<=number_of_nodes; i++ ) {
+      first_arc[i] = n >> (sizeof(n)*8 - bits);
+      n <<= bits;
+      k -= bits;
+      if (k < 0) {
+	read_num(&n,sizeof(n),file);
+	first_arc[i] |= n >> (sizeof(n)*8 + k);
+	n <<= -k;
+	k += (int)sizeof(n) * 8;
+      }
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::read_target_nodes                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::read_target_nodes( FILE *file )
+
+  {
+    int k=0;
+    unsigned int n=0;
+    int bits=(int)ceil(log(number_of_nodes)/log(2));
+
+    for( size_t i=0; i<number_of_arcs; i++ ) {
+      target_node[i] = n >> (sizeof(n)*8 - bits);
+      n <<= bits;
+      k -= bits;
+      if (k < 0) {
+	read_num(&n,sizeof(n),file);
+	target_node[i] |= n >> (sizeof(n)*8 + k);
+	n <<= -k;
+	k += (int)sizeof(n) * 8;
+      }
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::read_labels                                 */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::read_labels( FILE *file )
+
+  {
+    size_t N=0;
+    vector<Label> Num2Label(alphabet.size());
+    for( Alphabet::const_iterator it=alphabet.begin();
+	 it != alphabet.end(); it++ )
+      {
+	Label l=*it;
+	Num2Label[N++] = l;
+      }
+
+    int k=0;
+    unsigned int n=0;
+    int bits=(int)ceil(log((double)alphabet.size())/log(2));
+
+    for( size_t i=0; i<number_of_arcs; i++ ) {
+      unsigned int l = n >> (sizeof(n)*8 - bits);
+      n <<= bits;
+      k -= bits;
+      if (k < 0) {
+	read_num(&n,sizeof(n),file);
+	l |= n >> (sizeof(n)*8 + k);
+	n <<= -k;
+	k += (int)sizeof(n) * 8;
+      }
+      label[i] = Num2Label[l];
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::read_probs                                  */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::read_probs( FILE *file )
+
+  {
+    size_t n,m;
+    fread(&n, sizeof(n), 1, file);
+    if (fread(&m, sizeof(n), 1, file) != 1 ||
+	n != node_count() || m != arc_count())
+      {
+	fprintf(stderr,"Error: incompatible probability file!\n");
+	exit(1);
+      }
+    final_logprob = new float[n];
+    arc_logprob = new float[m];
+    fread(final_logprob, sizeof(float), n, file);
+    if (fread(arc_logprob, sizeof(float), n, file) != n) {
+      fprintf(stderr,"Error: in probability file!\n");
+      exit(1);
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::CompactTransducer                           */
+  /*                                                                 */
+  /*******************************************************************/
+
+  CompactTransducer::CompactTransducer( FILE *file, FILE *pfile )
+
+  {
+    both_layers = false;
+    simplest_only = false;
+
+    if (fgetc(file) != 'c')
+      throw "Error: wrong file format (not a compact transducer)\n";
+
+    alphabet.read(file);
+
+    read_num(&number_of_nodes,sizeof(number_of_nodes),file);
+    read_num(&number_of_arcs,sizeof(number_of_arcs),file);
+
+    if (!ferror(file)) {
+      // memory allocation
+      finalp = new char[number_of_nodes];
+      first_arc = new unsigned[number_of_nodes+1];
+      label = new Label[number_of_arcs];
+      target_node = new unsigned[number_of_arcs];
+    
+      // reading the data
+      read_finalp(file);
+      read_first_arcs(file);
+      read_labels(file);
+      read_target_nodes(file);
+    }
+
+    if (pfile == NULL)
+      arc_logprob = final_logprob = (float*)NULL;
+    else
+      read_probs(pfile);
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::longest_match2                              */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::longest_match2(unsigned int n, char *string, int l, 
+					 CAnalysis &ca, int &bl, CAnalysis &ba)
+  {
+    // n: transducer state
+    // string: rest string
+    // l: length of current analysis
+    // bl: length of the currently longest match
+    // ca: current analysis
+    // ba: best analysis
+
+    if (finalp[n] && l > bl) {
+      // store the new analysis
+      bl = l;
+      ba = ca; // copy the arc vector
+    }
+
+    // follow the epsilon transitions
+    unsigned int i;
+    for( i=first_arc[n]; 
+	 i<first_arc[n+1] && label[i].upper_char() == Label::epsilon; 
+	 i++)
+      {
+	ca.push_back(i);
+	longest_match2(target_node[i], string, l, ca, bl, ba);
+	ca.pop_back();
+      }
+
+    // follow the non-epsilon transitions
+    char *end=string;
+    int c=alphabet.next_code(end, false, false);
+    l += (int)(end - string);
+    if (c != EOF) {
+      // find the set of arcs with matching upper character in the sort list
+      pair<Label*,Label*>range = 
+	equal_range(label+i, label+first_arc[n+1], Label((Character)c), 
+		    label_less());
+      unsigned int to = (unsigned int)(range.second - label);
+      for( i=(unsigned)(range.first-label); i<to; i++) {
+	ca.push_back(i);
+	longest_match2(target_node[i], end, l, ca, bl, ba);
+	ca.pop_back();
+      }
+    }
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::print_analysis                              */
+  /*                                                                 */
+  /*******************************************************************/
+
+  char *CompactTransducer::print_analysis( CAnalysis &cana )
+
+  {
+    Analysis ana;
+    convert(cana, ana);
+    return alphabet.print_analysis( ana, both_layers );
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::longest_match                               */
+  /*                                                                 */
+  /*******************************************************************/
+
+  const char *CompactTransducer::longest_match( char* &string )
+
+  {
+    vector<char> analysis;
+    CAnalysis ca, ba;
+    int l=0;
+    longest_match2(0, string, 0, ca, l, ba);
+
+    // no match? return the next character
+    if (ba.size() == 0) {
+      int c=alphabet.next_code(string, false, false);
+      return alphabet.code2symbol((Character)c);
+    }
+
+    string += l;
+    return print_analysis( ba );
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::disambiguate                                */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::disambiguate( vector<CAnalysis> &analyses )
+
+  {
+    // compute the scores
+    int bestscore=INT_MIN;
+    vector<int> score;
+    Analysis ana;
+
+    for( size_t i=0; i<analyses.size(); i++ ) {
+      convert(analyses[i], ana);
+      score.push_back(alphabet.compute_score(ana));
+      if (bestscore < score[i])
+	bestscore = score[i];
+    }
+
+    // delete suboptimal analyses
+    size_t k=0;
+    for( size_t i=0; i<analyses.size(); i++ )
+      if (score[i] == bestscore)
+	analyses[k++] = analyses[i];
+    analyses.resize(k);
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::train2                                      */
+  /*                                                                 */
+  /*******************************************************************/
+
+  bool CompactTransducer::train2( char *s, vector<double> &arcfreq, 
+				  vector<double> &finalfreq )
+  {
+    vector<CAnalysis> analyses;
+    vector<Label> input;
+    alphabet.string2labelseq( s, input );
+
+    CAnalysis ca; // data structure where the analysis is stored
+    unsigned int n=0;
+    bool failure=false;
+    for( size_t i=0; i<input.size(); i++ ) {
+      failure = true;
+      for( unsigned int k=first_arc[n]; k<first_arc[n+1]; k++) {
+	if (label[k] == input[i]) {
+	  ca.push_back(k);
+	  n = target_node[k];
+	  failure = false;
+	  break;
+	}
+      }
+      if (failure)
+	break;
+    }
+    if (failure || !finalp[n]) {
+      fprintf(stderr,"Warning: The following input is not covered:\n%s\n", s);
+      return false;
+    }
+
+    for( size_t k=0; k<ca.size(); k++ )
+      arcfreq[ca[k]]++;
+    finalfreq[target_node[ca.back()]]++;
+
+    return true;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::train                                       */
+  /*                                                                 */
+  /*******************************************************************/
+
+  bool CompactTransducer::train( char *s, vector<double> &arcfreq, 
+				 vector<double> &finalfreq )
+  {
+    vector<CAnalysis> analyses;
+    vector<Character> input;
+    alphabet.string2symseq( s, input );
+
+    CAnalysis ca; // data structure where the current incomplete analysis
+    // is stored
+    analyze(0, input, 0, ca, analyses); // start the analysis
+
+    if (analyses.size() > 10000)
+      return true; // ignore inputs with more than 10000 analyses
+    else if (analyses.size() == 0)
+      return false;
+  
+    if (simplest_only && analyses.size() > 1)
+      disambiguate( analyses ); // select the simplest analyses
+
+    if (analyses.size() > 0) {
+      double incr = 1.0 / (double)analyses.size();
+      CAnalysis arcs;
+
+      for( size_t i=0; i<analyses.size(); i++ ) {
+	CAnalysis &arcs=analyses[i];
+	for( size_t k=0; k<arcs.size(); k++ )
+	  arcfreq[arcs[k]] += incr;
+	finalfreq[target_node[arcs.back()]] += incr;
+      }
+    }
+    return true;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::estimate_probs                              */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::estimate_probs( vector<double> &arcfreq, 
+					  vector<double> &finalfreq )
+  {
+    // turn frequencies into probabilities
+    for( size_t n=0; n<finalfreq.size(); n++ ) {
+      double sum = finalfreq[n];
+      for( size_t a=first_arc[n]; a<first_arc[n+1]; a++ )
+	sum += arcfreq[a];
+      if (sum == 0.0)
+	sum = 1.0;
+      finalfreq[n] = finalfreq[n] / sum;
+      for( size_t a=first_arc[n]; a<first_arc[n+1]; a++ )
+	arcfreq[a] = arcfreq[a] / sum;
+    }
+  }
+
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  CompactTransducer::compute_probs                               */
+  /*                                                                 */
+  /*******************************************************************/
+
+  void CompactTransducer::compute_probs( vector<CAnalysis> &analyses, 
+					 vector<double> &prob )
+  {
+    prob.resize(analyses.size());
+    double sum=0.0;
+    for( size_t i=0; i<analyses.size(); i++ ) {
+      CAnalysis &a=analyses[i];
+
+      // compute the probability
+      double logprob=0.0;
+      for( size_t k=0; k<a.size(); k++ )
+	logprob += arc_logprob[a[k]];
+      logprob += final_logprob[target_node[a.back()]];
+      prob[i] = exp(logprob);
+      sum += prob[i];
+    }
+
+    // sort the analyses
+    vector<CAnalysis> oldanalyses(analyses);
+    vector<double> oldprob(prob);
+    for( size_t i=0; i<analyses.size(); i++ ) {
+      prob[i] = -1.0;
+      size_t n=0;
+      for( size_t k=0; k<oldanalyses.size(); k++ )
+	if (prob[i] < oldprob[k]) {
+	  prob[i] = oldprob[k];
+	  n = k;
+	}
+      analyses[i] = oldanalyses[n];
+      oldprob[n] = -1.0;
+      prob[i] /= sum; // normalization
+    }
+  }
+}