ruby-sfst 0.4.3 → 0.4.4

data/ext/sfst_machine/alphabet.h

@@ -1,273 +0,0 @@
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     alphabet.h                                            */
-/*  MODULE   alphabet                                              */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*  PURPOSE  finite state tools                                    */
-/*                                                                 */
-/*******************************************************************/
-
-#ifndef _ALPHABET_H_
-#define _ALPHABET_H_
-
-#include <stdio.h>
-
-#include "basic.h"
-
-#include <set>
-#include <vector>
-
-#include <iostream>
-
-#include <cstring>
-
-#include "sgi.h"
-
-#ifndef CODE_DATA_TYPE
-typedef unsigned short Character;  // data type of the symbol codes
-#else
-typedef unsigned CODE_DATA_TYPE Character;
-#endif
-
-// data type used to indicate whether some action is to be performed
-// on the analysis level (lower) or the surface level (upper)
-typedef enum {upper, lower} Level;
-
-extern char EpsilonString[]; // holds the symbol representing the empty string
-                             // which is usually "<>"
-
-
-/*****************  class Label  ***********************************/
-
-class Label {
-
- private:
-  // data structure where the two symbols are stored
-  struct {
-    Character lower;
-    Character upper;
-  } label;
-
- public:
-  static const Character epsilon=0; // code of the empty symbol
-
-  // new label with two identical symbols
-  Label( Character c=epsilon ) { label.lower = label.upper = c; };
-
-  // new label with two different symbols
-  Label( Character c1, Character c2 )
-    { label.lower = c1; label.upper = c2; };
-
-  // returns the indicated symbol of the label
-  Character get_char( Level l ) const
-    { return ((l==upper)? label.upper: label.lower); };
-
-  // returns the "upper" symbol of the label (i.e. the surface symbol)
-  Character upper_char() const {  return label.upper; };
-
-  // returns the "lower" symbol of the label (i.e. the analysis symbol)
-  Character lower_char() const {  return label.lower; };
-
-  // replaces symbols in a label
-  Label replace_char( Character c, Character nc ) const {
-    Label l = *this;
-    if (l.label.lower == c)
-      l.label.lower = nc;
-    if (l.label.upper == c)
-      l.label.upper = nc;
-    return l;
-  };
-
-  // operators checking the equality of labels
-  int operator==( Label l ) const
-    { return (label.lower==l.label.lower && label.upper==l.label.upper); };
-  int operator!=( Label l ) const
-    { return !(l == *this); };
-
-  // comparison operator needed for sorting labels
-  int operator<( Label l ) const { 
-    return (upper_char() < l.upper_char()); };
-
-  // check whether the label is epsilon (i.e. both symbols are epsilon)
-  // transitions with epsilon labels are epsilon transitions
-  int is_epsilon() const
-    { return (label.upper == epsilon && label.lower == epsilon); };
-
-  // check whether the "upper" symbol is epsilon
-  int upper_is_epsilon() const
-    { return (label.upper == epsilon); };
-
-  // check whether the "lower" symbol is epsilon
-  int lower_is_epsilon() const
-    { return (label.lower == epsilon); };
-
-  // hash function needed to store labels in a hash table
-  struct label_hash {
-    size_t operator() ( const Label l ) const {
-      return (size_t)l.lower_char() ^ 
-	((size_t)l.upper_char() << 16) ^
-	((size_t)l.upper_char() >> 16);
-    }
-  };
-
-  // hash function needed to store labels in a hash table
-  struct label_cmp {
-    bool operator() ( const Label l1, const Label l2 ) const {
-      return (l1.lower_char() < l2.lower_char() ||
-	      (l1.lower_char() == l2.lower_char() && 
-	       l1.upper_char() < l2.upper_char()));
-    }
-  };
-
-  // comparison operator needed to store labels in a hash table
-  struct label_eq {
-    bool operator() ( const Label l1, const Label l2 ) const {
-      return (l1.lower_char() == l2.lower_char() &&
-	      l1.upper_char() == l2.upper_char());
-    }
-  };
-};
-
-typedef std::vector<Label> Analysis;
-
-
-/*****************  class Alphabet  *******************************/
-
-class Alphabet {
-
-  // string comparison operators needed to stored strings in a hash table
-  struct eqstr {
-      bool operator()(const char* s1, const char* s2) const {
-	  return strcmp(s1, s2) == 0;
-      }
-  };
-
-  // data structure storing labels without repetitions (i.e. as a set)
-  typedef std::set<Label, Label::label_cmp> LabelSet;
-
-  // hash table used to map the symbols to their codes
-  typedef hash_map<const char*, Character, hash<const char*>,eqstr> SymbolMap;
-
-  // hash table used to map the codes back to the symbols
-  typedef hash_map<Character, char*> CharMap;
-
- private:
-  SymbolMap sm; // maps symbols to codes
-
-  CharMap  cm; // maps codes to symbols
-  LabelSet ls; // set of labels known to the alphabet
-
-  // add a new symbol with symbol code c
-  void add( const char *symbol, Character c );
-
- public:
-  bool utf8;
-
-  // iterators over the set of known labels
-  typedef LabelSet::iterator iterator;
-  typedef LabelSet::const_iterator const_iterator;
-  Alphabet();
-  ~Alphabet() { clear(); };
-  const_iterator begin() const { return ls.begin(); };
-  const_iterator end() const { return ls.end(); };
-  size_t size() const { return ls.size(); };
-
-  void clear();
-  void clear_char_pairs() { ls.clear(); };
-
-  // lookup a label in the alphabet
-  iterator find( Label l ) { return ls.find(l); };
-
-  // insert a label in the alphabet
-  void insert( Label l ) { if (!l.is_epsilon()) ls.insert(l); };
-
-  // insert the known symbols from another alphabet
-  void insert_symbols( const Alphabet& );
-
-  // insert the labels and known symbols from another alphabet
-  void copy( const Alphabet& );
-
-  // create the alphabet of a transducer obtained by a composition operation
-  void compose( const Alphabet &la, const Alphabet &ua );
-
-  // add a symbol to the alphabet and return its code
-  Character add_symbol(const char *symbol);
-
-  // add a symbol to the alphabet with a given code
-  void add_symbol(const char *symbol, Character c );
-
-  // create a new marker symbol and return its code
-  Character new_marker( void );
-  void delete_markers();
-
-  // compute the complement of a symbol set
-  void complement( std::vector<Character> &sym );
-  
-  // return the code of the argument symbol
-  int symbol2code( const char *s ) const { 
-      SymbolMap::const_iterator p = sm.find(s);
-      if (p != sm.end()) return p->second;
-      return EOF;
-  };
-
-  // return the symbol for the given symbol code
-  const char *code2symbol( Character c ) const {
-    CharMap::const_iterator p=cm.find(c);
-    if (p == cm.end())
-      return NULL;
-    else
-      return p->second;
-  };
-
-  // write the symbol for the given symbol code into a string
-  void write_char( Character c, char *buffer, int *pos,
-		   bool with_brackets=true ) const;
-
-  // write the symbol pair of a given label into a string
-  void write_label( Label l, char *buffer, int *pos,
-		    bool with_brackets=true ) const;
-
-  // write the symbol for the given symbol code into a buffer and return
-  // a pointer to it
-  // the flag "with_brackets" indicates whether the angle brackets
-  // surrounding multi-character symbols are to be printed or not
-  const char *write_char( Character c, bool with_brackets=true ) const;
-
-  // write the symbol pair of a given label into a string
-  // and return a pointer to it
-  const char *write_label( Label l, bool with_brackets=true ) const;
-
-  // scan the next multi-character symbol in the argument string
-  int next_mcsym( char*&, bool insert=true );
-
-  // scan the next symbol in the argument string
-  int next_code( char*&, bool extended=true, bool insert=true );
-
-  // convert a character string into a symbol or label sequence
-  void string2symseq( char*, std::vector<Character>& );
-  void string2labelseq( char*, std::vector<Label>& );
-
-  // scan the next label in the argument string
-  Label next_label( char*&, bool extended=true );
-
-  // store the alphabet in the argument file (in binary form)
-  void store( FILE* ) const;
-
-  // read the alphabet from the argument file
-  void read( FILE* );
-
-  // disambiguation and printing of analyses
-  int compute_score( Analysis &ana );
-  void disambiguate( std::vector<Analysis> &analyses );
-  char *print_analysis( Analysis &ana, bool both_layers );
-
-  friend std::ostream &operator<<(std::ostream&, const Alphabet&);
-};
-
-// write the alphabet to the output stream (in readable form)
-std::ostream &operator<<(std::ostream&, const Alphabet&);
-
-
-#endif

data/ext/sfst_machine/basic.cc

@@ -1,84 +0,0 @@
-
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     basic.C                                               */
-/*  MODULE   basic                                                 */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*  PURPOSE                                                        */
-/*                                                                 */
-/*******************************************************************/
-
-#include <stdlib.h>
-#include <string.h>
-
-#include "basic.h"
-
-bool Switch_Bytes=false;
-
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  fst_strdup                                                     */
-/*                                                                 */
-/*******************************************************************/
-
-char* fst_strdup(const char* pString)
-
-{
-  char* pStringCopy = (char*)malloc(strlen(pString) + 1);
-  if (pStringCopy == NULL) {
-    fprintf(stderr, "\nError: out of memory (malloc failed)\naborted.\n");
-    exit(1);
-  }
-  strcpy(pStringCopy, pString);
-  return pStringCopy;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  read_string                                                    */
-/*                                                                 */
-/*******************************************************************/
-
-int read_string( char *buffer, int size, FILE *file )
-
-{
-  for( int i=0; i<size; i++ ) {
-    int c=fgetc(file);
-    if (c == EOF || c == 0) {
-      buffer[i] = 0;
-      return (c==0);
-    }
-    buffer[i] = (char)c;
-  }
-  buffer[size-1] = 0;
-  return 0;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  read_num                                                       */
-/*                                                                 */
-/*******************************************************************/
-
-size_t read_num( void *p, size_t n, FILE *file )
-
-{
-  char *pp=(char*)p;
-  size_t result=fread( pp, 1, n, file );
-  if (Switch_Bytes) {
-    size_t e=n/2;
-    for( size_t i=0; i<e; i++ ) {
-      char tmp=pp[i];
-      pp[i] = pp[--n];
-      pp[n] = tmp;
-    }
-  }
-  return result;
-}
-

data/ext/sfst_machine/compact.cc

@@ -1,616 +0,0 @@
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     compact.C                                             */
-/*  MODULE   compact                                               */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*  PURPOSE  Code needed for analysing data                        */
-/*                                                                 */
-/*******************************************************************/
-
-#include <stdio.h>
-#include <math.h>
-
-#include <limits.h>
-
-#include "compact.h"
-
-using std::equal_range;
-using std::vector;
-using std::pair;
-
-const int BUFFER_SIZE=1000;
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::convert                                     */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::convert( CAnalysis &cana, Analysis &ana )
-
-{
-  ana.resize(cana.size());
-  for( size_t i=0; i<cana.size(); i++ )
-    ana[i] = label[cana[i]];
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::analyze                                     */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::analyze(unsigned int n, vector<Character> &input,
-			       size_t ipos, CAnalysis &ca, 
-			       vector<CAnalysis> &analyses )
-{
-  // "n" is the number of the current transducer node/state
-  // "input" is the sequence of input symbols
-  // "ipos" is the input position currently analysed
-  // "ca" stores the incomplete analysis string
-  // "analyses" stores the analyses found so far
-
-  if (analyses.size() > 10000)
-    return; // limit the maximal number of analyses
-
-  // Is the input string fully analyzed and the current node a final node?
-  if (finalp[n] && ipos == input.size())
-    // store the new analysis
-    analyses.push_back(ca);
-
-  // follow the epsilon transitions
-  // first_arc[n] is the number of the first outgoing transition of node n
-  // first_arc[n+1]-1 is the number of the last outgoing transition of node n
-  // first_arc[n+1] is the number of the first outgoing transition of node n+1
-  unsigned int i;
-  for( i=first_arc[n]; 
-       i<first_arc[n+1] && label[i].upper_char() == Label::epsilon; 
-       i++)
-    {
-      ca.push_back(i);
-      analyze(target_node[i], input, ipos, ca, analyses);
-      ca.pop_back();
-    }
-
-  // follow the non-epsilon transitions
-
-  // scan the next input symbol
-  if (ipos < input.size()) {
-    // find the set of arcs with matching upper character in the sorted list
-    pair<Label*,Label*>range = 
-      equal_range(label+i, label+first_arc[n+1], Label(input[ipos]));
-    unsigned int to = (unsigned int)(range.second - label);
-
-    // follow the non-epsilon transitions
-    for( i=range.first-label; i<to; i++) {
-      ca.push_back(i);
-      analyze(target_node[i], input, ipos+1, ca, analyses);
-      ca.pop_back();
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::analyze_string                              */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::analyze_string( char *s, vector<CAnalysis> &analyses )
-
-{
-  // "s" input string to be analyzed
-  // "analyses" is the data structure in which the results are stored
-  // and returned
-
-  vector<Character> input;
-  alphabet.string2symseq( s, input );
-
-  analyses.clear();
-  CAnalysis ca; // data structure where the current incomplete analysis
-                   // is stored
-  analyze(0, input, 0, ca, analyses); // start the analysis
-
-  if (analyses.size() > 10000)
-    fprintf(stderr,"Warning: Only the first 10000 analyses considered for \"%s\"!\n", s);
-  
-  if (simplest_only && analyses.size() > 1)
-    disambiguate( analyses ); // select the simplest analyses
-}
-
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::~CompactTransducer                          */
-/*                                                                 */
-/*******************************************************************/
-
-CompactTransducer::~CompactTransducer()
-
-{
-  delete[] finalp;
-  delete[] first_arc;
-  delete[] label;
-  delete[] target_node;
-  delete[] final_logprob;
-  delete[] arc_logprob;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::CompactTransducer                           */
-/*                                                                 */
-/*******************************************************************/
-
-CompactTransducer::CompactTransducer()
-
-{
-  both_layers = false;
-  simplest_only = false;
-  number_of_nodes = 0;
-  number_of_arcs = 0;
-  finalp = NULL;
-  first_arc = NULL;
-  label = NULL;
-  target_node = NULL;
-  arc_logprob = final_logprob = (float*)NULL;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::read_finalp                                 */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::read_finalp( FILE *file )
-
-{
-  int k=0;
-  unsigned char n=0;
-  for( size_t i=0; i<number_of_nodes; i++ ) {
-    if (k == 0) {
-      n = (unsigned char)fgetc(file);
-      k = 8;
-    }
-    k--;
-    if (n & (1 << k))
-      finalp[i] = 1;
-    else
-      finalp[i] = 0;
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::read_first_arcs                             */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::read_first_arcs( FILE *file )
-
-{
-  int k=0;
-  unsigned int n=0;
-  size_t bits=(size_t)ceil(log(number_of_arcs+1)/log(2));
-
-  for( size_t i=0; i<=number_of_nodes; i++ ) {
-    first_arc[i] = n >> (sizeof(n)*8 - bits);
-    n <<= bits;
-    k -= bits;
-    if (k < 0) {
-      read_num(&n,sizeof(n),file);
-      first_arc[i] |= n >> (sizeof(n)*8 + k);
-      n <<= -k;
-      k += sizeof(n) * 8;
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::read_target_nodes                           */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::read_target_nodes( FILE *file )
-
-{
-  int k=0;
-  unsigned int n=0;
-  size_t bits=(size_t)ceil(log(number_of_nodes)/log(2));
-
-  for( size_t i=0; i<number_of_arcs; i++ ) {
-    target_node[i] = n >> (sizeof(n)*8 - bits);
-    n <<= bits;
-    k -= bits;
-    if (k < 0) {
-      read_num(&n,sizeof(n),file);
-      target_node[i] |= n >> (sizeof(n)*8 + k);
-      n <<= -k;
-      k += sizeof(n) * 8;
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::read_labels                                 */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::read_labels( FILE *file )
-
-{
-  size_t N=0;
-  Label Num2Label[alphabet.size()];
-  for( Alphabet::const_iterator it=alphabet.begin();
-       it != alphabet.end(); it++ )
-    {
-      Label l=*it;
-      Num2Label[N++] = l;
-    }
-
-  int k=0;
-  unsigned int n=0;
-  size_t bits=(size_t)ceil(log(alphabet.size())/log(2));
-
-  for( size_t i=0; i<number_of_arcs; i++ ) {
-    unsigned int l = n >> (sizeof(n)*8 - bits);
-    n <<= bits;
-    k -= bits;
-    if (k < 0) {
-      read_num(&n,sizeof(n),file);
-      l |= n >> (sizeof(n)*8 + k);
-      n <<= -k;
-      k += sizeof(n) * 8;
-    }
-    label[i] = Num2Label[l];
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::read_probs                                  */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::read_probs( FILE *file )
-
-{
-  size_t n,m;
-  fread(&n, sizeof(n), 1, file);
-  if (fread(&m, sizeof(n), 1, file) != 1 ||
-      n != node_count() || m != arc_count())
-    {
-      fprintf(stderr,"Error: incompatible probability file!\n");
-      exit(1);
-    }
-  final_logprob = new float[n];
-  arc_logprob = new float[m];
-  fread(final_logprob, sizeof(float), n, file);
-  if (fread(arc_logprob, sizeof(float), n, file) != n) {
-    fprintf(stderr,"Error: in probability file!\n");
-    exit(1);
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::CompactTransducer                           */
-/*                                                                 */
-/*******************************************************************/
-
-CompactTransducer::CompactTransducer( FILE *file, FILE *pfile )
-
-{
-  both_layers = false;
-  simplest_only = false;
-
-  if (fgetc(file) != 'c')
-      throw "Error: wrong file format (not a compact transducer)\n";
-
-  alphabet.read(file);
-
-  read_num(&number_of_nodes,sizeof(number_of_nodes),file);
-  read_num(&number_of_arcs,sizeof(number_of_arcs),file);
-
-  if (!ferror(file)) {
-    // memory allocation
-    finalp = new char[number_of_nodes];
-    first_arc = new unsigned[number_of_nodes+1];
-    label = new Label[number_of_arcs];
-    target_node = new unsigned[number_of_arcs];
-    
-    // reading the data
-    read_finalp(file);
-    read_first_arcs(file);
-    read_labels(file);
-    read_target_nodes(file);
-  }
-
-  if (pfile == NULL)
-    arc_logprob = final_logprob = (float*)NULL;
-  else
-    read_probs(pfile);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::longest_match2                              */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::longest_match2(unsigned int n, char *string, int l, 
-				      CAnalysis &ca, int &bl, CAnalysis &ba)
-{
-  // n: transducer state
-  // string: rest string
-  // l: length of current analysis
-  // bl: length of the currently longest match
-  // ca: current analysis
-  // ba: best analysis
-
-  if (finalp[n] && l > bl) {
-    // store the new analysis
-    bl = l;
-    ba = ca; // copy the arc vector
-  }
-
-  // follow the epsilon transitions
-  unsigned int i;
-  for( i=first_arc[n]; 
-       i<first_arc[n+1] && label[i].upper_char() == Label::epsilon; 
-       i++)
-    {
-      ca.push_back(i);
-      longest_match2(target_node[i], string, l, ca, bl, ba);
-      ca.pop_back();
-    }
-
-  // follow the non-epsilon transitions
-  char *end=string;
-  int c=alphabet.next_code(end, false, false);
-  l += end-string;
-  if (c != EOF) {
-    // find the set of arcs with matching upper character in the sort list
-    pair<Label*,Label*>range = 
-      equal_range(label+i, label+first_arc[n+1], Label((Character)c));
-    unsigned int to = (unsigned int)(range.second - label);
-    for( i=range.first-label; i<to; i++) {
-      ca.push_back(i);
-      longest_match2(target_node[i], end, l, ca, bl, ba);
-      ca.pop_back();
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::print_analysis                              */
-/*                                                                 */
-/*******************************************************************/
-
-char *CompactTransducer::print_analysis( CAnalysis &cana )
-
-{
-  Analysis ana;
-  convert(cana, ana);
-  return alphabet.print_analysis( ana, both_layers );
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::longest_match                               */
-/*                                                                 */
-/*******************************************************************/
-
-const char *CompactTransducer::longest_match( char* &string )
-
-{
-  vector<char> analysis;
-  CAnalysis ca, ba;
-  int l=0;
-  longest_match2(0, string, 0, ca, l, ba);
-
-  // no match? return the next character
-  if (ba.size() == 0) {
-    int c=alphabet.next_code(string, false, false);
-    return alphabet.code2symbol(c);
-  }
-
-  string += l;
-  return print_analysis( ba );
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::disambiguate                                */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::disambiguate( vector<CAnalysis> &analyses )
-
-{
-  // compute the scores
-  int bestscore=INT_MIN;
-  vector<int> score;
-  Analysis ana;
-
-  for( size_t i=0; i<analyses.size(); i++ ) {
-    convert(analyses[i], ana);
-    score.push_back(alphabet.compute_score(ana));
-    if (bestscore < score[i])
-      bestscore = score[i];
-  }
-
-  // delete suboptimal analyses
-  size_t k=0;
-  for( size_t i=0; i<analyses.size(); i++ )
-    if (score[i] == bestscore)
-      analyses[k++] = analyses[i];
-  analyses.resize(k);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::train2                                      */
-/*                                                                 */
-/*******************************************************************/
-
-bool CompactTransducer::train2( char *s, vector<double> &arcfreq, 
-			       vector<double> &finalfreq )
-{
-  vector<CAnalysis> analyses;
-  vector<Label> input;
-  alphabet.string2labelseq( s, input );
-
-  CAnalysis ca; // data structure where the analysis is stored
-  unsigned int n=0;
-  bool failure=false;
-  for( size_t i=0; i<input.size(); i++ ) {
-    failure = true;
-    for( unsigned int k=first_arc[n]; k<first_arc[n+1]; k++) {
-      if (label[k] == input[i]) {
-	ca.push_back(k);
-	n = target_node[k];
-	failure = false;
-	break;
-      }
-    }
-    if (failure)
-      break;
-  }
-  if (failure || !finalp[n]) {
-    fprintf(stderr,"Warning: The following input is not covered:\n%s\n", s);
-    return false;
-  }
-
-  for( size_t k=0; k<ca.size(); k++ )
-    arcfreq[ca[k]]++;
-  finalfreq[target_node[ca.back()]]++;
-
-  return true;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::train                                       */
-/*                                                                 */
-/*******************************************************************/
-
-bool CompactTransducer::train( char *s, vector<double> &arcfreq, 
-			      vector<double> &finalfreq )
-{
-  vector<CAnalysis> analyses;
-  vector<Character> input;
-  alphabet.string2symseq( s, input );
-
-  CAnalysis ca; // data structure where the current incomplete analysis
-                   // is stored
-  analyze(0, input, 0, ca, analyses); // start the analysis
-
-  if (analyses.size() > 10000)
-    return true; // ignore inputs with more than 10000 analyses
-  else if (analyses.size() == 0)
-    return false;
-  
-  if (simplest_only && analyses.size() > 1)
-    disambiguate( analyses ); // select the simplest analyses
-
-  if (analyses.size() > 0) {
-    double incr = 1.0 / analyses.size();
-    CAnalysis arcs;
-
-    for( size_t i=0; i<analyses.size(); i++ ) {
-      CAnalysis &arcs=analyses[i];
-      for( size_t k=0; k<arcs.size(); k++ )
-	arcfreq[arcs[k]] += incr;
-      finalfreq[target_node[arcs.back()]] += incr;
-    }
-  }
-  return true;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::estimate_probs                              */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::estimate_probs( vector<double> &arcfreq, 
-				       vector<double> &finalfreq )
-{
-  // turn frequencies into probabilities
-  for( size_t n=0; n<finalfreq.size(); n++ ) {
-    double sum = finalfreq[n];
-    for( size_t a=first_arc[n]; a<first_arc[n+1]; a++ )
-      sum += arcfreq[a];
-    if (sum == 0.0)
-      sum = 1.0;
-    finalfreq[n] = finalfreq[n] / sum;
-    for( size_t a=first_arc[n]; a<first_arc[n+1]; a++ )
-      arcfreq[a] = arcfreq[a] / sum;
-  }
-}
-
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  CompactTransducer::compute_probs                               */
-/*                                                                 */
-/*******************************************************************/
-
-void CompactTransducer::compute_probs( vector<CAnalysis> &analyses, 
-				      vector<double> &prob )
-{
-  prob.resize(analyses.size());
-  double sum=0.0;
-  for( size_t i=0; i<analyses.size(); i++ ) {
-    CAnalysis &a=analyses[i];
-
-    // compute the probability
-    double logprob=0.0;
-    for( size_t k=0; k<a.size(); k++ )
-      logprob += arc_logprob[a[k]];
-    logprob += final_logprob[target_node[a.back()]];
-    prob[i] = exp(logprob);
-    sum += prob[i];
-  }
-
-  // sort the analyses
-  vector<CAnalysis> oldanalyses(analyses);
-  vector<double> oldprob(prob);
-  for( size_t i=0; i<analyses.size(); i++ ) {
-    prob[i] = -1.0;
-    int n=0;
-    for( size_t k=0; k<oldanalyses.size(); k++ )
-      if (prob[i] < oldprob[k]) {
-	prob[i] = oldprob[k];
-	n = k;
-      }
-    analyses[i] = oldanalyses[n];
-    oldprob[n] = -1.0;
-    prob[i] /= sum; // normalization
-  }
-}