ruby-sfst 0.4.3 → 0.4.4

data/ext/sfst_machine/compact.h

@@ -1,98 +0,0 @@
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     compact.h                                             */
-/*  MODULE   compact                                               */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*  PURPOSE  finite state tools                                    */
-/*                                                                 */
-/*******************************************************************/
-
-#ifndef _COMPACT_H_
-#define _COMPACT_H_
-
-#include "alphabet.h"
-
-#include <vector>
-
-typedef std::vector<unsigned int> CAnalysis;
-    
-class CompactTransducer {
-    
-protected:
-
-  // the following data structures are used to store the nodes
-
-  unsigned int number_of_nodes; // number of nodes in the transducer
-  char *finalp;  // finalp[i] is 1 if node i is final and 0 otherwise
-  unsigned int *first_arc;  // first_arc[i] is the number of the first
-			    // arc outgoing from node i
-
-  // the following data structures are used to store the transition arcs
-
-  unsigned int number_of_arcs; // total number of arcs in the transducer
-  Label *label;              // the label (character pair) of arc i
-  unsigned int *target_node; // target node of arc i
-
-  // the following data structures are used to store the stochastic parameters
-  float *final_logprob;
-  float *arc_logprob;
-
-  // functions needed to read the transducer from a file
-
-  void read_finalp( FILE *file );
-  void read_first_arcs( FILE *file );
-  void read_target_nodes( FILE *file );
-  void read_labels( FILE *file );
-  void read_probs( FILE *file );
-
-  // functions needed to analyze data with the transducer
-
-  void analyze( unsigned int n, std::vector<Character> &ch, size_t ipos,
-		CAnalysis&, std::vector<CAnalysis>&);
-
-  // function selecting the simplest morphological analysis
-
-  int compute_score( CAnalysis &ana );
-  void disambiguate( std::vector<CAnalysis> &analyses );
-
-  // functions for longest-match analysis of input data
-
-  void longest_match2(unsigned int, char*, int, CAnalysis&, int&, CAnalysis&);
-
-  void convert( CAnalysis &cana, Analysis &ana );
-  
-public:
-  size_t node_count() { return number_of_nodes; };
-  size_t arc_count() { return number_of_arcs; };
-
-  bool both_layers;   // print surface and analysis symbols
-  bool simplest_only; // print only the simplest analyses
-
-  Alphabet alphabet;  // data structure which maps symbols to numeric codes
-  CompactTransducer(); // dummy constructor
-  CompactTransducer( FILE*, FILE *pfile=NULL ); // reads a (stochastic) transducer
-  ~CompactTransducer();  // destroys a transducer
-  
-  // the analysis function returns the set of analyses for the string "s"
-  // in the argument "analyses"
-  void analyze_string( char *s, std::vector<CAnalysis > &analyses );
-
-  void compute_probs( std::vector<CAnalysis> &analyses, std::vector<double> &prob );
-  char *print_analysis( CAnalysis &ana );
-  
-  // longest-match analysis
-  const char *longest_match( char*& );
-
-  // EM training
-  bool train2( char *s, std::vector<double> &arcfreq, std::vector<double> &finalfreq );
-  bool train( char *s, std::vector<double> &arcfreq, std::vector<double> &finalfreq );
-  void estimate_probs( std::vector<double> &arcfreq, std::vector<double> &finalfreq );
-
-  // robust analysis
-  float robust_analyze_string( char *string, std::vector<CAnalysis> &analyses,
-			       float ErrorsAllowed );
-};
-
-#endif

data/ext/sfst_machine/determinise.cc

@@ -1,303 +0,0 @@
-
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     determinise.C                                         */
-/*  MODULE   determinise                                           */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*******************************************************************/
-
-
-#include "fst.h"
-
-using std::vector;
-using std::pair;
-using std::set;
-
-/*****************  class NodeSet  *********************************/
-
-class NodeSet {
-  // This class is used to store a set of nodes.
-  // Whenever a new node is added, all nodes accessible
-  // through epsilon transitions are added as well.
-
- private:
-  set<Node*> ht;
-  
- public:
-  typedef set<Node*>::iterator iterator;
-  NodeSet() {};
-  void add( Node* );
-  bool insert(Node *node) { 
-    pair<iterator, bool> result = ht.insert(node);
-    return result.second;
-  };
-  iterator begin() const { return ht.begin(); }
-  iterator end() const { return ht.end(); }
-  size_t size() const { return ht.size(); }
-  void clear() { ht.clear(); }
-};
-
-
-/*****************  class NodeArray  *******************************/
-
-class NodeArray {
-
-private:
-  size_t sizev;
-  bool final;
-  Node **node;
-
-public:
-  NodeArray( NodeSet& );
-  ~NodeArray() { delete[] node; };
-  size_t size() const { return sizev; }
-  bool is_final() const { return final; };
-  Node* &operator[]( int i ) const { return node[i]; }
-};
-
-
-/*****************  class Transition  ******************************/
-
-class Transition {
-public:
-  Label label;
-  NodeArray *nodes;
-  Transition(Label l, NodeArray *na) { label = l; nodes = na; };
-};
-
-
-/*****************  class NodeMapping  ****************************/
-
-class NodeMapping {
-  // This class is used to map a node set from one transducer
-  // to a single node in another transducer
-
- private:
-  struct hashf {
-    size_t operator()(const NodeArray *na) const { 
-      size_t key=na->size() ^ na->is_final();
-      for( size_t i=0; i<na->size(); i++)
-	key = (key<<1) ^ (size_t)(*na)[i];
-      return key;
-    }
-  };
-  struct equalf {
-    int operator()(const NodeArray *na1, const NodeArray *na2) const {
-      if (na1->size() != na2->size() || na1->is_final() != na2->is_final())
-	return 0;
-      for( size_t i=0; i<na1->size(); i++)
-	if ((*na1)[i] != (*na2)[i])
-	  return 0;
-      return 1;
-    }
-  };
-  typedef hash_map<NodeArray*, Node*, hashf, equalf> NodeMap;
-  NodeMap hm;
-  
- public:
-  typedef NodeMap::iterator iterator;
-  ~NodeMapping();
-  iterator begin() { return hm.begin(); };
-  iterator end() { return hm.end(); };
-  iterator find( NodeArray *na) { return hm.find( na ); };
-  Node* &operator[]( NodeArray *na ) { return hm.operator[](na); };
-  
-};
-
-
-/*****************  class LabelMapping  ****************************/
-
-class LabelMapping {
-  // This class is used to map a label to a node set
-
- private:
-  struct hashf {
-    size_t operator()(const Label l) const { 
-      return l.lower_char() | (l.upper_char() << 16);
-    }
-  };
-  struct equalf {
-    int operator()(const Label l1, const Label l2) const {
-      return l1==l2;
-    }
-  };
-  typedef hash_map<const Label, NodeSet, hashf, equalf> LabelMap;
-  LabelMap lm;
-  
- public:
-  LabelMapping(): lm(8) {};
-  typedef LabelMap::iterator iterator;
-  iterator begin() { return lm.begin(); };
-  iterator end() { return lm.end(); };
-  size_t   size() { return lm.size(); };
-  iterator find( Label l) { return lm.find( l ); };
-  NodeSet &operator[]( const Label l ) { return lm.operator[]( l ); };
-  
-};
-
-static void determinise_node( NodeArray&, Node*, Transducer*, NodeMapping&, long );
-
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  NodeSet::add                                                   */
-/*                                                                 */
-/*******************************************************************/
-
-void NodeSet::add( Node *node )
-
-{
-  pair<iterator, bool> result = ht.insert(node);
-  if (result.second) {
-    // new node, add nodes reachable with epsilon transitions
-    for( ArcsIter p(node->arcs(),ArcsIter::eps); p; p++ ) {
-      Arc *arc=p;
-      if (!arc->label().is_epsilon())
-	break;
-      add(arc->target_node());
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  NodeArray::NodeArray                                           */
-/*                                                                 */
-/*******************************************************************/
-
-NodeArray::NodeArray( NodeSet &ns )
-
-{
-  sizev = 0;
-  NodeSet::iterator it;
-
-  final = false;
-  node = new Node*[ns.size()];
-  for( it=ns.begin(); it!=ns.end(); it++ ) {
-    Node *nn = *it;
-    if (nn->arcs()->non_epsilon_transition_exists())
-      node[sizev++] = nn;
-    final |= nn->is_final();
-  }
-  std::sort(node, node+sizev);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  NodeMapping::~NodeMapping                                      */
-/*                                                                 */
-/*******************************************************************/
-
-NodeMapping::~NodeMapping()
-
-{
-  // if we delete NodeArrays without removing them from NodeMapping,
-  // the system will crash when NodeMapping is deleted.
-  for( iterator it=hm.begin(); it!=hm.end(); ) {
-    NodeArray *na=it->first;
-    iterator old = it++;
-    hm.erase(old);
-    delete na;
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  compute_transitions                                            */
-/*                                                                 */
-/*******************************************************************/
-
-static void compute_transitions( NodeArray &na, vector<Transition> &t )
-
-{
-  LabelMapping lmap;
-    
-  // for all nodes in the current set
-  for( size_t i=0; i<na.size(); i++) {
-    Node *n = na[i];    // old node
-    
-    // For each non-epsilon transition, add the target node
-    // to the respective node set.
-    for( ArcsIter p(n->arcs(),ArcsIter::non_eps); p; p++ ) {
-      Arc *arc=p;
-      lmap[arc->label()].add(arc->target_node());
-    }
-  }
-  
-  t.reserve(lmap.size());
-  for( LabelMapping::iterator it=lmap.begin(); it!=lmap.end(); it++ )
-    t.push_back(Transition(it->first, new NodeArray( it->second )));
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  determinise_node                                               */
-/*                                                                 */
-/*******************************************************************/
-
-static void determinise_node( NodeArray &na, Node *node, Transducer *a, 
-			      NodeMapping &map, long depth )
-{
-  if (depth > 10000)
-    fprintf(stderr,"\r%ld",depth);
-  node->set_final(na.is_final());
-
-  vector<Transition> t;
-  compute_transitions( na, t );
-
-  for( size_t i=0; i<t.size(); i++ ) {
-    NodeMapping::iterator it=map.find(t[i].nodes);
-    if (it == map.end()) {
-      // new node set
-      Node *target_node = a->new_node();
-      map[t[i].nodes] = target_node;
-      node->add_arc( t[i].label, target_node, a );
-      determinise_node( *t[i].nodes, target_node, a, map, depth+1 );
-    }
-    else {
-      delete t[i].nodes;
-      node->add_arc( t[i].label, it->second, a );
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::determinise                                        */
-/*                                                                 */
-/*******************************************************************/
-
-Transducer &Transducer::determinise()
-
-{
-  // initialisations
-  NodeMapping map;
-
-  Transducer *a = new Transducer();
-  a->alphabet.copy(alphabet);
-
-  // creation of the initial node set consisting of all nodes
-  // reachable from the start node via epsilon transitions.
-  NodeArray *na;
-  {
-    NodeSet ns;
-    ns.add(root_node());
-    na = new NodeArray(ns);
-  }
-
-  // map the node set to the new root node
-  map[na] = a->root_node();
-
-  // determinise the transducer recursively
-  determinise_node( *na, a->root_node(), a, map, 0);
-  a->deterministic = 1;
-  return *a;
-}

data/ext/sfst_machine/fst.cc

@@ -1,1000 +0,0 @@
-
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     fst.C                                                 */
-/*  MODULE   fst                                                   */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*  PURPOSE  basic FST functions                                   */
-/*                                                                 */
-/*******************************************************************/
-
-#include "fst.h"
-
-using std::vector;
-using std::istream;
-using std::ostream;
-using std::cerr;
-
-const int BUFFER_SIZE=100000;
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Arcs::size                                                     */
-/*                                                                 */
-/*******************************************************************/
-
-int Arcs::size() const
-
-{
-  int n=0;
-  for( Arc *p=first_arcp; p; p=p->next ) n++;
-  for( Arc *p=first_epsilon_arcp; p; p=p->next ) n++;
-  return n;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Arcs::target_node                                              */
-/*                                                                 */
-/*******************************************************************/
-
-Node *Arcs::target_node( Label l )
-
-{
-  Arc *arc;
-
-  for( arc=first_arcp; arc; arc=arc->next)
-    if (arc->label() == l)
-      return arc->target_node();
-  
-  return NULL;
-}
-
-const Node *Arcs::target_node( Label l ) const
-
-{
-  const Arc *arc;
-
-  for( arc=first_arcp; arc; arc=arc->next)
-    if (arc->label() == l)
-      return arc->target_node();
-  
-  return NULL;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Arcs::add_arc                                                  */
-/*                                                                 */
-/*******************************************************************/
-
-void Arcs::add_arc( Label l, Node *node, Transducer *a )
-
-{
-  Arc *arc=a->new_arc( l, node );
-
-  if (l.is_epsilon()) {
-    arc->next = first_epsilon_arcp;
-    first_epsilon_arcp = arc;
-  }
-  else {
-    arc->next = first_arcp;
-    first_arcp = arc;
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Arcs::remove_arc                                               */
-/*                                                                 */
-/*******************************************************************/
-
-int Arcs::remove_arc( Arc *arc )
-
-{
-  Arc **p = (arc->label().is_epsilon()) ? &first_epsilon_arcp : &first_arcp;
-  for( ; *p; p=&(*p)->next )
-    if (*p == arc) {
-      *p = arc->next;
-      return 1;
-    }
-  return 0;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Node::init                                                     */
-/*                                                                 */
-/*******************************************************************/
-
-void Node::init()
-
-{
-  final = false;
-  visited = 0;
-  arcsp.init();
-  forwardp = NULL;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Node::clear_visited                                            */
-/*                                                                 */
-/*******************************************************************/
-
-void Node::clear_visited( NodeHashSet &nodeset )
-
-{
-  if (nodeset.find( this ) == nodeset.end()) {
-    visited = 0;
-    nodeset.insert( this );
-    fprintf(stderr," %lu", nodeset.size());
-    for( ArcsIter p(arcs()); p; p++ ) {
-      Arc *arc=p;
-      arc->target_node()->clear_visited( nodeset );
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  NodeNumbering::number_node                                     */
-/*                                                                 */
-/*******************************************************************/
-
-void NodeNumbering::number_node( Node *node, Transducer &a )
-
-{
-  if (!node->was_visited( a.vmark )) {
-    nummap[node] = nodes.size();
-    nodes.push_back(node);
-    for( ArcsIter p(node->arcs()); p; p++ ) {
-      Arc *arc=p;
-      number_node( arc->target_node(), a );
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  NodeNumbering::NodeNumbering                                   */
-/*                                                                 */
-/*******************************************************************/
-
-NodeNumbering::NodeNumbering( Transducer &a )
-
-{
-  a.incr_vmark();
-  number_node( a.root_node(), a );
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::new_node                                           */
-/*                                                                 */
-/*******************************************************************/
-
-Node *Transducer::new_node()
-
-{
-  Node *node=(Node*)mem.alloc( sizeof(Node) );
-
-  node->init();
-  return node;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::new_arc                                            */
-/*                                                                 */
-/*******************************************************************/
-
-Arc *Transducer::new_arc( Label l, Node *target )
-
-{
-  Arc *arc=(Arc*)mem.alloc( sizeof(Arc) );
-
-  arc->init( l, target);
-  return arc;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::add_string                                         */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::add_string( char *s, bool extended, Alphabet *a )
-
-{
-  if (a == NULL)
-    a = &alphabet;
-
-  Node *node=root_node();
-  Label l;
-  while (!(l = a->next_label(s, extended)).is_epsilon()) {
-    a->insert(l);
-    Arcs *arcs=node->arcs();
-    node = arcs->target_node( l );
-    if (node == NULL) {
-      node = new_node();
-      arcs->add_arc( l, node, this );
-    }
-  }
-  node->set_final(1);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::Transducer                                         */
-/*                                                                 */
-/*******************************************************************/
-
-Transducer::Transducer( vector<Label> &path )
-  : root(), mem()
-{
-  Node *node=root_node();
-
-  vmark = 0;
-  deterministic = minimised = true;
-  for( size_t i=0; i<path.size(); i++ ) {
-    Arcs *arcs=node->arcs();
-    node = new_node();
-    arcs->add_arc( path[i], node, this );
-  }
-  node->set_final(1);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::Transducer                                         */
-/*                                                                 */
-/*******************************************************************/
-
-Transducer::Transducer( istream &is, const Alphabet *a, bool verbose  )
-  : root(), mem()
-{
-  bool extended=false;
-  int n=0;
-  char buffer[10000];
-
-  vmark = 0;
-  deterministic = true;
-  minimised = false;
-  if (a) {
-    alphabet.copy(*a);
-    extended = true;
-  }
-  while (is.getline(buffer, 10000)) {
-    if (verbose && ++n % 10000 == 0) {
-      if (n == 10000)
-	cerr << "\n";
-      cerr << "\r" << n << " words";
-    }
-    // delete final whitespace characters
-    int l;
-    for( l=strlen(buffer)-1; l>=0; l-- )
-      if ((buffer[l] != ' ' && buffer[l] != '\t' && buffer[l] != '\r') ||
-	  (l > 0 && buffer[l-1] == '\\'))
-	break;
-    buffer[l+1] = 0;
-
-    add_string(buffer, extended);
-  }
-  if (verbose && n >= 10000)
-    cerr << "\n";
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::Transducer                                         */
-/*                                                                 */
-/*******************************************************************/
-
-Transducer::Transducer( char *s, const Alphabet *a, bool extended )
-  : root(), mem()
-{
-  vmark = 0;
-  deterministic = minimised = true;
-  if (a)
-    alphabet.copy(*a);
-  add_string(s, extended);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::clear                                              */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::clear()
-
-{
-  vmark = 0;
-  deterministic = minimised = false;
-  root.init();
-  mem.clear();
-  alphabet.clear();
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::store_symbols                                      */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::store_symbols(Node *node, SymbolMap &symbol, LabelSet &labels)
-
-{
-  if (!node->was_visited( vmark )) {
-    Arcs *arcs=node->arcs();
-    for( ArcsIter p(arcs); p; p++ ) {
-      Arc *arc=p;
-      Label l=arc->label();
-
-      labels.insert(l);
-
-      Character c = l.upper_char();
-      if (symbol.find(c) == symbol.end()) {
-	const char *s = alphabet.code2symbol(c);
-	if (s)
-	  symbol[c] = fst_strdup(s);
-      }
-
-      c = l.lower_char();
-      if (symbol.find(c) == symbol.end()) {
-	const char *s = alphabet.code2symbol(c);
-	if (s)
-	  symbol[c] = fst_strdup(s);
-      }
-
-      store_symbols( arc->target_node(), symbol, labels );
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::minimise_alphabet                                  */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::minimise_alphabet()
-
-{
-  SymbolMap symbols;
-  LabelSet labels;
-  incr_vmark();
-  store_symbols(root_node(), symbols, labels);
-  alphabet.clear();
-  for( SymbolMap::iterator it=symbols.begin(); it!=symbols.end(); it++ ) {
-    alphabet.add_symbol( it->second, it->first );
-    free(it->second);
-  }
-  for( LabelSet::iterator it=labels.begin(); it!=labels.end(); it++ )
-    alphabet.insert(*it);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::minimise                                           */
-/*                                                                 */
-/*******************************************************************/
-
-Transducer &Transducer::minimise( bool verbose )
-
-{
-  if (minimised)
-    return copy();
-
-  Transducer *a1, *a2;
-
-  a1 = &reverse();
-  a2 = &a1->determinise();
-  delete a1;
-
-  a1 = &a2->reverse();
-  delete a2;
-
-  a2 = &a1->determinise();
-  delete a1;
-
-  a2->minimised = true;
-  a2->minimise_alphabet();
-
-  return *a2;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::enumerate_paths_node                               */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::enumerate_paths_node( Node *node, vector<Label> &path, 
-				       NodeHashSet &previous,
-				       vector<Transducer*> &result )
-{
-  if (node->is_final())
-    result.push_back(new Transducer(path));
-
-  for( ArcsIter it_arc(node->arcs()); it_arc; it_arc++ ) {
-    Arc *arc=it_arc;
-
-    NodeHashSet::iterator it_node=previous.insert(node).first;
-    path.push_back(arc->label());
-    enumerate_paths_node( arc->target_node(), path, previous, result );
-    path.pop_back();
-    previous.erase(it_node);
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::enumerate_paths                                    */
-/*                                                                 */
-/*******************************************************************/
-
-bool Transducer::enumerate_paths( vector<Transducer*> &result )
-
-{
-  if (is_infinitely_ambiguous())
-    return true;
-  for( size_t i=0; i<result.size(); i++ )
-    delete result[i];
-  result.clear();
-
-  vector<Label> path;
-  NodeHashSet previous;
-  enumerate_paths_node( root_node(), path, previous, result );
-  return false;
-}
-
-
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::print_strings_node                                 */
-/*                                                                 */
-/*******************************************************************/
-
-int Transducer::print_strings_node(Node *node, char *buffer, int pos,
-				  FILE *file, bool with_brackets )
-{
-  int result = 0;
-
-  if (node->was_visited( vmark )) {
-    if (node->forward() != NULL) { // cycle detected
-      cerr << "Warning: cyclic analyses (cycle aborted)\n";
-      return 0;
-    }
-    node->set_forward(node);  // used like a flag for loop detection
-  }
-  if (pos == BUFFER_SIZE)
-    throw "Output string in function print_strings_node is too long";
-  if (node->is_final()) {
-    buffer[pos] = '\0';
-    fprintf(file,"%s\n", buffer);
-    result = 1;
-  }
-  for( ArcsIter i(node->arcs()); i; i++ ) {
-    int p=pos;
-    Arc *arc=i;
-    Label l=arc->label();
-    alphabet.write_label(l, buffer, &p, with_brackets);
-    result |= print_strings_node(arc->target_node(), buffer, p, 
-				 file, with_brackets );
-  }
-  node->set_forward(NULL);
-
-  return result;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::print_strings                                      */
-/*                                                                 */
-/*******************************************************************/
-
-int Transducer::print_strings( FILE *file, bool with_brackets )
-
-{
-  char buffer[BUFFER_SIZE];
-  incr_vmark();
-  return print_strings_node( root_node(), buffer, 0, file, with_brackets );
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::analyze_string                                     */
-/*                                                                 */
-/*******************************************************************/
-
-bool Transducer::analyze_string( char *string, FILE *file, bool with_brackets )
-
-{
-  vector<Character> input;
-  alphabet.string2symseq( string, input );
-  vector<Label> labels;
-  for( size_t i=0; i<input.size(); i++ )
-    labels.push_back(Label(input[i]));
-
-  Transducer a1(labels);
-  Transducer *a2=&(*this || a1);
-  Transducer *a3=&(a2->lower_level());
-  delete a2;
-  a2 = &a3->minimise();
-  delete a3;
-
-  a2->alphabet.copy(alphabet);
-  bool result = a2->print_strings( file, with_brackets );
-  delete a2;
-  return result;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::generate_string                                    */
-/*                                                                 */
-/*******************************************************************/
-
-bool Transducer::generate_string( char *string, FILE *file, bool with_brackets)
-
-{
-  Transducer a1(string, &alphabet, false);
-  Transducer *a2=&(a1 || *this);
-  Transducer *a3=&(a2->upper_level());
-  delete a2;
-  a2 = &a3->minimise();
-  delete a3;
-
-  a2->alphabet.copy(alphabet);
-  bool result = a2->print_strings( file, with_brackets );
-  delete a2;
-  return result;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  complete                                                       */
-/*                                                                 */
-/*******************************************************************/
-
-static void complete( Node *node, Alphabet &alphabet, int vmark)
-
-{
-  if (node->was_visited( vmark ))
-    return;
-  for( ArcsIter p(node->arcs()); p; p++ ) {
-    Arc *arc=p;
-    if (!arc->label().is_epsilon())
-      alphabet.insert(arc->label());
-    complete(arc->target_node(), alphabet, vmark);
-  }  
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::complete_alphabet                                  */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::complete_alphabet()
-
-{
-  incr_vmark();
-  complete(root_node(), alphabet, vmark);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  print_node                                                     */
-/*                                                                 */
-/*******************************************************************/
-
-static void print_node( ostream &s, Node *node, NodeNumbering &index, 
-			long vmark, Alphabet &abc )
-
-{
-  if (!node->was_visited( vmark )) {
-    Arcs *arcs=node->arcs();
-    for( ArcsIter p(arcs); p; p++ ) {
-      Arc *arc=p;
-      s << index[node] << "\t" << index[arc->target_node()];
-      s << "\t" << abc.write_char(arc->label().lower_char());
-      s << "\t" << abc.write_char(arc->label().upper_char());
-      s << "\n";
-    }
-    if (node->is_final())
-      s << index[node] << "\n";
-    for( ArcsIter p(arcs); p; p++ ) {
-      Arc *arc=p;
-      print_node( s, arc->target_node(), index, vmark, abc );
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  operator<<                                                     */
-/*                                                                 */
-/*******************************************************************/
-
-ostream &operator<<( ostream &s, Transducer &a )
-
-{
-  NodeNumbering index(a);
-  a.incr_vmark();
-  print_node( s, a.root_node(), index, a.vmark,  a.alphabet );
-  return s;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  store_node_info                                                */
-/*                                                                 */
-/*******************************************************************/
-
-static void store_node_info( FILE *file, Node *node )
-
-{
-  // write final flag
-  char c=node->is_final();
-  fwrite(&c,sizeof(c),1,file);
-  
-  // write the number of arcs
-  int nn = node->arcs()->size();
-  if (nn > 65535)
-    throw "Error: in function store_node\n";
-  unsigned short n=(unsigned short)nn;
-  fwrite(&n,sizeof(n),1,file);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  store_arc_label                                                */
-/*                                                                 */
-/*******************************************************************/
-
-static void store_arc_label( FILE *file, Arc *arc )
-
-{
-  Label l=arc->label();
-  Character lc=l.lower_char();
-  Character uc=l.upper_char();
-  fwrite(&lc,sizeof(lc),1,file);
-  fwrite(&uc,sizeof(uc),1,file);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  store_node                                                     */
-/*                                                                 */
-/*******************************************************************/
-
-static void store_node( FILE *file, Node *node, NodeNumbering &index, 
-			long vmark )
-{
-  if (!node->was_visited( vmark )) {
-
-    store_node_info( file, node );
-  
-    // write the arcs
-    for( ArcsIter p(node->arcs()); p; p++ ) {
-      Arc *arc=p;
-      store_arc_label( file, arc );
-      unsigned int t=index[arc->target_node()];
-      fwrite(&t,sizeof(t),1,file);
-      store_node(file, arc->target_node(), index, vmark );
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  store_lowmem_node                                              */
-/*                                                                 */
-/*******************************************************************/
-
-static void store_lowmem_node( FILE *file, Node *node, NodeNumbering &index,
-			       vector<unsigned int> &startpos)
-{
-  store_node_info( file, node );
-  
-  // write the arcs
-  for( ArcsIter p(node->arcs()); p; p++ ) {
-    Arc *arc=p;
-    store_arc_label( file, arc );
-    unsigned int t=startpos[index[arc->target_node()]];
-    fwrite(&t,sizeof(t),1,file);
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::store_lowmem                                       */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::store_lowmem( FILE *file )
-
-{
-  fputc('l',file);
-  alphabet.store(file);
-
-  // storing size of index table
-  NodeNumbering index(*this);
-
-  // compute the start position of the first node
-  unsigned int pos=(unsigned int)ftell(file);
-  vector<unsigned int> startpos;
-  for( size_t i=0; i<index.number_of_nodes(); i++ ) {
-    startpos.push_back(pos);
-    Node *node=index.get_node(i);
-    Arcs *arcs=node->arcs();
-    pos += sizeof(char) // size of final flag
-      + sizeof(unsigned short) // size of number of arcs
-      + arcs->size() * (sizeof(Character) * 2 + sizeof(unsigned int)); // size of n arcs
-  }
-
-  // storing nodes
-  for( size_t i=0; i<index.number_of_nodes(); i++ )
-    store_lowmem_node( file, index.get_node(i), index, startpos );
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::store                                              */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::store( FILE *file )
-
-{
-  fputc('a',file);
-
-  NodeNumbering index(*this);
-  incr_vmark();
-  unsigned int n=index.number_of_nodes();
-  fwrite(&n,sizeof(n),1,file);
-  store_node( file, root_node(), index, vmark );
-
-  alphabet.store(file);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  read_node                                                      */
-/*                                                                 */
-/*******************************************************************/
-
-static void read_node( FILE *file, Node *node, Node **p, Transducer *a )
-{
-  char c;
-  fread(&c,sizeof(c),1,file);
-  node->set_final(c);
-
-  unsigned short n;
-  fread( &n, sizeof(n), 1, file);
-
-  for( int i=0; i<n; i++ ) {
-    Character lc,uc;
-    unsigned int t;
-    fread(&lc,sizeof(lc),1,file);
-    fread(&uc,sizeof(uc),1,file);
-    fread(&t,sizeof(t),1,file);
-    if (ferror(file))
-      throw "Error encountered while reading transducer from file";
-    if (p[t])
-      node->add_arc( Label(lc,uc), p[t], a );
-    else {
-      p[t] = a->new_node();
-      node->add_arc( Label(lc,uc), p[t], a );
-      read_node(file, p[t], p, a );
-    }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::read_transducer_binary                             */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::read_transducer_binary( FILE *file )
-
-{
-  if (fgetc(file) != 'a')
-    throw "Error: wrong file format (not a standard transducer)\n";
-
-  vmark = deterministic = 0;
-  unsigned int n;
-  fread(&n,sizeof(n),1,file); // number of nodes
-  if (ferror(file))
-    throw "Error encountered while reading transducer from file";
-
-  Node **p=new Node*[n];  // maps indices to nodes
-  p[0] = root_node();
-  for( unsigned int i=1; i<n; i++)
-    p[i] = NULL;
-  read_node( file, root_node(), p, this );
-  delete[] p;
-
-  alphabet.read(file);
-
-  vmark = 1;
-  deterministic = minimised = 1;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  error_message                                                  */
-/*                                                                 */
-/*******************************************************************/
-
-static void error_message( size_t line )
-
-{
-  static char message[1000];
-  sprintf(message, "Error: in line %u of text transducer file", 
-	  (unsigned int)line);
-  throw message;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::create_node                                        */
-/*                                                                 */
-/*******************************************************************/
-
-Node *Transducer::create_node( vector<Node*> &node, char *s, size_t line )
-
-{
-  char *p;
-  long n = strtol(s, &p, 10);
-
-  if (s == p || n < 0)
-    error_message( line );
-  if ((long)node.size() <= n)
-    node.resize(n+1, NULL);
-  if (node[n] == NULL)
-    node[n] = new Node;
-
-  return node[n];
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  next_string                                                    */
-/*                                                                 */
-/*******************************************************************/
-
-static char *next_string( char* &s, size_t line )
-
-{
-  // scan the input up to the next tab or newline character
-  // and unquote symbols preceded by a backslash
-  char *p = s;
-  char *q = s;
-  while (*q!=0 && *q!='\t' && *q!='\n' && *q!='\r') {
-    if (*q == '\\')
-      q++;
-    *(p++) = *(q++);
-  }
-  if (p == s)
-    error_message(line); // no string found
-
-  char *result=s;
-  // skip over following whitespace
-  while (*q == ' ' || *q == '\t' || *q == '\n' || *q == '\r')
-    q++;
-  
-  if (*q == 0)
-    s = NULL; // end of string was reached
-  else
-    s = q;  // move the string pointer s
-
-  *p = 0; // mark the end of the result string
-  
-  return result;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::read_transducer_text                               */
-/*                                                                 */
-/*******************************************************************/
-
-void Transducer::read_transducer_text( FILE *file )
-
-{
-  vector<Node*> nodes;
-  nodes.push_back(root_node());
-
-  vmark = deterministic = 0;
-  char buffer[10000];
-  for( size_t line=0; fgets(buffer, 10000, file ); line++ ) {
-    char *p = buffer;
-    char *s = next_string(p, line);
-    Node *node = create_node( nodes, s, line );
-    if (p == NULL)
-      node->set_final(true);
-    else {
-      s = next_string(p, line);
-      Node *target = create_node( nodes, s, line );
-
-      s = next_string(p, line);
-      Character lc = alphabet.add_symbol(s);
-      s = next_string(p, line);
-      Character uc = alphabet.add_symbol(s);
-      Label l(lc,uc);
-      if (l == Label::epsilon)
-	error_message( line );
-
-      alphabet.insert(l);
-      node->add_arc( l, target, this );
-    }
-  }
-
-  vmark = 1;
-  deterministic = minimised = 1;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  Transducer::Transducer                                         */
-/*                                                                 */
-/*******************************************************************/
-
-Transducer::Transducer( FILE *file, bool binary )
-
-{
-  if (binary)
-    read_transducer_binary( file );
-  else
-    read_transducer_text( file );
-}
-