ruby-sfst 0.1.1 → 0.2.0

data/CHANGELOG

@@ -1,2 +1,3 @@
+v0.2.0. update to SFST 1.3
 v0.1.1. overflow and g++ 4.3 fixes
 v0.1.0. initial release

data/Manifest

@@ -11,6 +11,7 @@ ext/sfst_machine/utf8-scanner.ll
 ext/sfst_machine/determinise.C
 ext/sfst_machine/interface.C
 ext/sfst_machine/compact.h
+ext/sfst_machine/sgi.h
 ext/sfst_machine/basic.h
 ext/sfst_machine/fst.h
 ext/sfst_machine/make-compact.h

data/README.rdoc

@@ -2,6 +2,8 @@
 
 A wrapper for the Stuttgart Finite State Transducer Tools (SFST).
 
+The wrapper is based on SFST 1.3.
+
 == Installation
 
   gem install ruby-sfst

data/Rakefile

@@ -4,7 +4,7 @@ require 'rake'
 begin
   require 'echoe'
  
-  Echoe.new('ruby-sfst', '0.1.1') do |p|
+  Echoe.new('ruby-sfst', '0.2.0') do |p|
     p.summary = "A wrapper for the Stuttgart Finite State Transducer Tools (SFST)."
     p.author = 'Marius L. Jøhndal'
     p.email = "mariuslj (at) ifi [dot] uio (dot) no"

data/ext/sfst_machine/alphabet.C

@@ -10,14 +10,14 @@
 /*                                                                 */
 /*******************************************************************/
 
-#include <limits.h>
+#include <climits>
+#include <cstring>
+
 #include "utf8.h"
 #include "alphabet.h"
 
 using std::vector;
 using std::ostream;
-using __gnu_cxx::hash_map;
-using __gnu_cxx::hash_set;
 
 const int BUFFER_SIZE=100000;
 
@@ -425,13 +425,13 @@ ostream &operator<<( ostream &s, const Alphabet &a )
 /*  Alphabet::next_mcsym                                           */
 /*                                                                 */
 /*  recognizes multi-character symbols which are enclosed with     */
-/*  angle brackets <...>. If the value of the argument flag is     */
-/*  smaller than 2, the multi-character symbol must be already in  */
-/*  the lexicon in order to be recognized.                         */
+/*  angle brackets <...>. If the argument flag insert is true,     */
+/*  the multi-character symbol must be already in the lexicon in   */
+/*  order to be recognized.                                        */
 /*                                                                 */
 /*******************************************************************/
 
-int Alphabet::next_mcsym( char* &string, int extended )
+int Alphabet::next_mcsym( char* &string, bool insert )
 
 {
   char *start=string;
@@ -446,7 +446,7 @@ int Alphabet::next_mcsym( char* &string, int extended )
 	*end = 0;
 
 	int c;
-	if (extended <= 2)
+	if (insert)
 	  c = add_symbol( start );
 	else
 	  c = symbol2code(start);
@@ -473,13 +473,13 @@ int Alphabet::next_mcsym( char* &string, int extended )
 /*                                                                 */
 /*******************************************************************/
 
-int Alphabet::next_code( char* &string, int extended )
+int Alphabet::next_code( char* &string, bool extended, bool insert )
 
 {
   if (*string == 0)
     return EOF; // finished
 
-  int c = next_mcsym(string, extended);
+  int c = next_mcsym(string, insert);
   if (c != EOF)
     return c;
 
@@ -506,7 +506,7 @@ int Alphabet::next_code( char* &string, int extended )
 /*                                                                 */
 /*******************************************************************/
 
-Label Alphabet::next_label( char* &string, int extended )
+Label Alphabet::next_label( char* &string, bool extended )
 
 {
   // read first character
@@ -517,7 +517,7 @@ Label Alphabet::next_label( char* &string, int extended )
   Character lc=(Character)c;
   if (!extended || *string != ':') { // single character?
     if (lc == Label::epsilon)
-      return next_label(string); // ignore epsilon
+      return next_label(string, extended); // ignore epsilon
     return Label(lc);
   }
 
@@ -532,7 +532,7 @@ Label Alphabet::next_label( char* &string, int extended )
 
   Label l(lc, (Character)c);
   if (l.is_epsilon())
-    return next_label(string); // ignore epsilon transitions
+    return next_label(string, extended); // ignore epsilon transitions
   return l;
 }
 
@@ -782,8 +782,12 @@ char *Alphabet::print_analysis( Analysis &ana, bool both_layers )
     const char *s;
 
     // either print the analysis symbol or the whole label
-    if (both_layers)
+    if (both_layers) {
       s = write_label(l);
+      // quote colons
+      if (strcmp(s,":") == 0)
+	ch.push_back('\\');
+    }
     else if (l.lower_char() != Label::epsilon)
       s = write_char(l.lower_char());
     else

data/ext/sfst_machine/alphabet.h

@@ -13,7 +13,6 @@
 #define _ALPHABET_H_
 
 #include <stdio.h>
-#include <string.h>
 
 #include "basic.h"
 
@@ -22,6 +21,10 @@
 
 #include <iostream>
 
+#include <cstring>
+
+#include "sgi.h"
+
 #ifndef CODE_DATA_TYPE
 typedef unsigned short Character;  // data type of the symbol codes
 #else
@@ -32,18 +35,6 @@ typedef unsigned CODE_DATA_TYPE Character;
 // on the analysis level (lower) or the surface level (upper)
 typedef enum {upper, lower} Level;
 
-#ifdef SGIext
-
-#include <ext/hash_set>
-#include <ext/hash_map>
-
-#else
-
-#include <hash_set>
-#include <hash_map>
-
-#endif
-
 extern char EpsilonString[]; // holds the symbol representing the empty string
                              // which is usually "<>"
 
@@ -157,10 +148,10 @@ class Alphabet {
   typedef std::set<Label, Label::label_cmp> LabelSet;
 
   // hash table used to map the symbols to their codes
-  typedef __gnu_cxx::hash_map<const char*, Character, __gnu_cxx::hash<const char*>,eqstr> SymbolMap;
+  typedef hash_map<const char*, Character, hash<const char*>,eqstr> SymbolMap;
 
   // hash table used to map the codes back to the symbols
-  typedef __gnu_cxx::hash_map<Character, char*> CharMap;
+  typedef hash_map<Character, char*> CharMap;
 
  private:
   SymbolMap sm; // maps symbols to codes
@@ -249,17 +240,17 @@ class Alphabet {
   const char *write_label( Label l, bool with_brackets=true ) const;
 
   // scan the next multi-character symbol in the argument string
-  int next_mcsym( char*&, int extended=1 );
+  int next_mcsym( char*&, bool insert=true );
 
   // scan the next symbol in the argument string
-  int next_code( char*&, int extended=1 );
+  int next_code( char*&, bool extended=true, bool insert=true );
 
   // convert a character string into a symbol or label sequence
   void string2symseq( char*, std::vector<Character>& );
   void string2labelseq( char*, std::vector<Label>& );
 
   // scan the next label in the argument string
-  Label next_label( char*&, int extended=1 );
+  Label next_label( char*&, bool extended=true );
 
   // store the alphabet in the argument file (in binary form)
   void store( FILE* ) const;
@@ -276,7 +267,7 @@ class Alphabet {
 };
 
 // write the alphabet to the output stream (in readable form)
-std::ostream &operator<<(std::ostream&, Alphabet&);
+std::ostream &operator<<(std::ostream&, const Alphabet&);
 
 
 #endif

data/ext/sfst_machine/compact.C

@@ -383,7 +383,7 @@ void CompactTransducer::longest_match2(unsigned int n, char *string, int l,
 
   // follow the non-epsilon transitions
   char *end=string;
-  int c=alphabet.next_code(end, false);
+  int c=alphabet.next_code(end, false, false);
   l += end-string;
   if (c != EOF) {
     // find the set of arcs with matching upper character in the sort list
@@ -430,7 +430,7 @@ const char *CompactTransducer::longest_match( char* &string )
 
   // no match? return the next character
   if (ba.size() == 0) {
-    int c=alphabet.next_code(string, false);
+    int c=alphabet.next_code(string, false, false);
     return alphabet.code2symbol(c);
   }
 

data/ext/sfst_machine/determinise.C

@@ -14,7 +14,6 @@
 using std::vector;
 using std::pair;
 using std::set;
-using __gnu_cxx::hash_map;
 
 /*****************  class NodeSet  *********************************/
 

data/ext/sfst_machine/extconf.rb

@@ -1,4 +1,4 @@
 require 'mkmf'
 CONFIG['CC'] = 'g++'
-$CFLAGS='-Wall -O3 -Wall -Wcast-qual -Wconversion -DSGIext -DREADLINE'
+$CFLAGS='-Wall -O3 -Wall -Wcast-qual -Wconversion -DSGI__gnu_cxx -DREADLINE'
 create_makefile "sfst_machine"

data/ext/sfst_machine/fst-compiler.C

@@ -72,7 +72,7 @@
      PRINT = 261,
      POS = 262,
      INSERT = 263,
-     REV = 264,
+     SWITCH = 264,
      ARROW = 265,
      REPLACE = 266,
      SYMBOL = 267,
@@ -94,7 +94,7 @@
 #define PRINT 261
 #define POS 262
 #define INSERT 263
-#define REV 264
+#define SWITCH 264
 #define ARROW 265
 #define REPLACE 266
 #define SYMBOL 267
@@ -532,7 +532,7 @@ static const yytype_uint8 yyrline[] =
 static const char *const yytname[] =
 {
   "$end", "error", "$undefined", "NEWLINE", "ALPHA", "COMPOSE", "PRINT",
-  "POS", "INSERT", "REV", "ARROW", "REPLACE", "SYMBOL", "VAR", "SVAR",
+  "POS", "INSERT", "SWITCH", "ARROW", "REPLACE", "SYMBOL", "VAR", "SVAR",
   "RVAR", "RSVAR", "STRING", "STRING2", "UTF8CHAR", "CHARACTER", "'|'",
   "'-'", "'&'", "SEQ", "'!'", "'^'", "'_'", "'*'", "'+'", "'='", "'?'",
   "'('", "')'", "'{'", "'}'", "':'", "'['", "']'", "'.'", "','", "$accept",
@@ -2020,7 +2020,7 @@ yyreduce:
 
   case 71:
 #line 165 "fst-compiler.yy"
-    { (yyval.longchar)=utf8toint((yyvsp[(1) - (1)].value)); ;}
+    { (yyval.longchar)=utf8toint((yyvsp[(1) - (1)].value)); free((yyvsp[(1) - (1)].value)); ;}
     break;
 
   case 72:
@@ -2358,6 +2358,8 @@ yyreturn:
 
 
 extern FILE  *yyin;
+static int Compact=0;
+static int LowMem=0;
 
 /*******************************************************************/
 /*                                                                 */
@@ -2373,3 +2375,124 @@ void yyerror(char *text)
   exit(1);
 }
 
+
+/*******************************************************************/
+/*                                                                 */
+/*  warn                                                           */
+/*                                                                 */
+/*******************************************************************/
+
+void warn(char *text)
+
+{
+  cerr << "\n" << FileName << ":" << yylineno << ": warning: " << text << "!\n";
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  warn2                                                          */
+/*                                                                 */
+/*******************************************************************/
+
+void warn2(char *text, char *text2)
+
+{
+  cerr << "\n" << FileName << ":" << yylineno << ": warning: " << text << ": ";
+  cerr << text2 << "\n";
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  get_flags                                                      */
+/*                                                                 */
+/*******************************************************************/
+
+void get_flags( int *argc, char **argv )
+
+{
+  for( int i=1; i<*argc; i++ ) {
+    if (strcmp(argv[i],"-c") == 0) {
+      Compact = 1;
+      argv[i] = NULL;
+    }
+    else if (strcmp(argv[i],"-l") == 0) {
+      LowMem = 1;
+      argv[i] = NULL;
+    }
+    else if (strcmp(argv[i],"-q") == 0) {
+      Verbose = 0;
+      argv[i] = NULL;
+    }
+    else if (strcmp(argv[i],"-s") == 0) {
+      Switch = 1;
+      argv[i] = NULL;
+    }
+  }
+  // remove flags from the argument list
+  int k;
+  for( int i=k=1; i<*argc; i++)
+    if (argv[i] != NULL)
+      argv[k++] = argv[i];
+  *argc = k;
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  main                                                           */
+/*                                                                 */
+/*******************************************************************/
+
+int main( int argc, char *argv[] )
+
+{
+  FILE *file;
+
+  get_flags(&argc, argv);
+  if (argc < 3) {
+    fprintf(stderr,"\nUsage: %s [options] infile outfile\n", argv[0]);
+    fprintf(stderr,"\nOPTIONS:\n");
+    fprintf(stderr,"-c\tStore the transducer in fst-infl2 format.\n");
+    fprintf(stderr,"-l\tStore the transducer in fst-infl3 format.\n");
+    fprintf(stderr,"-s\tSwitch the upper and lower levels producing a transducer for generation rather than recognition.\n");
+    fprintf(stderr,"-q\tquiet mode\n\n");
+    exit(1);
+  }
+  if ((file = fopen(argv[1],"rt")) == NULL) {
+    fprintf(stderr,"\nError: Cannot open grammar file \"%s\"\n\n", argv[1]);
+    exit(1);
+  }
+  FileName = argv[1];
+  Result = NULL;
+  TheAlphabet.utf8 = UTF8;
+  yyin = file;
+  try {
+    yyparse();
+    Result->alphabet.utf8 = UTF8;
+    if (Verbose)
+      cerr << "\n";
+    if (Result->is_empty()) 
+      warn("resulting transducer is empty"); 
+    if ((file = fopen(argv[2],"wb")) == NULL) {
+	fprintf(stderr,"\nError: Cannot open output file %s\n\n", argv[2]);
+	exit(1);
+    }
+    if (Compact) {
+      MakeCompactTransducer ca(*Result);
+      delete Result;
+      ca.store(file);
+    }
+    else if (LowMem)
+      Result->store_lowmem(file);
+    else
+      Result->store(file);
+    fclose(file);
+  }
+  catch(const char* p) {
+      cerr << "\n" << p << "\n\n";
+      exit(1);
+  }
+}
+

data/ext/sfst_machine/fst-compiler.h

@@ -45,7 +45,7 @@
      PRINT = 261,
      POS = 262,
      INSERT = 263,
-     REV = 264,
+     SWITCH = 264,
      ARROW = 265,
      REPLACE = 266,
      SYMBOL = 267,
@@ -67,7 +67,7 @@
 #define PRINT 261
 #define POS 262
 #define INSERT 263
-#define REV 264
+#define SWITCH 264
 #define ARROW 265
 #define REPLACE 266
 #define SYMBOL 267

data/ext/sfst_machine/fst-compiler.yy

@@ -43,7 +43,7 @@ Transducer *Result;
   Contexts   *contexts;
 }
 
-%token <number> NEWLINE ALPHA COMPOSE PRINT POS INSERT REV
+%token <number> NEWLINE ALPHA COMPOSE PRINT POS INSERT SWITCH
 %token <type>   ARROW
 %token <rtype>  REPLACE
 %token <name>   SYMBOL VAR SVAR RVAR RSVAR
@@ -111,7 +111,7 @@ RE:         RE ARROW CONTEXTS2      { $$ = restriction($1,$2,$3,0); }
           | RE '?'           { $$ = optional($1); }
           | RE RE %prec SEQ  { $$ = catenate($1, $2); }
           | '!' RE           { $$ = negation($2); }
-          | REV RE           { $$ = switch_levels($2); }
+          | SWITCH RE        { $$ = switch_levels($2); }
           | '^' RE           { $$ = upper_level($2); }
           | '_' RE           { $$ = lower_level($2); }
           | RE '&' RE        { $$ = conjunction($1, $3); }
@@ -162,7 +162,7 @@ VALUE:      LCHAR '-' LCHAR	   { $$=add_values($1,$3,NULL); }
           ;
 
 LCHAR:      CHARACTER	{ $$=$1; }
-          | UTF8CHAR	{ $$=utf8toint($1); }
+          | UTF8CHAR	{ $$=utf8toint($1); free($1); }
 	  | SCHAR       { $$=$1; }
           ;
 

data/ext/sfst_machine/fst.C

@@ -124,6 +124,27 @@ void Node::init()
 }
 
 
+/*******************************************************************/
+/*                                                                 */
+/*  Node::clear_visited                                            */
+/*                                                                 */
+/*******************************************************************/
+
+void Node::clear_visited( NodeHashSet &nodeset )
+
+{
+  if (nodeset.find( this ) == nodeset.end()) {
+    visited = 0;
+    nodeset.insert( this );
+    fprintf(stderr," %lu", nodeset.size());
+    for( ArcsIter p(arcs()); p; p++ ) {
+      Arc *arc=p;
+      arc->target_node()->clear_visited( nodeset );
+    }
+  }
+}
+
+
 /*******************************************************************/
 /*                                                                 */
 /*  NodeNumbering::number_node                                     */
@@ -196,14 +217,16 @@ Arc *Transducer::new_arc( Label l, Node *target )
 /*                                                                 */
 /*******************************************************************/
 
-void Transducer::add_string( char *s, bool extended )
+void Transducer::add_string( char *s, bool extended, Alphabet *a )
 
 {
+  if (a == NULL)
+    a = &alphabet;
+
   Node *node=root_node();
-  
   Label l;
-  while (!(l = alphabet.next_label(s, extended)).is_epsilon()) {
-    alphabet.insert(l);
+  while (!(l = a->next_label(s, extended)).is_epsilon()) {
+    a->insert(l);
     Arcs *arcs=node->arcs();
     node = arcs->target_node( l );
     if (node == NULL) {
@@ -271,7 +294,7 @@ Transducer::Transducer( istream &is, const Alphabet *a, bool verbose  )
 	break;
     buffer[l+1] = 0;
 
-    add_string(buffer,extended);
+    add_string(buffer, extended);
   }
   if (verbose && n >= 10000)
     cerr << "\n";
@@ -515,7 +538,13 @@ int Transducer::print_strings( FILE *file, bool with_brackets )
 bool Transducer::analyze_string( char *string, FILE *file, bool with_brackets )
 
 {
-  Transducer a1(string, &alphabet, false);
+  vector<Character> input;
+  alphabet.string2symseq( string, input );
+  vector<Label> labels;
+  for( size_t i=0; i<input.size(); i++ )
+    labels.push_back(Label(input[i]));
+
+  Transducer a1(labels);
   Transducer *a2=&(*this || a1);
   Transducer *a3=&(a2->lower_level());
   delete a2;
@@ -598,14 +627,15 @@ static void print_node( ostream &s, Node *node, NodeNumbering &index,
 {
   if (!node->was_visited( vmark )) {
     Arcs *arcs=node->arcs();
-    if (node->is_final())
-      s << "final\t" << index[node] << "\n";
     for( ArcsIter p(arcs); p; p++ ) {
       Arc *arc=p;
-      s << index[node] << "\t";
-      s << abc.write_label(arc->label()) << "\t";
-      s << index[arc->target_node()] << "\n";
+      s << index[node] << "\t" << index[arc->target_node()];
+      s << "\t" << abc.write_char(arc->label().lower_char());
+      s << "\t" << abc.write_char(arc->label().upper_char());
+      s << "\n";
     }
+    if (node->is_final())
+      s << index[node] << "\n";
     for( ArcsIter p(arcs); p; p++ ) {
       Arc *arc=p;
       print_node( s, arc->target_node(), index, vmark, abc );
@@ -928,18 +958,22 @@ void Transducer::read_transducer_text( FILE *file )
   for( size_t line=0; fgets(buffer, 10000, file ); line++ ) {
     char *p = buffer;
     char *s = next_string(p, line);
-    if (strcmp(s, "final") == 0) {
-      s = next_string(p, line);
-      create_node( nodes, s, line )->set_final(true);
-    }
+    Node *node = create_node( nodes, s, line );
+    if (p == NULL)
+      node->set_final(true);
     else {
-      Node *node = create_node( nodes, s, line );
-      s = next_string(p, line);
-      Label l = alphabet.next_label( s, 2 );
-      if (*s != 0 || l == Label::epsilon)
-	error_message( line );
       s = next_string(p, line);
       Node *target = create_node( nodes, s, line );
+
+      s = next_string(p, line);
+      Character lc = alphabet.add_symbol(s);
+      s = next_string(p, line);
+      Character uc = alphabet.add_symbol(s);
+      Label l(lc,uc);
+      if (l == Label::epsilon)
+	error_message( line );
+
+      alphabet.insert(l);
       node->add_arc( l, target, this );
     }
   }

data/ext/sfst_machine/fst.h

@@ -25,7 +25,7 @@
 
 #include "mem.h"
 
-typedef unsigned long VType;
+typedef unsigned short VType;
 
 extern int Quiet;
 
@@ -41,8 +41,7 @@ struct hashf {
 struct equalf {
   int operator()(const Node *n1, const Node *n2) const { return n1==n2; }
 };
-typedef __gnu_cxx::hash_set<Node*, hashf, equalf> NodeHashSet;
-
+typedef hash_set<Node*, hashf, equalf> NodeHashSet;
 
 
 /*****************  class Arc  *************************************/
@@ -154,6 +153,7 @@ class Node {
   Arcs *arcs( void ) { return &arcsp; };
   const Arcs *arcs( void ) const { return &arcsp; };
   Node *forward( void ) { return forwardp; };
+  void clear_visited( NodeHashSet &nodeset );
   bool was_visited( VType vmark ) {
     if (visited == vmark)
       return true;
@@ -179,7 +179,7 @@ class Node2Int {
       return (n1 == n2);
     }
   };
-  typedef __gnu_cxx::hash_map<Node*, int, hashf, equalf> NL;
+  typedef hash_map<Node*, int, hashf, equalf> NL;
 
  private:
   int current_number;
@@ -231,7 +231,7 @@ class PairMapping {
       return (p1.first==p2.first && p1.second == p2.second);
     }
   };
-  typedef __gnu_cxx::hash_map<NodePair, Node*, hashf, equalf> PairMap;
+  typedef hash_map<NodePair, Node*, hashf, equalf> PairMap;
   PairMap pm;
   
  public:
@@ -256,8 +256,16 @@ class Transducer {
   Mem mem;
 
   typedef std::set<Label, Label::label_cmp> LabelSet;
-  typedef __gnu_cxx::hash_map<Character, char*> SymbolMap;
+  typedef hash_map<Character, char*> SymbolMap;
 
+  void incr_vmark( void ) {
+    if (++vmark == 0) {
+      NodeHashSet nodes;
+      root.clear_visited( nodes );
+      fprintf(stderr,"clearing flags\n");
+      vmark = 1;
+    }
+  };
   void reverse_node( Node *old_node, Transducer *new_node );
   Label recode_label( Label, bool lswitch, bool recode, Alphabet& );
   Node *copy_nodes( Node *n, Transducer *a, 
@@ -287,10 +295,6 @@ class Transducer {
 
  public:
   VType vmark;
-  void incr_vmark( void ) {
-    if (++vmark == 0)
-      throw "Overflow of generation counter!";
-  };
   Alphabet alphabet; // The set of all labels, i.e. character pairs
 
   Transducer( void ) : root(), mem()
@@ -308,7 +312,7 @@ class Transducer {
   const Node *root_node( void ) const { return &root; };  // returns the root node
   Node *new_node( void );                // memory alocation for a new node
   Arc *new_arc( Label l, Node *target ); // memory alocation for a new arc
-  void add_string( char *s, bool extended=false );
+  void add_string( char *s, bool extended=false, Alphabet *a=NULL );
   void complete_alphabet( void );
   void minimise_alphabet( void );
   void prune( void );  // remove unnecessary arcs