ruby-sfst 0.1.0

data/ext/sfst_machine/interface.h

@@ -0,0 +1,94 @@
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     interface.h                                           */
+/*  MODULE   interface                                             */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*******************************************************************/
+
+#ifndef _INTERFACE_H_
+#define _INTERFACE_H_
+
+#include "utf8.h"
+#include "fst.h"
+
+typedef enum {twol_left,twol_right,twol_both} Twol_Type;
+
+typedef enum {repl_left,repl_right,repl_up,repl_down} Repl_Type;
+
+typedef struct range_t {
+  Character character;
+  struct range_t *next;
+} Range;
+
+typedef struct ranges_t {
+  Range  *range;
+  struct ranges_t *next;
+} Ranges;
+
+
+typedef struct contexts_t {
+  Transducer *left, *right;
+  struct contexts_t *next;
+} Contexts;
+
+
+extern bool Verbose;
+extern bool UTF8;
+extern char *FileName;
+extern Alphabet TheAlphabet;
+
+void error2( char *message, char *input );
+Transducer *new_transducer( Range*, Range* );
+Transducer *read_words( char *filename );
+Transducer *read_transducer( char *filename );
+Transducer *var_value( char *name );
+Transducer *rvar_value( char *name );
+Range *svar_value( char *name );
+Range *complement_range( Range* );
+Range *rsvar_value( char *name );
+Character character_code( unsigned int uc );
+Character symbol_code( char *s );
+
+Range *add_value( Character, Range*);
+Range *add_var_values( char *name, Range*);
+Range *add_values( unsigned int, unsigned int, Range*);
+Range *append_values( Range *r2, Range *r );
+void add_alphabet( Transducer* );
+void store_transducer( Transducer *a, char *filename );
+
+// These functions delete their argument automata
+
+void def_alphabet( Transducer *a );
+bool def_var( char *name, Transducer *a );
+bool def_rvar( char *name, Transducer *a );
+bool def_svar( char *name, Range *r );
+Transducer *explode( Transducer *a );
+Transducer *catenate( Transducer *a1, Transducer *a2 );
+Transducer *disjunction( Transducer *a1, Transducer *a2 );
+Transducer *conjunction( Transducer *a1, Transducer *a2 );
+Transducer *subtraction( Transducer *a1, Transducer *a2 );
+Transducer *composition( Transducer *a1, Transducer *a2 );
+Transducer *restriction( Transducer *a, Twol_Type type, Contexts *c, int );
+Transducer *replace( Transducer *a, Repl_Type type, bool optional );
+Transducer *replace_in_context( Transducer *a, Repl_Type type, Contexts *c, bool optional );
+Transducer *negation( Transducer *a );
+Transducer *upper_level( Transducer *a );
+Transducer *lower_level( Transducer *a );
+Transducer *minimise( Transducer *a );
+Transducer *switch_levels( Transducer *a );
+Transducer *repetition( Transducer *a );
+Transducer *repetition2( Transducer *a );
+Transducer *optional( Transducer *a );
+Transducer *make_rule( Transducer *lc, Range *r1, Twol_Type type,
+		      Range *r2, Transducer *rc );
+Transducer *freely_insert( Transducer *a, Character c, Character c );
+Transducer *make_mapping( Ranges*, Ranges* );
+Ranges *add_range( Range*, Ranges* );
+Contexts *make_context( Transducer *l, Transducer *r );
+Contexts *add_context( Contexts *nc, Contexts *c );
+Transducer *result( Transducer*, bool );
+void write_to_file( Transducer*, char *filename);
+
+#endif

data/ext/sfst_machine/make-compact.C

@@ -0,0 +1,328 @@
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     make-compact.C                                        */
+/*  MODULE   make-compact                                          */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*  PURPOSE  Code needed for generating compact automata           */
+/*                                                                 */
+/*******************************************************************/
+
+#include <math.h>
+
+#include "make-compact.h"
+
+using std::equal_range;
+using std::sort;
+using std::cerr;
+using __gnu_cxx::hash_map;
+
+class ARC {
+public:
+  int cv;
+  Label label;
+  unsigned int target_node;
+  
+  bool operator< ( const ARC a ) const {
+    return cv < a.cv;
+  };
+};
+
+typedef hash_map<Label, size_t, Label::label_hash, Label::label_eq> LabelNumber;
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::sort                                    */
+/*                                                                 */
+/*******************************************************************/
+
+void MakeCompactTransducer::sort( Level level )
+
+{
+  for( unsigned int n=0; n<number_of_nodes; n++) {
+    unsigned int from=first_arc[n]; 
+    unsigned int to=first_arc[n+1]; 
+    int l=to-from; 
+
+    // copy the arcs to a temporary table
+    ARC *arc=new ARC[l];
+    for( unsigned int i=from; i<to; i++) {
+      arc[i-from].cv = (int)label[i].get_char(level);
+      // make sure that epsilon arcs are stored at the beginning
+      // even if epsilon is not 0
+      if (arc[i-from].cv == (int)Label::epsilon)
+	arc[i-from].cv = -1;
+      arc[i-from].label = label[i];
+      arc[i-from].target_node = target_node[i];
+    }
+
+    // sort the table
+    ::sort( arc, arc+l );
+
+    // copy the arcs back to the original table
+    for( unsigned int i=from; i<to; i++) {
+      label[i] = arc[i-from].label;
+      target_node[i] = arc[i-from].target_node;
+    }
+
+    delete[] arc;
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::count_arcs                              */
+/*                                                                 */
+/*******************************************************************/
+
+void MakeCompactTransducer::count_arcs( Node *node, NodeNumbering &index, 
+				       long vmark )
+{
+  if (!node->was_visited( vmark )) {
+      unsigned int n = index[node];
+      finalp[n] = node->is_final();
+      first_arc[n] = 0;
+      Arcs *arcs=node->arcs();
+      for( ArcsIter p(arcs); p; p++ ) {
+	  Arc *arc=p;
+	  first_arc[n]++;
+	  count_arcs(arc->target_node(), index, vmark);
+      }
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::store_arcs                              */
+/*                                                                 */
+/*******************************************************************/
+
+void MakeCompactTransducer::store_arcs( Node *node, NodeNumbering &index, 
+				       long vmark )
+{
+  if (!node->was_visited( vmark )) {
+      unsigned int n=first_arc[index[node]];
+      Arcs *arcs=node->arcs();
+      for( ArcsIter p(arcs); p; p++ ) {
+        Arc *arc=p;
+	label[n] = arc->label();
+	target_node[n++] = index[arc->target_node()];
+	store_arcs(arc->target_node(), index, vmark);
+      }
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::MakeCompactTransducer                   */
+/*                                                                 */
+/*******************************************************************/
+
+MakeCompactTransducer::MakeCompactTransducer( Transducer &a, Level l )
+
+{
+  if (a.is_infinitely_ambiguous()) {
+    cerr << "Error: resulting transducer contains an infinite loop!\n";
+    exit(1);
+  }
+
+  NodeNumbering index(a);
+
+  alphabet.copy(a.alphabet);
+
+  // memory allocation
+  number_of_nodes = index.number_of_nodes();
+  finalp = new char[number_of_nodes];
+  first_arc = new unsigned int[number_of_nodes+1];
+
+  // count the number of outgoing arcs for each node
+  // and store them in first_arc[]
+  a.incr_vmark();
+  count_arcs( a.root_node(), index, a.vmark );
+  for( int n=number_of_nodes; n>0; n-- )
+    first_arc[n] = first_arc[n-1];
+  first_arc[0] = 0;
+  for( unsigned int n=0; n<number_of_nodes; n++ )
+    first_arc[n+1] += first_arc[n];
+  number_of_arcs = first_arc[number_of_nodes];
+
+  // memory allocation
+  label = new Label[number_of_arcs];
+  target_node = new unsigned int[number_of_arcs];
+
+  // store the arcs
+  a.incr_vmark();
+  store_arcs( a.root_node(), index, a.vmark );
+
+  // sort the arcs
+  sort( l );
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::store_finalp                            */
+/*                                                                 */
+/*******************************************************************/
+
+void MakeCompactTransducer::store_finalp( FILE *file )
+
+{
+  int k=0;
+  unsigned char n=0;
+  
+  for( size_t i=0; i<number_of_nodes; i++ ) {
+    n = n << 1;
+    if (finalp[i])
+      n |= 1;
+    if (++k == 8) {
+      fputc(n, file);
+      n = 0;
+      k = 0;
+    }
+  }
+  if (k > 0) {
+    n <<= 8-k;
+    fputc(n, file);
+  }
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::store_first_arcs                        */
+/*                                                                 */
+/*  The data is encoded with the minimal number of bits needed.    */
+/*                                                                 */
+/*******************************************************************/
+
+void MakeCompactTransducer::store_first_arcs( FILE *file )
+
+{
+  int k=0;
+  unsigned int n=0;
+  // compute number of bits required for storing each item
+  size_t bits=(size_t)ceil(log(number_of_arcs+1)/log(2));
+
+  for( size_t i=0; i<=number_of_nodes; i++ ) {
+    unsigned int m=first_arc[i];
+    m <<= (sizeof(n)*8) - bits;
+    m >>= k;
+    n = n | m;
+    k += bits;
+    if (k >= (int)sizeof(n)*8) {
+      fwrite(&n, sizeof(n), 1, file);
+      k -= sizeof(n) * 8;
+      n = first_arc[i];
+      if (k == 0)
+	n = 0;
+      else
+	n = first_arc[i] << (sizeof(n) * 8 - k);
+    }
+  }
+  if (k > 0)
+    fwrite(&n, sizeof(n), 1, file);
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::store_target_nodes                      */
+/*                                                                 */
+/*******************************************************************/
+
+void MakeCompactTransducer::store_target_nodes( FILE *file )
+
+{
+  int k=0;
+  unsigned int n=0;
+  size_t bits=(size_t)ceil(log(number_of_nodes)/log(2));
+
+  for( size_t i=0; i<number_of_arcs; i++ ) {
+    unsigned int m=target_node[i];
+    m <<= (sizeof(n)*8) - bits;
+    m >>= k;
+    n = n | m;
+    k += bits;
+    if (k >= (int)sizeof(n)*8) {
+      fwrite(&n, sizeof(n), 1, file);
+      k -= sizeof(n)*8;
+      if (k == 0)
+	n = 0;
+      else
+	n = target_node[i] << (sizeof(n) * 8 - k);
+    }
+  }
+  if (k > 0)
+    fwrite(&n, sizeof(n), 1, file);
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::store_labels                            */
+/*                                                                 */
+/*******************************************************************/
+
+void MakeCompactTransducer::store_labels( FILE *file )
+
+{
+  size_t N=0;
+  LabelNumber LNum;
+  for( Alphabet::const_iterator it=alphabet.begin();
+       it != alphabet.end(); it++ )
+    {
+      Label l=*it;
+      LNum[l] = N++;
+    }
+
+  int k=0;
+  unsigned int n=0;
+  size_t bits=(size_t)ceil(log(alphabet.size())/log(2));
+
+  for( size_t i=0; i<number_of_arcs; i++ ) {
+    unsigned int l = LNum[label[i]];
+    unsigned int m=l;
+    m = m << (sizeof(n)*8) - bits;
+    m = m >> k;
+    n = n | m;
+    k += bits;
+    if (k >= (int)sizeof(n)*8) {
+      fwrite(&n, sizeof(n), 1, file);
+      k -= sizeof(n)*8;
+      if (k == 0)
+	n = 0;
+      else
+	n = l << (sizeof(n) * 8 - k);
+    }
+  }
+  if (k > 0)
+    fwrite(&n, sizeof(n), 1, file);
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  MakeCompactTransducer::store                                   */
+/*                                                                 */
+/*******************************************************************/
+
+void MakeCompactTransducer::store( FILE *file )
+
+{
+  fputc('c',file);
+  alphabet.store(file);
+  fwrite(&number_of_nodes, sizeof(number_of_nodes), 1, file);
+  fwrite(&number_of_arcs, sizeof(number_of_arcs), 1, file);
+  store_finalp(file);
+  store_first_arcs(file);
+  store_labels(file);
+  store_target_nodes(file);
+  if (ferror(file))
+    throw "Error encountered while writing transducer to file\n";
+}

data/ext/sfst_machine/make-compact.h

@@ -0,0 +1,34 @@
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     make-compact.h                                        */
+/*  MODULE   make-compact                                          */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*******************************************************************/
+
+#ifndef _MAKE_COMPACT_H_
+#define _MAKE_COMPACT_H_
+
+#include "fst.h"
+#include "compact.h"
+
+
+class MakeCompactTransducer : CompactTransducer {
+    
+private:
+  void count_arcs(Node *node, NodeNumbering &index, long vmark);
+  void store_arcs(Node *node, NodeNumbering &index, long vmark);
+  void store_finalp( FILE *file );
+  void store_first_arcs( FILE *file );
+  void store_target_nodes( FILE *file );
+  void store_labels( FILE *file );
+
+public:
+  MakeCompactTransducer( Transducer &a, Level sort=upper );
+
+  void sort( Level );
+  void store( FILE *file );
+};
+
+#endif

data/ext/sfst_machine/mem.h

@@ -0,0 +1,74 @@
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     mem.h                                                 */
+/*  MODULE   mem                                                   */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*  PURPOSE  memory management functions                           */
+/*                                                                 */
+/*******************************************************************/
+
+#ifndef _MEM_H_
+#define _MEM_H_
+
+#include <stdlib.h>
+
+#define MEMBUFFER_SIZE 100000
+
+
+/*****************  class Mem  *************************************/
+
+class Mem {
+
+ private:
+
+  struct MemBuffer {
+    char buffer[MEMBUFFER_SIZE];
+    struct MemBuffer *next;
+  };
+
+  MemBuffer *first_buffer;
+  long pos;
+  void add_buffer() {
+    MemBuffer *mb=(MemBuffer*)malloc(sizeof(MemBuffer));
+    mb->next = first_buffer;
+    first_buffer = mb;
+    pos = 0;
+  }
+
+public:
+  Mem() { first_buffer = NULL; add_buffer(); }
+  ~Mem() { clear(); }
+
+  void clear() {
+    while (first_buffer) {
+      MemBuffer *next = first_buffer->next;
+      free(first_buffer);
+      first_buffer = next;
+    }
+    pos = 0;
+  }
+
+  void *alloc( size_t n ) {
+    void *result;
+    
+    /* do memory alignment to multiples of 4 */
+    if (n % 4)
+      n += 4 - (n % 4);
+    
+    if (first_buffer == NULL || pos+n > MEMBUFFER_SIZE)
+      add_buffer();
+    if (pos+n > MEMBUFFER_SIZE)
+      throw "Allocation of memory block larger than MEMBUFFER_SIZE attempted!";
+    
+    result = (void*)(first_buffer->buffer + pos);
+    pos += n;
+    return result;
+  }
+
+  //class MemError {};
+
+};
+
+#endif