ruby-sfst 0.4.3 → 0.4.4

data/ext/sfst_machine/interface.h

@@ -1,93 +0,0 @@
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     interface.h                                           */
-/*  MODULE   interface                                             */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*******************************************************************/
-
-#ifndef _INTERFACE_H_
-#define _INTERFACE_H_
-
-#include "utf8.h"
-#include "fst.h"
-
-typedef enum {twol_left,twol_right,twol_both} Twol_Type;
-
-typedef enum {repl_left,repl_right,repl_up,repl_down} Repl_Type;
-
-typedef struct range_t {
-  Character character;
-  struct range_t *next;
-} Range;
-
-typedef struct ranges_t {
-  Range  *range;
-  struct ranges_t *next;
-} Ranges;
-
-
-typedef struct contexts_t {
-  Transducer *left, *right;
-  struct contexts_t *next;
-} Contexts;
-
-
-extern bool Verbose;
-extern bool UTF8;
-extern char *FileName;
-extern Alphabet TheAlphabet;
-
-void error2( char *message, char *input );
-Transducer *new_transducer( Range*, Range* );
-Transducer *read_words( char *filename );
-Transducer *read_transducer( char *filename );
-Transducer *var_value( char *name );
-Transducer *rvar_value( char *name );
-Range *svar_value( char *name );
-Range *complement_range( Range* );
-Range *rsvar_value( char *name );
-Character character_code( unsigned int uc );
-Character symbol_code( char *s );
-
-Range *add_value( Character, Range*);
-Range *add_var_values( char *name, Range*);
-Range *add_values( unsigned int, unsigned int, Range*);
-Range *append_values( Range *r2, Range *r );
-void add_alphabet( Transducer* );
-
-// These functions delete their argument automata
-
-void def_alphabet( Transducer *a );
-bool def_var( char *name, Transducer *a );
-bool def_rvar( char *name, Transducer *a );
-bool def_svar( char *name, Range *r );
-Transducer *explode( Transducer *a );
-Transducer *catenate( Transducer *a1, Transducer *a2 );
-Transducer *disjunction( Transducer *a1, Transducer *a2 );
-Transducer *conjunction( Transducer *a1, Transducer *a2 );
-Transducer *subtraction( Transducer *a1, Transducer *a2 );
-Transducer *composition( Transducer *a1, Transducer *a2 );
-Transducer *restriction( Transducer *a, Twol_Type type, Contexts *c, int );
-Transducer *replace( Transducer *a, Repl_Type type, bool optional );
-Transducer *replace_in_context( Transducer *a, Repl_Type type, Contexts *c, bool optional );
-Transducer *negation( Transducer *a );
-Transducer *upper_level( Transducer *a );
-Transducer *lower_level( Transducer *a );
-Transducer *minimise( Transducer *a );
-Transducer *switch_levels( Transducer *a );
-Transducer *repetition( Transducer *a );
-Transducer *repetition2( Transducer *a );
-Transducer *optional( Transducer *a );
-Transducer *make_rule( Transducer *lc, Range *r1, Twol_Type type,
-		      Range *r2, Transducer *rc );
-Transducer *freely_insert( Transducer *a, Character lc, Character uc );
-Transducer *make_mapping( Ranges*, Ranges* );
-Ranges *add_range( Range*, Ranges* );
-Contexts *make_context( Transducer *l, Transducer *r );
-Contexts *add_context( Contexts *nc, Contexts *c );
-Transducer *result( Transducer*, bool );
-void write_to_file( Transducer*, char *filename);
-
-#endif

data/ext/sfst_machine/make-compact.cc

@@ -1,327 +0,0 @@
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     make-compact.C                                        */
-/*  MODULE   make-compact                                          */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*  PURPOSE  Code needed for generating compact automata           */
-/*                                                                 */
-/*******************************************************************/
-
-#include <math.h>
-
-#include "make-compact.h"
-
-using std::equal_range;
-using std::sort;
-using std::cerr;
-
-class ARC {
-public:
-  int cv;
-  Label label;
-  unsigned int target_node;
-  
-  bool operator< ( const ARC a ) const {
-    return cv < a.cv;
-  };
-};
-
-typedef hash_map<Label, size_t, Label::label_hash, Label::label_eq> LabelNumber;
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::sort                                    */
-/*                                                                 */
-/*******************************************************************/
-
-void MakeCompactTransducer::sort( Level level )
-
-{
-  for( unsigned int n=0; n<number_of_nodes; n++) {
-    unsigned int from=first_arc[n]; 
-    unsigned int to=first_arc[n+1]; 
-    int l=to-from; 
-
-    // copy the arcs to a temporary table
-    ARC *arc=new ARC[l];
-    for( unsigned int i=from; i<to; i++) {
-      arc[i-from].cv = (int)label[i].get_char(level);
-      // make sure that epsilon arcs are stored at the beginning
-      // even if epsilon is not 0
-      if (arc[i-from].cv == (int)Label::epsilon)
-	arc[i-from].cv = -1;
-      arc[i-from].label = label[i];
-      arc[i-from].target_node = target_node[i];
-    }
-
-    // sort the table
-    ::sort( arc, arc+l );
-
-    // copy the arcs back to the original table
-    for( unsigned int i=from; i<to; i++) {
-      label[i] = arc[i-from].label;
-      target_node[i] = arc[i-from].target_node;
-    }
-
-    delete[] arc;
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::count_arcs                              */
-/*                                                                 */
-/*******************************************************************/
-
-void MakeCompactTransducer::count_arcs( Node *node, NodeNumbering &index, 
-				       long vmark )
-{
-  if (!node->was_visited( vmark )) {
-      unsigned int n = index[node];
-      finalp[n] = node->is_final();
-      first_arc[n] = 0;
-      Arcs *arcs=node->arcs();
-      for( ArcsIter p(arcs); p; p++ ) {
-	  Arc *arc=p;
-	  first_arc[n]++;
-	  count_arcs(arc->target_node(), index, vmark);
-      }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::store_arcs                              */
-/*                                                                 */
-/*******************************************************************/
-
-void MakeCompactTransducer::store_arcs( Node *node, NodeNumbering &index, 
-				       long vmark )
-{
-  if (!node->was_visited( vmark )) {
-      unsigned int n=first_arc[index[node]];
-      Arcs *arcs=node->arcs();
-      for( ArcsIter p(arcs); p; p++ ) {
-        Arc *arc=p;
-	label[n] = arc->label();
-	target_node[n++] = index[arc->target_node()];
-	store_arcs(arc->target_node(), index, vmark);
-      }
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::MakeCompactTransducer                   */
-/*                                                                 */
-/*******************************************************************/
-
-MakeCompactTransducer::MakeCompactTransducer( Transducer &a, Level l )
-
-{
-  if (a.is_infinitely_ambiguous()) {
-    cerr << "Error: resulting transducer contains an infinite loop!\n";
-    exit(1);
-  }
-
-  NodeNumbering index(a);
-
-  alphabet.copy(a.alphabet);
-
-  // memory allocation
-  number_of_nodes = index.number_of_nodes();
-  finalp = new char[number_of_nodes];
-  first_arc = new unsigned int[number_of_nodes+1];
-
-  // count the number of outgoing arcs for each node
-  // and store them in first_arc[]
-  a.incr_vmark();
-  count_arcs( a.root_node(), index, a.vmark );
-  for( int n=number_of_nodes; n>0; n-- )
-    first_arc[n] = first_arc[n-1];
-  first_arc[0] = 0;
-  for( unsigned int n=0; n<number_of_nodes; n++ )
-    first_arc[n+1] += first_arc[n];
-  number_of_arcs = first_arc[number_of_nodes];
-
-  // memory allocation
-  label = new Label[number_of_arcs];
-  target_node = new unsigned int[number_of_arcs];
-
-  // store the arcs
-  a.incr_vmark();
-  store_arcs( a.root_node(), index, a.vmark );
-
-  // sort the arcs
-  sort( l );
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::store_finalp                            */
-/*                                                                 */
-/*******************************************************************/
-
-void MakeCompactTransducer::store_finalp( FILE *file )
-
-{
-  int k=0;
-  unsigned char n=0;
-  
-  for( size_t i=0; i<number_of_nodes; i++ ) {
-    n = n << 1;
-    if (finalp[i])
-      n |= 1;
-    if (++k == 8) {
-      fputc(n, file);
-      n = 0;
-      k = 0;
-    }
-  }
-  if (k > 0) {
-    n <<= 8-k;
-    fputc(n, file);
-  }
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::store_first_arcs                        */
-/*                                                                 */
-/*  The data is encoded with the minimal number of bits needed.    */
-/*                                                                 */
-/*******************************************************************/
-
-void MakeCompactTransducer::store_first_arcs( FILE *file )
-
-{
-  int k=0;
-  unsigned int n=0;
-  // compute number of bits required for storing each item
-  size_t bits=(size_t)ceil(log(number_of_arcs+1)/log(2));
-
-  for( size_t i=0; i<=number_of_nodes; i++ ) {
-    unsigned int m=first_arc[i];
-    m <<= (sizeof(n)*8) - bits;
-    m >>= k;
-    n = n | m;
-    k += bits;
-    if (k >= (int)sizeof(n)*8) {
-      fwrite(&n, sizeof(n), 1, file);
-      k -= sizeof(n) * 8;
-      n = first_arc[i];
-      if (k == 0)
-	n = 0;
-      else
-	n = first_arc[i] << (sizeof(n) * 8 - k);
-    }
-  }
-  if (k > 0)
-    fwrite(&n, sizeof(n), 1, file);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::store_target_nodes                      */
-/*                                                                 */
-/*******************************************************************/
-
-void MakeCompactTransducer::store_target_nodes( FILE *file )
-
-{
-  int k=0;
-  unsigned int n=0;
-  size_t bits=(size_t)ceil(log(number_of_nodes)/log(2));
-
-  for( size_t i=0; i<number_of_arcs; i++ ) {
-    unsigned int m=target_node[i];
-    m <<= (sizeof(n)*8) - bits;
-    m >>= k;
-    n = n | m;
-    k += bits;
-    if (k >= (int)sizeof(n)*8) {
-      fwrite(&n, sizeof(n), 1, file);
-      k -= sizeof(n)*8;
-      if (k == 0)
-	n = 0;
-      else
-	n = target_node[i] << (sizeof(n) * 8 - k);
-    }
-  }
-  if (k > 0)
-    fwrite(&n, sizeof(n), 1, file);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::store_labels                            */
-/*                                                                 */
-/*******************************************************************/
-
-void MakeCompactTransducer::store_labels( FILE *file )
-
-{
-  size_t N=0;
-  LabelNumber LNum;
-  for( Alphabet::const_iterator it=alphabet.begin();
-       it != alphabet.end(); it++ )
-    {
-      Label l=*it;
-      LNum[l] = N++;
-    }
-
-  int k=0;
-  unsigned int n=0;
-  size_t bits=(size_t)ceil(log(alphabet.size())/log(2));
-
-  for( size_t i=0; i<number_of_arcs; i++ ) {
-    unsigned int l = LNum[label[i]];
-    unsigned int m=l;
-    m = m << (sizeof(n)*8) - bits;
-    m = m >> k;
-    n = n | m;
-    k += bits;
-    if (k >= (int)sizeof(n)*8) {
-      fwrite(&n, sizeof(n), 1, file);
-      k -= sizeof(n)*8;
-      if (k == 0)
-	n = 0;
-      else
-	n = l << (sizeof(n) * 8 - k);
-    }
-  }
-  if (k > 0)
-    fwrite(&n, sizeof(n), 1, file);
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  MakeCompactTransducer::store                                   */
-/*                                                                 */
-/*******************************************************************/
-
-void MakeCompactTransducer::store( FILE *file )
-
-{
-  fputc('c',file);
-  alphabet.store(file);
-  fwrite(&number_of_nodes, sizeof(number_of_nodes), 1, file);
-  fwrite(&number_of_arcs, sizeof(number_of_arcs), 1, file);
-  store_finalp(file);
-  store_first_arcs(file);
-  store_labels(file);
-  store_target_nodes(file);
-  if (ferror(file))
-    throw "Error encountered while writing transducer to file\n";
-}

data/ext/sfst_machine/mem.h

@@ -1,74 +0,0 @@
-/*******************************************************************/
-/*                                                                 */
-/*  FILE     mem.h                                                 */
-/*  MODULE   mem                                                   */
-/*  PROGRAM  SFST                                                  */
-/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
-/*                                                                 */
-/*  PURPOSE  memory management functions                           */
-/*                                                                 */
-/*******************************************************************/
-
-#ifndef _MEM_H_
-#define _MEM_H_
-
-#include <stdlib.h>
-
-#define MEMBUFFER_SIZE 100000
-
-
-/*****************  class Mem  *************************************/
-
-class Mem {
-
- private:
-
-  struct MemBuffer {
-    char buffer[MEMBUFFER_SIZE];
-    struct MemBuffer *next;
-  };
-
-  MemBuffer *first_buffer;
-  long pos;
-  void add_buffer() {
-    MemBuffer *mb=(MemBuffer*)malloc(sizeof(MemBuffer));
-    mb->next = first_buffer;
-    first_buffer = mb;
-    pos = 0;
-  }
-
-public:
-  Mem() { first_buffer = NULL; add_buffer(); }
-  ~Mem() { clear(); }
-
-  void clear() {
-    while (first_buffer) {
-      MemBuffer *next = first_buffer->next;
-      free(first_buffer);
-      first_buffer = next;
-    }
-    pos = 0;
-  }
-
-  void *alloc( size_t n ) {
-    void *result;
-    
-    /* do memory alignment to multiples of 4 */
-    if (n % 4)
-      n += 4 - (n % 4);
-    
-    if (first_buffer == NULL || pos+n > MEMBUFFER_SIZE)
-      add_buffer();
-    if (pos+n > MEMBUFFER_SIZE)
-      throw "Allocation of memory block larger than MEMBUFFER_SIZE attempted!";
-    
-    result = (void*)(first_buffer->buffer + pos);
-    pos += n;
-    return result;
-  }
-
-  //class MemError {};
-
-};
-
-#endif