ruby-sfst 0.4.3 → 0.4.4

data/ext/{sfst_machine → sfst}/sfst_machine.cc

@@ -14,19 +14,19 @@ VALUE mSFST = Qnil;
 VALUE mCompactTransducer = Qnil;
 VALUE mRegularTransducer = Qnil;
 
-static void compact_transducer_free(CompactTransducer *t)
+static void compact_transducer_free(SFST::CompactTransducer *t)
 {
   if (t)
     delete t;
 }
 
-static void compact_transducer_mark(CompactTransducer *t)
+static void compact_transducer_mark(SFST::CompactTransducer *t)
 {
 }
 
 static VALUE compact_transducer_alloc(VALUE klass)
 {
-  CompactTransducer *t = NULL;
+  SFST::CompactTransducer *t = NULL;
 
   return Data_Wrap_Struct(klass, compact_transducer_mark, compact_transducer_free, t);
 }
@@ -34,7 +34,7 @@ static VALUE compact_transducer_alloc(VALUE klass)
 static VALUE compact_transducer_init(VALUE obj, VALUE filename)
 {
   FILE *file;
-  CompactTransducer *t;
+  SFST::CompactTransducer *t;
 
   file = fopen(RSTRING_PTR(filename), "rb");
 
@@ -43,7 +43,7 @@ static VALUE compact_transducer_init(VALUE obj, VALUE filename)
   }
 
   try {
-    t = new CompactTransducer(file);
+    t = new SFST::CompactTransducer(file);
     fclose(file);
   }
   catch (const char *p) {
@@ -57,13 +57,13 @@ static VALUE compact_transducer_init(VALUE obj, VALUE filename)
 static VALUE compact_transducer_analyze(VALUE self, VALUE string)
 {
   VALUE accepted = Qfalse;
-  CompactTransducer *t;
+  SFST::CompactTransducer *t;
 
   Check_Type(string, T_STRING);
 
-  Data_Get_Struct(self, CompactTransducer, t);
+  Data_Get_Struct(self, SFST::CompactTransducer, t);
 
-  std::vector<CAnalysis> analyses;
+  std::vector<SFST::CAnalysis> analyses;
   t->analyze_string(RSTRING_PTR(string), analyses);
 
   for (size_t k = 0; k < analyses.size(); k++) {
@@ -79,19 +79,19 @@ static VALUE compact_transducer_analyze(VALUE self, VALUE string)
   return accepted;
 }
 
-static void regular_transducer_free(Transducer *t)
+static void regular_transducer_free(SFST::Transducer *t)
 {
   if (t)
     delete t;
 }
 
-static void regular_transducer_mark(Transducer *t)
+static void regular_transducer_mark(SFST::Transducer *t)
 {
 }
 
 static VALUE regular_transducer_alloc(VALUE klass)
 {
-  Transducer *t = NULL;
+  SFST::Transducer *t = NULL;
 
   return Data_Wrap_Struct(klass, regular_transducer_mark, regular_transducer_free, t);
 }
@@ -99,7 +99,7 @@ static VALUE regular_transducer_alloc(VALUE klass)
 static VALUE regular_transducer_init(VALUE obj, VALUE filename)
 {
   FILE *file;
-  Transducer *t;
+  SFST::Transducer *t;
 
   file = fopen(RSTRING_PTR(filename), "rb");
 
@@ -108,7 +108,7 @@ static VALUE regular_transducer_init(VALUE obj, VALUE filename)
   }
 
   try {
-    t = new Transducer(file);
+    t = new SFST::Transducer(file);
     fclose(file);
   }
   catch (const char *p) {
@@ -123,7 +123,7 @@ using std::vector;
 
 enum { BOTH, LOWER, UPPER };
 
-static VALUE _alphabet_to_rb_str(Alphabet *a, Character c)
+static VALUE _alphabet_to_rb_str(SFST::Alphabet *a, SFST::Character c)
 {
   const char *s = a->code2symbol(c);
 
@@ -151,68 +151,79 @@ static VALUE rb_ary_push_pair(VALUE ary, VALUE a, VALUE b)
   return ary;
 }
 
-static void _regular_transducer_generate(Transducer *t, Node *node,
-    Node2Int &visitations, VALUE a, int mode, bool epsilons)
-{
-  if (node->is_final())
-    rb_yield(a);
-
-  visitations[node]++;
-
-  vector<Arc*> arc;
-  for (ArcsIter p(node->arcs()); p; p++) {
-    Arc *a = p;
-    Node *n = a->target_node();
-    size_t i;
-    for (i = 0; i < arc.size(); i++)
-      if (visitations[n] < visitations[arc[i]->target_node()])
-        break;
-    arc.push_back(NULL);
-    for (size_t k = arc.size() - 1; k > i; k--)
-      arc[k] = arc[k - 1];
-    arc[i] = a;
-  }
+class Gen {
+public:
 
-  for (size_t i = 0; i < arc.size(); i++) {
-    Label l = arc[i]->label();
-    VALUE lower, upper;
+  SFST::Node *node;
+  SFST::Index previous;
+  SFST::Label label;
 
-    Character lc = l.lower_char();
-    if ((mode == BOTH || mode == LOWER) && (epsilons || lc != Label::epsilon)) {
-      lower = _alphabet_to_rb_str(&(t->alphabet), lc);
-    } else
-      lower = Qnil;
+  Gen(SFST::Node *n, SFST::Label l = SFST::Label::epsilon, SFST::Index p = SFST::undef):
+    node(n), previous(p), label(l) {}
 
-    Character uc = l.upper_char();
-    if ((mode == BOTH || mode == UPPER) && (epsilons || uc != Label::epsilon)) {
-      upper = _alphabet_to_rb_str(&(t->alphabet), uc);
-    } else
-      upper = Qnil;
+  void print(vector<Gen> &paths, VALUE a, int levels, bool epsilons, SFST::Transducer *t) {
+    if (previous != SFST::undef) {
+      paths[previous].print(paths, a, levels, epsilons, t);
+
+      SFST::Label l = label;
+
+      VALUE lower, upper;
+
+      SFST::Character lc = l.lower_char();
+      if ((levels == BOTH || levels == LOWER) && (epsilons || lc != SFST::Label::epsilon)) {
+        lower = _alphabet_to_rb_str(&(t->alphabet), lc);
+      } else
+        lower = Qnil;
+
+      SFST::Character uc = l.upper_char();
+      if ((levels == BOTH || levels == UPPER) && (epsilons || uc != SFST::Label::epsilon)) {
+        upper = _alphabet_to_rb_str(&(t->alphabet), uc);
+      } else
+        upper = Qnil;
 
-    switch (mode) {
-      case BOTH:
-        rb_ary_push_pair(a, lower, upper);
-        break;
+      switch (levels) {
+        case BOTH:
+          rb_ary_push_pair(a, lower, upper);
+          break;
 
-      case UPPER:
-        rb_ary_push(a, upper);
-        break;
+        case UPPER:
+          rb_ary_push(a, upper);
+          break;
 
-      case LOWER:
-        rb_ary_push(a, lower);
-        break;
+        case LOWER:
+          rb_ary_push(a, lower);
+          break;
+      }
     }
+  }
+};
+
+static void _generate(SFST::Transducer *t, int levels, bool epsilons)
+{
+  vector<Gen> paths;
+  paths.push_back(Gen(t->root_node()));
+
+  for (size_t i = 0; i < paths.size(); i++) {
+    Gen &gen = paths[i];
+    SFST::Node *node = gen.node;
 
-    _regular_transducer_generate(t, arc[i]->target_node(), visitations, a, mode, epsilons);
+    if (node->is_final()) {
+      VALUE a = rb_ary_new();
+      gen.print(paths, a, levels, epsilons, t);
+      rb_yield(a);
+    }
 
-    rb_ary_pop(a);
+    for (SFST::ArcsIter p(node->arcs()); p; p++) {
+      SFST::Arc *arc = p;
+      paths.push_back(Gen(arc->target_node(), arc->label(), (SFST::Index)i));
+    }
   }
 }
 
 static VALUE regular_transducer_generate_language(VALUE self, VALUE levels_arg, VALUE mode_arg)
 {
-  Transducer *t;
-  Data_Get_Struct(self, Transducer, t);
+  SFST::Transducer *t;
+  Data_Get_Struct(self, SFST::Transducer, t);
 
   static ID id_upper = rb_intern("upper");
   static ID id_lower = rb_intern("lower");
@@ -246,8 +257,8 @@ static VALUE regular_transducer_generate_language(VALUE self, VALUE levels_arg,
   if (!rb_block_given_p())
     rb_raise(rb_eRuntimeError, "block expected");
 
-  Node2Int visitations;
-  Transducer *a2;
+  SFST::Transducer *a2;
+
   switch (levels) {
     case UPPER:
       a2 = &(t->upper_level().minimise());
@@ -259,13 +270,13 @@ static VALUE regular_transducer_generate_language(VALUE self, VALUE levels_arg,
       a2 = t;
       break;
   }
-  _regular_transducer_generate(a2, a2->root_node(), visitations, rb_ary_new(),
-      levels, epsilons);
+
+  _generate(a2, levels, epsilons);
 
   return Qnil;
 }
 
-static bool _regular_transducer_yield(Transducer *t, Node *node, VALUE result_array)
+static bool _regular_transducer_yield(SFST::Transducer *t, SFST::Node *node, VALUE result_array)
 {
   int accepted = 0;
 
@@ -284,9 +295,9 @@ static bool _regular_transducer_yield(Transducer *t, Node *node, VALUE result_ar
     accepted = 1;
   }
 
-  for (ArcsIter i(node->arcs()); i; i++) {
-    Arc *arc = i;
-    Label l = arc->label();
+  for (SFST::ArcsIter i(node->arcs()); i; i++) {
+    SFST::Arc *arc = i;
+    SFST::Label l = arc->label();
 
     rb_ary_push(result_array, rb_str_new2(t->alphabet.write_label(l)));
 
@@ -300,10 +311,10 @@ static bool _regular_transducer_yield(Transducer *t, Node *node, VALUE result_ar
   return accepted == 1 ? true : false;
 }
 
-static VALUE _regular_transducer_analyze_or_generate(Transducer *t, VALUE string, bool generate)
+static VALUE _regular_transducer_analyze_or_generate(SFST::Transducer *t, VALUE string, bool generate)
 {
-  Transducer *a2, *a3;
-  Transducer a1(RSTRING_PTR(string), &(t->alphabet), false);
+  SFST::Transducer *a2, *a3;
+  SFST::Transducer a1(RSTRING_PTR(string), &(t->alphabet), false);
   if (generate) {
     a2 = &(a1 || *t);
     a3 = &(a2->upper_level());
@@ -324,23 +335,23 @@ static VALUE _regular_transducer_analyze_or_generate(Transducer *t, VALUE string
 
 static VALUE regular_transducer_generate(VALUE self, VALUE string)
 {
-  Transducer *t;
+  SFST::Transducer *t;
   Check_Type(string, T_STRING);
-  Data_Get_Struct(self, Transducer, t);
+  Data_Get_Struct(self, SFST::Transducer, t);
   return _regular_transducer_analyze_or_generate(t, string, true);
 }
 
 static VALUE regular_transducer_analyze(VALUE self, VALUE string)
 {
-  Transducer *t;
+  SFST::Transducer *t;
   Check_Type(string, T_STRING);
-  Data_Get_Struct(self, Transducer, t);
+  Data_Get_Struct(self, SFST::Transducer, t);
   return _regular_transducer_analyze_or_generate(t, string, false);
 }
 
 extern "C"
 
-void Init_sfst_machine(void)
+void Init_sfst(void)
 {
   mSFST = rb_define_module("SFST");
 

data/ext/sfst/sgi.h

@@ -0,0 +1,72 @@
+/*******************************************************************/
+/*                                                                 */
+/*     File: sgi.h                                                 */
+/*   Author: Helmut Schmid                                         */
+/*  Purpose:                                                       */
+/*  Created: Thu Sep 11 15:58:25 2008                              */
+/* Modified: Fri Sep 12 08:17:03 2008 (schmid)                     */
+/* Modified: Wed May 26 12:54:00 2010 (hfst)                       */
+/*******************************************************************/
+
+#ifndef _SGI_INCLUDED
+#define _SGI_INCLUDED
+
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif
+
+#if HAVE_BACKWARD_HASH_MAP
+#  include <backward/hash_map>
+#elif HAVE_EXT_HASH_MAP
+#  include <ext/hash_map>
+#elif HAVE_HASH_MAP
+#  include <hash_map>
+#elif SGIext
+#  include <ext/hash_map>
+#elif SGI__gnu_cxx
+#  include <ext/hash_map>
+#else
+#  warning "unknown hash_map"
+#  include <hash_map>
+#endif
+#if HAVE_BACKWARD_HASH_SET
+#  include <backward/hash_set>
+#elif HAVE_EXT_HASH_SET
+#  include <ext/hash_set>
+#elif HAVE_HASH_SET
+#  include <hash_set>
+#elif SGIext
+#  include <ext/hash_set>
+#elif SGI__gnu_cxx
+#  include <ext/hash_set>
+#else
+#  warning "missing hash_set"
+#  include <hash_set>
+#endif
+
+// Hfst addition
+namespace SFST 
+{
+// from <http://gcc.gnu.org/onlinedocs/libstdc++/manual/backwards.html>
+#ifdef __GNUC__
+#  if __GNUC__ < 3
+  using ::hash_map;
+  using ::hash_set;
+  using ::hash;
+#  elif __GNUC__ == 3 && __GNUC_MINOR__ == 0
+  using std::hash_map;
+  using std::hash_set;
+  using std::hash;
+#  else 
+  using __gnu_cxx::hash_map;
+  using __gnu_cxx::hash_set;
+  using __gnu_cxx::hash;
+#  endif
+#else
+  using std::hash_map;
+  using std::hash_set;
+  using std::hash;
+#endif
+}
+
+#endif

data/ext/sfst/utf8.cc

@@ -0,0 +1,149 @@
+
+/*******************************************************************/
+/*                                                                 */
+/*     File: utf8.C                                                */
+/*   Author: Helmut Schmid                                         */
+/*  Purpose:                                                       */
+/*  Created: Mon Sep  5 17:49:16 2005                              */
+/* Modified: Wed Sep 29 15:08:34 2010 (schmid)                     */
+/*                                                                 */
+/*******************************************************************/
+
+#include "string.h"
+#include "utf8.h"
+
+namespace SFST {
+
+  const unsigned char get3LSbits=7;
+  const unsigned char get4LSbits=15;
+  const unsigned char get5LSbits=31;
+  const unsigned char get6LSbits=63;
+
+  const unsigned char set1MSbits=128;
+  const unsigned char set2MSbits=192;
+  const unsigned char set3MSbits=224;
+  const unsigned char set4MSbits=240;
+
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  int2utf8                                                       */
+  /*                                                                 */
+  /*******************************************************************/
+
+  char *int2utf8( unsigned int sym )
+
+  {
+    static unsigned char ch[5];
+
+    if (sym < 128) {
+      // 1-byte UTF8 symbol, 7 bits
+      ch[0] = (unsigned char)sym;
+      ch[1] = 0;
+    }
+  
+    else if (sym < 2048) {
+      // 2-byte UTF8 symbol, 5+6 bits
+      ch[0] = (unsigned char)((sym >> 6) | set2MSbits);
+      ch[1] = (unsigned char)((sym & get6LSbits) | set1MSbits);
+      ch[2] = 0;
+    }
+  
+    else if (sym < 65536) {
+      // 3-byte UTF8 symbol, 4+6+6 bits
+      ch[0] = (unsigned char)((sym >> 12) | set3MSbits);
+      ch[1] = (unsigned char)(((sym >> 6) & get6LSbits) | set1MSbits);
+      ch[2] = (unsigned char)((sym & get6LSbits) | set1MSbits);
+      ch[3] = 0;
+    }
+  
+    else if (sym < 2097152) {
+      // 4-byte UTF8 symbol, 3+6+6+6 bits
+      ch[0] = (unsigned char)((sym >> 18) | set4MSbits);
+      ch[1] = (unsigned char)(((sym >> 12) & get6LSbits) | set1MSbits);
+      ch[2] = (unsigned char)(((sym >> 6) & get6LSbits) | set1MSbits);
+      ch[3] = (unsigned char)((sym & get6LSbits) | set1MSbits);
+      ch[4] = 0;
+    }
+  
+    else
+      return NULL;
+
+    return (char*)ch;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  utf8toint                                                      */
+  /*                                                                 */
+  /*******************************************************************/
+
+  unsigned int utf8toint( char **s )
+
+  {
+    int bytes_to_come;
+    unsigned int result=0;
+    unsigned char c=(unsigned char)**s;
+
+    if (c >= (unsigned char)set4MSbits) { // 1111xxxx
+      bytes_to_come = 3;
+      result = (result << 3) | (c & get3LSbits);
+    }
+      
+    else if (c >= (unsigned char) set3MSbits) { // 1110xxxx
+      // start of a three-byte symbol
+      bytes_to_come = 2;
+      result = (result << 4) | (c & get4LSbits);
+    }
+      
+    else if (c >= (unsigned char) set2MSbits) { // 1100xxxx
+      // start of a two-byte symbol
+      bytes_to_come = 1;
+      result = (result << 5) | (c & get5LSbits);
+    }
+      
+    else if (c < (unsigned char) set1MSbits) { // 0100xxxx
+      // one-byte symbol
+      bytes_to_come = 0;
+      result = c;
+    }
+
+    else
+      return 0; // error
+
+    while (bytes_to_come > 0) {
+      bytes_to_come--;
+      (*s)++;
+      c = (unsigned char)**s;
+      if (c < (unsigned char) set2MSbits &&
+	  c >= (unsigned char) set1MSbits)    // 1000xxxx
+	{
+	  result = (result << 6) | (c & get6LSbits);
+	}
+      else
+	return 0;
+    }
+
+    (*s)++;
+    return result;
+  }
+
+
+  /*******************************************************************/
+  /*                                                                 */
+  /*  utf8toint                                                      */
+  /*                                                                 */
+  /*******************************************************************/
+
+  unsigned int utf8toint( char *s )
+
+  {
+    unsigned int result = utf8toint( &s );
+    if (*s == 0) // all bytes converted?
+      return result;
+    return 0;
+  }
+
+}