ruby-sfst 0.1.0

data/ext/sfst_machine/utf8-scanner.ll

@@ -0,0 +1,179 @@
+%option 8Bit batch yylineno noyywrap
+
+/* the "incl" state is used to pick up the name of an include file */
+%x incl
+
+%{
+/*******************************************************************/
+/*                                                                 */
+/*  FILE     scanner.ll                                            */
+/*  MODULE   scanner                                               */
+/*  PROGRAM  SFST                                                  */
+/*  AUTHOR   Helmut Schmid, IMS, University of Stuttgart           */
+/*                                                                 */
+/*******************************************************************/
+
+#include <string.h>
+
+#include "interface.h"
+#include "fst-compiler.h"
+
+#define MAX_INCLUDE_DEPTH 10
+  
+int Include_Stack_Ptr = 0;
+YY_BUFFER_STATE Include_Stack[MAX_INCLUDE_DEPTH];
+char *Name_Stack[MAX_INCLUDE_DEPTH];
+int  Lineno_Stack[MAX_INCLUDE_DEPTH];
+
+char *FileName;
+
+bool UTF8=true;
+
+static char *unquote(char *string, bool del_quote=true) {
+  char *s=string, *result=string;
+  if (del_quote)
+    string++;
+
+  while (*string) {
+    if (*string == '\\')
+      string++;
+    *(s++) = *(string++);
+  }
+
+  if (del_quote)
+    s--;
+  *s = '\0';
+
+  return fst_strdup(result);
+}
+
+static void print_lineno() {
+  if (!Verbose)
+    return;
+  fputc('\r',stderr);
+  for( int i=0; i<Include_Stack_Ptr; i++ )
+    fputs("  ", stderr);
+  fprintf(stderr,"%s: %d", FileName, yylineno);
+}
+
+extern void yyerror(char *text);
+
+%}
+
+CC	[\x80-\xbf]
+C1	[A-Za-z0-9._/\-]
+C2	[A-Za-z0-9._/\-&()+,=?\^|~]
+C3	[A-Za-z0-9._/\-&()+,=?\^|~#<>]
+C4	[A-Za-z0-9._/\-&()+,=?\^|~$<>]
+C5	[\!-;\?-\[\]-\~=]
+FN	[A-Za-z0-9._/\-*+]
+
+%%
+
+#include           BEGIN(incl);
+<incl>[ \t]*       /* eat the whitespace */
+<incl>{FN}+        { error2("Missing quotes",yytext); }
+<incl>\"{FN}+\"    { /* got the include file name */
+                     FILE *file;
+                     char *name=unquote(yytext);
+                     if ( Include_Stack_Ptr >= MAX_INCLUDE_DEPTH )
+		       {
+			 fprintf( stderr, "Includes nested too deeply" );
+			 exit( 1 );
+		       }
+		     if (Verbose) fputc('\n', stderr);
+		     file = fopen( name, "rt" );
+		     if (!file)
+                       error2("Can't open include file",name);
+                     else
+                       {
+                         Name_Stack[Include_Stack_Ptr] = FileName;
+                         FileName = name;
+                         Lineno_Stack[Include_Stack_Ptr] = yylineno;
+			 yylineno = 1;
+		         Include_Stack[Include_Stack_Ptr++]=YY_CURRENT_BUFFER;
+		         yy_switch_to_buffer(
+                              yy_create_buffer(yyin, YY_BUF_SIZE));
+                         yyin = file;
+			 print_lineno();
+		         BEGIN(INITIAL);
+                       }
+                  }
+<<EOF>>           {
+                     if (Verbose)
+		       fputc('\n', stderr);
+                     if ( --Include_Stack_Ptr < 0 )
+		       yyterminate();
+		     else
+		       {
+                         free(FileName);
+                         FileName = Name_Stack[Include_Stack_Ptr];
+                         yylineno = Lineno_Stack[Include_Stack_Ptr];
+			 yy_delete_buffer( YY_CURRENT_BUFFER );
+			 yy_switch_to_buffer(Include_Stack[Include_Stack_Ptr]);
+                       }
+                  }
+
+
+^[ \t]*\%.*\r?\n  { print_lineno();  /* ignore comments */ }
+
+\%.*\\[ \t]*\r?\n { print_lineno();  /* ignore comments */ }
+
+\%.*              { /* ignore comments */ }
+
+
+^[ \t]*ALPHABET[ \t]*= { return ALPHA; }
+
+\|\|              { return COMPOSE; }
+"<=>"             { yylval.type = twol_both; return ARROW; }
+"=>"              { yylval.type = twol_right;return ARROW; }
+"<="              { yylval.type = twol_left; return ARROW; }
+"^->"             { yylval.rtype = repl_up;   return REPLACE; }
+"_->"             { yylval.rtype = repl_down; return REPLACE; }
+"/->"             { yylval.rtype = repl_right;return REPLACE; }
+"\\->"            { yylval.rtype = repl_left; return REPLACE; }
+">>"              { return PRINT; }
+"<<"              { return INSERT; }
+"__"              { return POS; }
+"^_"              { return REV; }
+
+[.,{}\[\]()&!?|*+:=_\^\-] { return yytext[0]; }
+
+\$=({C3}|(\\.))+\$ { yylval.name = fst_strdup(yytext); return RVAR; }
+
+\$({C3}|(\\.))+\$ { yylval.name = fst_strdup(yytext); return VAR; }
+
+#=({C4}|(\\.))+# { yylval.name = fst_strdup(yytext); return RSVAR; }
+
+#({C4}|(\\.))+# { yylval.name = fst_strdup(yytext); return SVAR; }
+
+\<({C5}|\\.)*\>   { yylval.name = unquote(yytext,false); return SYMBOL; }
+
+\"<{FN}+>\" { 
+                    yylval.value = unquote(yytext)+1;
+		    yylval.value[strlen(yylval.value)-1] = 0;
+                    return STRING2;
+                  }
+
+\"{FN}+\" { 
+                    yylval.value = unquote(yytext);
+                    return STRING;
+                  }
+
+[ \t]             { /* ignored */ }
+\\[ \t]*([ \t]\%.*)?\r?\n { print_lineno(); /* ignored */ }
+\r?\n             { print_lineno(); return NEWLINE; }
+
+\\[0-9]+          { long l=atol(yytext+1); 
+		    if (l <= 1114112) { yylval.uchar=l; return CHARACTER; }
+		    yyerror("invalid expression");
+                  }
+
+
+\\.                { yylval.value=fst_strdup(yytext+1); return UTF8CHAR; }
+[\x00-\x7f]        { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
+[\xc0-\xdf]{CC}    { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
+[\xe0-\xef]{CC}{2} { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
+[\xf0-\xff]{CC}{3} { yylval.value=fst_strdup(yytext); return UTF8CHAR; }
+
+%%

data/ext/sfst_machine/utf8.C

@@ -0,0 +1,146 @@
+
+/*******************************************************************/
+/*                                                                 */
+/*     File: utf8.C                                                */
+/*   Author: Helmut Schmid                                         */
+/*  Purpose:                                                       */
+/*  Created: Mon Sep  5 17:49:16 2005                              */
+/* Modified: Mon Mar  3 11:00:53 2008 (schmid)                     */
+/*                                                                 */
+/*******************************************************************/
+
+#include "string.h"
+
+#include "utf8.h"
+
+const unsigned char get3LSbits=7;
+const unsigned char get4LSbits=15;
+const unsigned char get5LSbits=31;
+const unsigned char get6LSbits=63;
+
+const unsigned char set1MSbits=128;
+const unsigned char set2MSbits=192;
+const unsigned char set3MSbits=224;
+const unsigned char set4MSbits=240;
+
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  int2utf8                                                       */
+/*                                                                 */
+/*******************************************************************/
+
+char *int2utf8( unsigned int sym )
+
+{
+  static unsigned char ch[5];
+
+  if (sym < 128) {
+    // 1-byte UTF8 symbol, 7 bits
+    ch[0] = sym;
+    ch[1] = 0;
+  }
+  
+  else if (sym < 2048) {
+    // 2-byte UTF8 symbol, 5+6 bits
+    ch[0] = (sym >> 6) | set2MSbits;
+    ch[1] = (sym & get6LSbits) | set1MSbits;
+    ch[2] = 0;
+  }
+  
+  else if (sym < 65536) {
+    // 3-byte UTF8 symbol, 4+6+6 bits
+    ch[0] = (sym >> 12) | set3MSbits;
+    ch[1] = ((sym >> 6) & get6LSbits) | set1MSbits;
+    ch[2] = (sym & get6LSbits) | set1MSbits;
+    ch[3] = 0;
+  }
+  
+  else if (sym < 2097152) {
+    // 4-byte UTF8 symbol, 3+6+6+6 bits
+    ch[0] = (sym >> 18) | set4MSbits;
+    ch[1] = ((sym >> 12) & get6LSbits) | set1MSbits;
+    ch[2] = ((sym >> 6) & get6LSbits) | set1MSbits;
+    ch[3] = (sym & get6LSbits) | set1MSbits;
+    ch[4] = 0;
+  }
+  
+  else
+    return NULL;
+
+  return (char*)ch;
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  utf8toint                                                      */
+/*                                                                 */
+/*******************************************************************/
+
+unsigned int utf8toint( char **s )
+
+{
+  int bytes_to_come;
+  unsigned int result=0;
+  unsigned char c=(unsigned char)**s;
+
+  if (c >= (unsigned char)set4MSbits) { // 1111xxxx
+    bytes_to_come = 3;
+    result = (result << 3) | (c & get3LSbits);
+  }
+      
+  else if (c >= (unsigned char) set3MSbits) { // 1110xxxx
+    // start of a three-byte symbol
+    bytes_to_come = 2;
+    result = (result << 4) | (c & get4LSbits);
+  }
+      
+  else if (c >= (unsigned char) set2MSbits) { // 1100xxxx
+    // start of a two-byte symbol
+    bytes_to_come = 1;
+    result = (result << 5) | (c & get5LSbits);
+  }
+      
+  else if (c < (unsigned char) set1MSbits) { // 0100xxxx
+    // one-byte symbol
+    bytes_to_come = 0;
+    result = c;
+  }
+
+  else
+    return 0; // error
+
+  while (bytes_to_come > 0) {
+    bytes_to_come--;
+    (*s)++;
+    c = (unsigned char)**s;
+    if (c < (unsigned char) set2MSbits &&
+	c >= (unsigned char) set1MSbits)    // 1000xxxx
+      {
+	result = (result << 6) | (c & get6LSbits);
+      }
+    else
+      return 0;
+  }
+
+  (*s)++;
+  return result;
+}
+
+
+/*******************************************************************/
+/*                                                                 */
+/*  utf8toint                                                      */
+/*                                                                 */
+/*******************************************************************/
+
+unsigned int utf8toint( char *s )
+
+{
+  unsigned int result = utf8toint( &s );
+  if (*s == 0) // all bytes converted?
+    return result;
+  return 0;
+}

data/ext/sfst_machine/utf8.h

@@ -0,0 +1,19 @@
+
+/*******************************************************************/
+/*                                                                 */
+/*     File: utf8.h                                                */
+/*   Author: Helmut Schmid                                         */
+/*  Purpose:                                                       */
+/*  Created: Mon Sep  5 17:49:16 2005                              */
+/* Modified: Mon Apr  7 08:26:39 2008 (schmid)                     */
+/*                                                                 */
+/*******************************************************************/
+
+#ifndef _UTF8_H_
+#define _UTF8_H_
+
+unsigned int utf8toint( char *s );
+unsigned int utf8toint( char **s );
+char *int2utf8( unsigned int );
+
+#endif

data/lib/sfst.rb

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+#
+# sfst.rb - SFST interface
+#
+# Written by Marius L. Jøhndal, 2008.
+#
+require 'sfst_machine'
+
+module SFST
+  # Compiles an SFST transducer +source+ and saves it as +machine+.
+  #
+  # ==== Options
+  # compact:: Compile a compact transducer.
+  def self.compile(source, machine, options = {})
+    unless options[:compact]
+      _compile_regular(source, machine)
+    else
+      _compile_compact(source, machine)
+    end
+  end
+
+  # A regular, i.e. not a compact, transducer.
+  class RegularTransducer
+    def initialize(file)
+      @fst = RegularTransducerMachine.new(file)
+    end
+
+    # Analyses a string +string+. Returns an array of analysed
+    # strings if the string is accepted, or an empty array if not.
+    #
+    # ==== Options
+    # * +symbol_sequence+ - Return each analysis as a sequence of symbols.
+    #   Multicharacter symbols will be strings on the form +<symbol>+.
+    def analyze(string, options = {})
+      x = []
+      @fst._analyze(string) do |a| 
+        if options[:symbol_sequence]
+          x << a.map { |s| s.match(/^<(.*)>$/) ? $1.to_sym : s }
+        else
+          x << a.join
+        end
+      end
+      x
+    end
+
+    # Checks if the string +string+ is accepted for analysis.
+    def accepted_analysis?(string)
+      @fst._analyze(string)
+    end
+
+    # Generates a string +string+. Returns an array of generated
+    # strings if the string is accepted or an empty array if not.
+    def generate(string)
+      x = []
+      @fst._generate(string) { |a| x << a.join }
+      x
+    end
+
+    # Checks if the string +string+ is accepted for generating.
+    def accepted_generating?(string)
+      @fst._generate(string)
+    end
+
+    # Generates upper or lower level or both. This only works with
+    # non-compact transducers.
+    #
+    # ==== Options
+    # * +levels+ - if <tt>:upper</tt>, generates only upper level. If <tt>:lower</tt> generates 
+    #   only lower level. If <tt>:both</tt>, generates both. Default is <tt>:both</tt>.
+    # * +epsilons+ - if +true+, produces epsilons. Default is +false+.
+    def generate_language(options = {}, &block)
+      @fst._generate_language(options[:levels] || :both, options[:epsilons] ? :all : :noepsilons, &block)
+    end
+
+    alias :analyse :analyze
+  end
+
+  # A compact transducer.
+  class CompactTransducer < CompactTransducerMachine
+    def initialize(file)
+      @fst = CompactTransducerMachine.new(file)
+    end
+
+    # Checks if the string +string+ is accepted for analysis.
+    def accepted_analysis?(string)
+      @fst._analyze(string)
+    end
+
+    # Analyses a string +string+. Returns an array of analysed
+    # strings if the string is accepted or an empty array if not.
+    def analyze(form)
+      x = []
+      @fst._analyze(form) { |a| x << a }
+      x
+    end
+
+    alias :analyse :analyze
+  end
+end

data/ruby-sfst.gemspec

@@ -0,0 +1,34 @@
+Gem::Specification.new do |s|
+  s.name = %q{ruby-sfst}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new("= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Marius L. J\303\270hndal"]
+  s.date = %q{2008-08-04}
+  s.description = %q{A wrapper for the Stuttgart Finite State Transducer Tools (SFST).}
+  s.email = %q{mariuslj (at) ifi [dot] uio (dot) no}
+  s.extensions = ["ext/sfst_machine/extconf.rb"]
+  s.extra_rdoc_files = ["README.rdoc", "lib/sfst.rb"]
+  s.files = ["README.rdoc", "Rakefile", "Manifest", "test/test_sfst.rb", "test/test_sfst.fst", "CHANGELOG", "ext/sfst_machine/fst-compiler.h", "ext/sfst_machine/utf8.C", "ext/sfst_machine/operators.C", "ext/sfst_machine/utf8-scanner.ll", "ext/sfst_machine/determinise.C", "ext/sfst_machine/interface.C", "ext/sfst_machine/compact.h", "ext/sfst_machine/basic.h", "ext/sfst_machine/fst.h", "ext/sfst_machine/make-compact.h", "ext/sfst_machine/fst-compiler.yy", "ext/sfst_machine/mem.h", "ext/sfst_machine/compact.C", "ext/sfst_machine/basic.C", "ext/sfst_machine/interface.h", "ext/sfst_machine/sfst_machine.cc", "ext/sfst_machine/extconf.rb", "ext/sfst_machine/alphabet.C", "ext/sfst_machine/fst.C", "ext/sfst_machine/alphabet.h", "ext/sfst_machine/make-compact.C", "ext/sfst_machine/fst-compiler.C", "ext/sfst_machine/utf8.h", "ext/sfst_machine/utf8-scanner.C", "lib/sfst.rb", "ruby-sfst.gemspec"]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/mlj/ruby-sfst}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Ruby-sfst", "--main", "README.rdoc"]
+  s.require_paths = ["lib", "ext"]
+  s.rubyforge_project = %q{sfst}
+  s.rubygems_version = %q{1.2.0}
+  s.summary = %q{A wrapper for the Stuttgart Finite State Transducer Tools (SFST).}
+  s.test_files = ["test/test_sfst.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if current_version >= 3 then
+      s.add_development_dependency(%q<echoe>, [">= 0"])
+    else
+      s.add_dependency(%q<echoe>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<echoe>, [">= 0"])
+  end
+end

data/test/test_sfst.fst

@@ -0,0 +1,3 @@
+ALPHABET = [a-z]
+$test$ = ({bar}:{foo} | {baz}:{foo})
+$test$

data/test/test_sfst.rb

@@ -0,0 +1,119 @@
+require 'sfst'
+require 'test/unit'
+
+TEST_DIRECTORY = File.expand_path(File.dirname(__FILE__))
+TEST_SCRIPT_FILE = File.join(TEST_DIRECTORY, 'test_sfst.fst')
+TEST_COMPILED_FILE = File.join(TEST_DIRECTORY, 'test_sfst.a')
+TEST_COMPILED_COMPACT_FILE = File.join(TEST_DIRECTORY, 'test_sfst_compact.a')
+TEST_COMPILED_REGULAR_FILE = File.join(TEST_DIRECTORY, 'test_sfst_regular.a')
+
+class SFSTTestCase < Test::Unit::TestCase
+  def test_sfst_compile_regular
+    SFST::compile(TEST_SCRIPT_FILE, TEST_COMPILED_FILE)
+    SFST::compile(TEST_SCRIPT_FILE, TEST_COMPILED_FILE, :compact => false)
+  end
+
+  def test_sfst_compile_compact
+    SFST::compile(TEST_SCRIPT_FILE, TEST_COMPILED_FILE, :compact => true)
+  end
+end
+
+class RegularTransducerTestCase < Test::Unit::TestCase
+  def setup
+    SFST::compile(TEST_SCRIPT_FILE, TEST_COMPILED_REGULAR_FILE, :compact => false)
+  end
+
+  def test_analyze_acceptance
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal true, fst.accepted_analysis?('foo')
+    assert_equal false, fst.accepted_analysis?('fox')
+  end
+
+  def test_analyze
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal ['bar', 'baz'], fst.analyse('foo').sort
+
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal [], fst.analyse('fox').sort
+  end
+
+  def test_analyze_symbol_sequence
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal [['b', 'a', 'r'], ['b', 'a', 'z']], fst.analyse('foo', :symbol_sequence => true).sort
+
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal [], fst.analyse('fox', :symbol_sequence => true).sort
+  end
+
+  def test_generate_acceptance
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal true, fst.accepted_generating?('bar')
+    assert_equal true, fst.accepted_generating?('baz')
+    assert_equal false, fst.accepted_generating?('bax')
+  end
+
+  def test_generate
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal ['foo'], fst.generate('bar').sort
+
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal ['foo'], fst.generate('baz').sort
+
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    assert_equal [], fst.generate('bax').sort
+  end
+
+  def test_generate_language_default
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    a = []
+    fst.generate_language do |u|
+      a << u.collect { |pair| pair.join(':') }.join
+    end
+    assert_equal ['b:fa:or:o', 'b:fa:oz:o'], a.sort
+  end
+
+  def test_generate_language_both
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    a = []
+    fst.generate_language(:levels => :both) do |u|
+      a << u.collect { |pair| pair.join(':') }.join
+    end
+    assert_equal ['b:fa:or:o', 'b:fa:oz:o'], a.sort
+  end
+
+  def test_generate_language_upper
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    a = []
+    fst.generate_language(:levels => :upper) do |u|
+      a << u.join
+    end
+    assert_equal ['foo'], a.sort
+  end
+
+  def test_generate_language_lower
+    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
+    a = []
+    fst.generate_language(:levels => :lower) do |u|
+      a << u.join
+    end
+    assert_equal ['bar', 'baz'], a.sort
+  end
+end
+
+class CompactTransducerTestCase < Test::Unit::TestCase
+  def setup
+    SFST::compile(TEST_SCRIPT_FILE, TEST_COMPILED_COMPACT_FILE, :compact => true)
+  end
+
+  def test_analyze_acceptance
+    fst = SFST::CompactTransducer.new(TEST_COMPILED_COMPACT_FILE)
+    assert_equal true, fst.accepted_analysis?('foo')
+    assert_equal false, fst.accepted_analysis?('fox')
+  end
+
+  def test_analyze
+    fst = SFST::CompactTransducer.new(TEST_COMPILED_COMPACT_FILE)
+    assert_equal ['bar', 'baz'], fst.analyse('foo').sort
+    assert_equal [], fst.analyse('fox').sort
+  end
+end