ruby-sfst 0.4.3 → 0.4.4

data/ext/sfst_machine/sgi.h

@@ -1,44 +0,0 @@
-
-/*******************************************************************/
-/*                                                                 */
-/*     File: sgi.h                                                 */
-/*   Author: Helmut Schmid                                         */
-/*  Purpose:                                                       */
-/*  Created: Thu Sep 11 15:58:25 2008                              */
-/* Modified: Fri Sep 12 08:17:03 2008 (schmid)                     */
-/*                                                                 */
-/*******************************************************************/
-
-#ifndef _SGI_INCLUDED
-#define _SGI_INCLUDED
-
-
-#ifdef SGIext
-
-#include <ext/hash_map>
-#include <ext/hash_set>
-using std::hash_map;
-using std::hash_set;
-using std::hash;
-
-#else
-
-#ifdef SGI__gnu_cxx
-
-#include <ext/hash_map>
-#include <ext/hash_set>
-
-#else
-
-#include <backward/hash_map>
-#include <backward/hash_set>
-
-#endif
-
-using __gnu_cxx::hash_map;
-using __gnu_cxx::hash_set;
-using __gnu_cxx::hash;
-
-#endif
-
-#endif

data/ext/sfst_machine/utf8.cc

@@ -1,146 +0,0 @@
-
-/*******************************************************************/
-/*                                                                 */
-/*     File: utf8.C                                                */
-/*   Author: Helmut Schmid                                         */
-/*  Purpose:                                                       */
-/*  Created: Mon Sep  5 17:49:16 2005                              */
-/* Modified: Mon Mar  3 11:00:53 2008 (schmid)                     */
-/*                                                                 */
-/*******************************************************************/
-
-#include "string.h"
-
-#include "utf8.h"
-
-const unsigned char get3LSbits=7;
-const unsigned char get4LSbits=15;
-const unsigned char get5LSbits=31;
-const unsigned char get6LSbits=63;
-
-const unsigned char set1MSbits=128;
-const unsigned char set2MSbits=192;
-const unsigned char set3MSbits=224;
-const unsigned char set4MSbits=240;
-
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  int2utf8                                                       */
-/*                                                                 */
-/*******************************************************************/
-
-char *int2utf8( unsigned int sym )
-
-{
-  static unsigned char ch[5];
-
-  if (sym < 128) {
-    // 1-byte UTF8 symbol, 7 bits
-    ch[0] = sym;
-    ch[1] = 0;
-  }
-  
-  else if (sym < 2048) {
-    // 2-byte UTF8 symbol, 5+6 bits
-    ch[0] = (sym >> 6) | set2MSbits;
-    ch[1] = (sym & get6LSbits) | set1MSbits;
-    ch[2] = 0;
-  }
-  
-  else if (sym < 65536) {
-    // 3-byte UTF8 symbol, 4+6+6 bits
-    ch[0] = (sym >> 12) | set3MSbits;
-    ch[1] = ((sym >> 6) & get6LSbits) | set1MSbits;
-    ch[2] = (sym & get6LSbits) | set1MSbits;
-    ch[3] = 0;
-  }
-  
-  else if (sym < 2097152) {
-    // 4-byte UTF8 symbol, 3+6+6+6 bits
-    ch[0] = (sym >> 18) | set4MSbits;
-    ch[1] = ((sym >> 12) & get6LSbits) | set1MSbits;
-    ch[2] = ((sym >> 6) & get6LSbits) | set1MSbits;
-    ch[3] = (sym & get6LSbits) | set1MSbits;
-    ch[4] = 0;
-  }
-  
-  else
-    return NULL;
-
-  return (char*)ch;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  utf8toint                                                      */
-/*                                                                 */
-/*******************************************************************/
-
-unsigned int utf8toint( char **s )
-
-{
-  int bytes_to_come;
-  unsigned int result=0;
-  unsigned char c=(unsigned char)**s;
-
-  if (c >= (unsigned char)set4MSbits) { // 1111xxxx
-    bytes_to_come = 3;
-    result = (result << 3) | (c & get3LSbits);
-  }
-      
-  else if (c >= (unsigned char) set3MSbits) { // 1110xxxx
-    // start of a three-byte symbol
-    bytes_to_come = 2;
-    result = (result << 4) | (c & get4LSbits);
-  }
-      
-  else if (c >= (unsigned char) set2MSbits) { // 1100xxxx
-    // start of a two-byte symbol
-    bytes_to_come = 1;
-    result = (result << 5) | (c & get5LSbits);
-  }
-      
-  else if (c < (unsigned char) set1MSbits) { // 0100xxxx
-    // one-byte symbol
-    bytes_to_come = 0;
-    result = c;
-  }
-
-  else
-    return 0; // error
-
-  while (bytes_to_come > 0) {
-    bytes_to_come--;
-    (*s)++;
-    c = (unsigned char)**s;
-    if (c < (unsigned char) set2MSbits &&
-	c >= (unsigned char) set1MSbits)    // 1000xxxx
-      {
-	result = (result << 6) | (c & get6LSbits);
-      }
-    else
-      return 0;
-  }
-
-  (*s)++;
-  return result;
-}
-
-
-/*******************************************************************/
-/*                                                                 */
-/*  utf8toint                                                      */
-/*                                                                 */
-/*******************************************************************/
-
-unsigned int utf8toint( char *s )
-
-{
-  unsigned int result = utf8toint( &s );
-  if (*s == 0) // all bytes converted?
-    return result;
-  return 0;
-}

data/test/test_sfst.fst

@@ -1,3 +0,0 @@
-ALPHABET = [a-z]
-$test$ = ({bar}:{foo} | {baz}:{foo})
-$test$

data/test/test_sfst.rb

@@ -1,114 +0,0 @@
-require 'sfst'
-require 'test/unit'
-
-TEST_DIRECTORY = File.expand_path(File.dirname(__FILE__))
-TEST_COMPILED_COMPACT_FILE = File.join(TEST_DIRECTORY, 'test_sfst_compact.a')
-TEST_COMPILED_REGULAR_FILE = File.join(TEST_DIRECTORY, 'test_sfst_regular.a')
-
-class RegularTransducerTestCase < Test::Unit::TestCase
-  def test_analyze_acceptance
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal true, fst.accepted_analysis?('foo')
-    assert_equal false, fst.accepted_analysis?('fox')
-  end
-
-  def test_analyze
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal ['bar', 'baz'], fst.analyse('foo').sort
-
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal [], fst.analyse('fox').sort
-  end
-
-  def test_analyze_symbol_sequence
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal [['b', 'a', 'r'], ['b', 'a', 'z']], fst.analyse('foo', :symbol_sequence => true).sort
-
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal [], fst.analyse('fox', :symbol_sequence => true).sort
-  end
-
-  def test_generate_acceptance
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal true, fst.accepted_generating?('bar')
-    assert_equal true, fst.accepted_generating?('baz')
-    assert_equal false, fst.accepted_generating?('bax')
-  end
-
-  def test_generate
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal ['foo'], fst.generate('bar').sort
-
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal ['foo'], fst.generate('baz').sort
-
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    assert_equal [], fst.generate('bax').sort
-  end
-
-  def test_generate_language_default
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    a = []
-    fst.generate_language do |u|
-      a << u.collect { |pair| pair.join(':') }.join
-    end
-    assert_equal ['b:fa:or:o', 'b:fa:oz:o'], a.sort
-  end
-
-  def test_generate_language_both
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    a = []
-    fst.generate_language(:levels => :both) do |u|
-      a << u.collect { |pair| pair.join(':') }.join
-    end
-    assert_equal ['b:fa:or:o', 'b:fa:oz:o'], a.sort
-  end
-
-  def test_generate_language_upper
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    a = []
-    fst.generate_language(:levels => :upper) do |u|
-      a << u.join
-    end
-    assert_equal ['foo'], a.sort
-  end
-
-  def test_generate_language_lower
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    a = []
-    fst.generate_language(:levels => :lower) do |u|
-      a << u.join
-    end
-    assert_equal ['bar', 'baz'], a.sort
-  end
-end
-
-class CompactTransducerTestCase < Test::Unit::TestCase
-  def test_analyze_acceptance
-    fst = SFST::CompactTransducer.new(TEST_COMPILED_COMPACT_FILE)
-    assert_equal true, fst.accepted_analysis?('foo')
-    assert_equal false, fst.accepted_analysis?('fox')
-  end
-
-  def test_analyze
-    fst = SFST::CompactTransducer.new(TEST_COMPILED_COMPACT_FILE)
-    assert_equal ['bar', 'baz'], fst.analyse('foo').sort
-    assert_equal [], fst.analyse('fox').sort
-  end
-end
-
-class StressTestCase < Test::Unit::TestCase
-  def test_repeated_analyses_regular
-    fst = SFST::RegularTransducer.new(TEST_COMPILED_REGULAR_FILE)
-    65536.times do
-      assert_equal ['bar', 'baz'], fst.analyse('foo').sort
-    end
-  end
-
-  def test_repeated_analyses_compact
-    fst = SFST::CompactTransducer.new(TEST_COMPILED_COMPACT_FILE)
-    65536.times do
-      assert_equal ['bar', 'baz'], fst.analyse('foo').sort
-    end
-  end
-end