ruby-stemmer 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2008 Aurelian Oancea
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README

@@ -0,0 +1,79 @@
+$Id: README 17 2008-01-07 16:59:10Z aurelian $
+
+== About
+
+ruby-stemmer, an extension to ruby using stemmer implementation in C.
+
+This package includes libstemmer_c library from http://snowball.tartarus.org/dist/libstemmer_c.tgz
+
+For details about libstemmer_c read libstemmer_c/README or http://snowball.tartarus.org.
+
+author: Aurelian Oancea, aurelian at locknet dot ro
+
+licence: MIT, see MIT-LICENSE for details
+
+== Install
+
+I) Using RubyGems
+
+  $ gem install ruby-stemmer
+
+II) From tarball
+
+Use sudo or run as root if you get Permission Deny issues
+
+Compile libstemmer_c and generate the Makefile:
+
+  $ ruby extconf.rb
+
+Compile the library:
+
+  $ make
+
+Test:
+
+  $ ./test.rb
+
+Install it:
+
+  $ make install
+
+Run the last command as root or with sudo if you get permission deny problems
+
+== Usage
+
+see test.rb
+
+== API
+
+module Lingua
+
+  class Steemer
+
+    # creates a new Steemer,
+    # defaults: language => en, encoding => UTF_8
+    # pass :language or :encoding to change them 
+    def initialize
+    end
+    
+    # stemms the word
+    def stem(word)
+    end
+    
+    # gets the length of the last stemmed word
+    # same as: 
+    # word = Lingua::Steemer.new.stem("installation") # ==> install (string)
+    # word.length # ==> 6 (int)
+    def length
+    end
+
+  end
+
+end
+
+== Todo
+
+# RDoc
+# Add (Array of Hashes) Lingua::Stemmer.list to list available languages/encodings
+# Windows?
+

data/Rakefile

@@ -0,0 +1,52 @@
+#
+# $Id: Rakefile 17 2008-01-07 16:59:10Z aurelian $
+#
+
+require 'rubygems'
+require 'rake'
+require 'rake/gempackagetask'
+
+rm_rf 'Makefile'
+
+PKG_FILES = FileList[
+  'extconf.rb',
+  'ruby-stemmer.c',
+  'test.rb',
+  '[A-Z]*',
+  'libstemmer_c/**/*'
+]
+
+PKG_FILES.exclude('*.o')
+PKG_FILES.exclude('**/*.o')
+PKG_FILES.exclude('stemwords')
+PKG_FILES.exclude('*.bundle')
+PKG_FILES.exclude('*.a')
+PKG_FILES.exclude('*.so')
+
+spec = Gem::Specification.new do | s |
+  s.name = 'ruby-stemmer'
+  s.version = '0.0.1'
+  s.summary = "Stemmer implementation to ruby using libstemmer_c."
+  s.description = <<-EOF
+    Stemmer implementation to ruby using libstemmer_c.
+  EOF
+
+  s.files = PKG_FILES.to_a
+  s.extensions << "extconf.rb"
+  s.has_rdoc = false
+
+  s.author = "Aurelian Oancea"
+  s.email = "aurelian@locknet.ro"
+  s.homepage = "http://nrr.rubyforge.org"
+  s.rubyforge_project = "nrr"
+end
+
+pt = Rake::GemPackageTask.new(spec) do |p|
+  p.need_tar = true
+  p.need_zip = true
+end
+
+task :default do
+  puts "Ok"
+end
+

data/extconf.rb

@@ -0,0 +1,14 @@
+#
+# $Id: extconf.rb 15 2008-01-05 11:56:57Z aurelian $
+#
+
+require "mkmf"
+
+system "cd libstemmer_c; make; cd #{File.dirname(__FILE__)};"
+
+$CFLAGS += " -I#{File.dirname(__FILE__)}/libstemmer_c/include "
+$libs   += " -L#{File.dirname(__FILE__)}/libstemmer_c -llibstemmer.o "
+
+have_header("libstemmer.h")
+create_makefile("stemmer")
+

data/libstemmer_c/MANIFEST

@@ -0,0 +1,72 @@
+README
+src_c/stem_ISO_8859_1_danish.c
+src_c/stem_ISO_8859_1_danish.h
+src_c/stem_ISO_8859_1_dutch.c
+src_c/stem_ISO_8859_1_dutch.h
+src_c/stem_ISO_8859_1_english.c
+src_c/stem_ISO_8859_1_english.h
+src_c/stem_ISO_8859_1_finnish.c
+src_c/stem_ISO_8859_1_finnish.h
+src_c/stem_ISO_8859_1_french.c
+src_c/stem_ISO_8859_1_french.h
+src_c/stem_ISO_8859_1_german.c
+src_c/stem_ISO_8859_1_german.h
+src_c/stem_ISO_8859_1_hungarian.c
+src_c/stem_ISO_8859_1_hungarian.h
+src_c/stem_ISO_8859_1_italian.c
+src_c/stem_ISO_8859_1_italian.h
+src_c/stem_ISO_8859_1_norwegian.c
+src_c/stem_ISO_8859_1_norwegian.h
+src_c/stem_ISO_8859_1_porter.c
+src_c/stem_ISO_8859_1_porter.h
+src_c/stem_ISO_8859_1_portuguese.c
+src_c/stem_ISO_8859_1_portuguese.h
+src_c/stem_ISO_8859_1_spanish.c
+src_c/stem_ISO_8859_1_spanish.h
+src_c/stem_ISO_8859_1_swedish.c
+src_c/stem_ISO_8859_1_swedish.h
+src_c/stem_ISO_8859_2_romanian.c
+src_c/stem_ISO_8859_2_romanian.h
+src_c/stem_KOI8_R_russian.c
+src_c/stem_KOI8_R_russian.h
+src_c/stem_UTF_8_danish.c
+src_c/stem_UTF_8_danish.h
+src_c/stem_UTF_8_dutch.c
+src_c/stem_UTF_8_dutch.h
+src_c/stem_UTF_8_english.c
+src_c/stem_UTF_8_english.h
+src_c/stem_UTF_8_finnish.c
+src_c/stem_UTF_8_finnish.h
+src_c/stem_UTF_8_french.c
+src_c/stem_UTF_8_french.h
+src_c/stem_UTF_8_german.c
+src_c/stem_UTF_8_german.h
+src_c/stem_UTF_8_hungarian.c
+src_c/stem_UTF_8_hungarian.h
+src_c/stem_UTF_8_italian.c
+src_c/stem_UTF_8_italian.h
+src_c/stem_UTF_8_norwegian.c
+src_c/stem_UTF_8_norwegian.h
+src_c/stem_UTF_8_porter.c
+src_c/stem_UTF_8_porter.h
+src_c/stem_UTF_8_portuguese.c
+src_c/stem_UTF_8_portuguese.h
+src_c/stem_UTF_8_romanian.c
+src_c/stem_UTF_8_romanian.h
+src_c/stem_UTF_8_russian.c
+src_c/stem_UTF_8_russian.h
+src_c/stem_UTF_8_spanish.c
+src_c/stem_UTF_8_spanish.h
+src_c/stem_UTF_8_swedish.c
+src_c/stem_UTF_8_swedish.h
+src_c/stem_UTF_8_turkish.c
+src_c/stem_UTF_8_turkish.h
+runtime/api.c
+runtime/api.h
+runtime/header.h
+runtime/utilities.c
+libstemmer/libstemmer.c
+libstemmer/libstemmer_utf8.c
+libstemmer/modules.h
+libstemmer/modules_utf8.h
+include/libstemmer.h

data/libstemmer_c/Makefile

@@ -0,0 +1,9 @@
+include mkinc.mak
+CFLAGS=-Iinclude
+all: libstemmer.o stemwords
+libstemmer.o: $(snowball_sources:.c=.o)
+	$(AR) -cru $@ $^
+stemwords: examples/stemwords.o libstemmer.o
+	$(CC) -o $@ $^
+clean:
+	rm -f stemwords *.o src_c/*.o runtime/*.o libstemmer/*.o

data/libstemmer_c/README

@@ -0,0 +1,125 @@
+libstemmer_c
+============
+
+This document pertains to the C version of the libstemmer distribution,
+available for download from:
+
+http://snowball.tartarus.org/dist/libstemmer_c.tgz
+
+
+Compiling the library
+=====================
+
+A simple makefile is provided for Unix style systems.  On such systems, it
+should be possible simply to run "make", and the file "libstemmer.o"
+and the example program "stemwords" will be generated.
+
+If this doesn't work on your system, you need to write your own build
+system (or call the compiler directly).  The files to compile are
+all contained in the "libstemmer", "runtime" and "src_c" directories,
+and the public header file is contained in the "include" directory.
+
+The library comes in two flavours; UTF-8 only, and UTF-8 plus other character
+sets.  To use the utf-8 only flavour, compile "libstemmer_utf8.c" instead of
+"libstemmer.c".
+
+For convenience "mkinc.mak" is a makefile fragment listing the source files and
+header files used to compile the standard version of the library.
+"mkinc_utf8.mak" is a comparable makefile fragment listing just the source
+files for the UTF-8 only version of the library.
+
+
+Using the library
+=================
+
+The library provides a simple C API.  Essentially, a new stemmer can
+be obtained by using "sb_stemmer_new".  "sb_stemmer_stem" is then
+used to stem a word, "sb_stemmer_length" returns the stemmed
+length of the last word processed, and "sb_stemmer_delete" is
+used to delete a stemmer.
+
+Creating a stemmer is a relatively expensive operation - the expected
+usage pattern is that a new stemmer is created when needed, used
+to stem many words, and deleted after some time.
+
+Stemmers are re-entrant, but not threadsafe.  In other words, if
+you wish to access the same stemmer object from multiple threads,
+you must ensure that all access is protected by a mutex or similar
+device.
+
+libstemmer does not currently incorporate any mechanism for caching the results
+of stemming operations.  Such caching can greatly increase the performance of a
+stemmer under certain situations, so suitable patches will be considered for
+inclusion.
+
+The standard libstemmer sources contain an algorithm for each of the supported
+languages.  The algorithm may be selected using the english name of the
+language, or using the 2 or 3 letter ISO 639 language codes.  In addition,
+the traditional "Porter" stemming algorithm for english is included for
+backwards compatibility purposes, but we recommend use of the "English"
+stemmer in preference for new projects.
+
+(Some minor algorithms which are included only as curiosities in the snowball
+website, such as the Lovins stemmer and the Kraaij Pohlmann stemmer, are not
+included in the standard libstemmer sources.  These are not really supported by
+the snowball project, but it would be possible to compile a modified libstemmer
+library containing these if desired.)
+
+
+The stemwords example
+=====================
+
+The stemwords example program allows you to run any of the stemmers
+compiled into the libstemmer library on a sample vocabulary.  For
+details on how to use it, run it with the "-h" command line option.
+
+
+Using the library in a larger system
+====================================
+
+If you are incorporating the library into the build system of a larger
+program, I recommend copying the unpacked tarball without modification into
+a subdirectory of the sources of your program.  Future versions of the
+library are intended to keep the same structure, so this will keep the
+work required to move to a new version of the library to a minimum.
+
+As an additional convenience, the list of source and header files used
+in the library is detailed in mkinc.mak - a file which is in a suitable
+format for inclusion by a Makefile.  By including this file in your build
+system, you can link the snowball system into your program with a few
+extra rules.
+
+Using the library in a system using GNU autotools
+=================================================
+
+The libstemmer_c library can be integrated into a larger system which uses the
+GNU autotool framework (and in particular, automake and autoconf) as follows:
+
+1) Unpack libstemmer_c.tgz in the top level project directory so that there is
+   a libstemmer_c subdirectory of the top level directory of the project.
+
+2) Add a file "Makefile.am" to the unpacked libstemmer_c folder, containing:
+   
+noinst_LTLIBRARIES = libstemmer.la
+include $(srcdir)/mkinc.mak
+noinst_HEADERS = $(snowball_headers)
+libstemmer_la_SOURCES = $(snowball_sources) 
+
+(You may also need to add other lines to this, for example, if you are using
+compiler options which are not compatible with compiling the libstemmer
+library.)
+
+3) Add libstemmer_c to the AC_CONFIG_FILES declaration in the project's
+   configure.ac file.
+
+4) Add to the top level makefile the following lines (or modify existing
+   assignments to these variables appropriately):
+
+AUTOMAKE_OPTIONS = subdir-objects
+AM_CPPFLAGS = -I$(top_srcdir)/libstemmer_c/include
+SUBDIRS=libstemmer_c
+<name>_LIBADD = libstemmer_c/libstemmer.la
+
+(Where <name> is the name of the library or executable which links against
+libstemmer.) 
+

data/libstemmer_c/include/libstemmer.h

@@ -0,0 +1,79 @@
+
+/* Make header file work when included from C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sb_stemmer;
+typedef unsigned char sb_symbol;
+
+/* FIXME - should be able to get a version number for each stemming
+ * algorithm (which will be incremented each time the output changes). */
+
+/** Returns an array of the names of the available stemming algorithms.
+ *  Note that these are the canonical names - aliases (ie, other names for
+ *  the same algorithm) will not be included in the list.
+ *  The list is terminated with a null pointer.
+ *
+ *  The list must not be modified in any way.
+ */
+const char ** sb_stemmer_list(void);
+
+/** Create a new stemmer object, using the specified algorithm, for the
+ *  specified character encoding.
+ *
+ *  All algorithms will usually be available in UTF-8, but may also be
+ *  available in other character encodings.
+ *
+ *  @param algorithm The algorithm name.  This is either the english
+ *  name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
+ *  language.  Note that case is significant in this parameter - the
+ *  value should be supplied in lower case.
+ *
+ *  @param charenc The character encoding.  NULL may be passed as
+ *  this value, in which case UTF-8 encoding will be assumed. Otherwise,
+ *  the argument may be one of "UTF_8", "ISO_8859_1" (ie, Latin 1),
+ *  "CP850" (ie, MS-DOS Latin 1) or "KOI8_R" (Russian).  Note that
+ *  case is significant in this parameter.
+ *
+ *  @return NULL if the specified algorithm is not recognised, or the
+ *  algorithm is not available for the requested encoding.  Otherwise,
+ *  returns a pointer to a newly created stemmer for the requested algorithm.
+ *  The returned pointer must be deleted by calling sb_stemmer_delete().
+ *
+ *  @note NULL will also be returned if an out of memory error occurs.
+ */
+struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
+
+/** Delete a stemmer object.
+ *
+ *  This frees all resources allocated for the stemmer.  After calling
+ *  this function, the supplied stemmer may no longer be used in any way.
+ *
+ *  It is safe to pass a null pointer to this function - this will have
+ *  no effect.
+ */
+void                sb_stemmer_delete(struct sb_stemmer * stemmer);
+
+/** Stem a word.
+ *
+ *  The return value is owned by the stemmer - it must not be freed or
+ *  modified, and it will become invalid when the stemmer is called again,
+ *  or if the stemmer is freed.
+ *
+ *  The length of the return value can be obtained using sb_stemmer_length().
+ *
+ *  If an out-of-memory error occurs, this will return NULL.
+ */
+const sb_symbol *   sb_stemmer_stem(struct sb_stemmer * stemmer,
+				    const sb_symbol * word, int size);
+
+/** Get the length of the result of the last stemmed word.
+ *  This should not be called before sb_stemmer_stem() has been called.
+ */
+int                 sb_stemmer_length(struct sb_stemmer * stemmer);
+
+#ifdef __cplusplus
+}
+#endif
+

data/libstemmer_c/libstemmer/libstemmer.c

@@ -0,0 +1,93 @@
+
+#include <stdlib.h>
+#include <string.h>
+#include "../include/libstemmer.h"
+#include "../runtime/api.h"
+#include "modules.h"
+
+struct sb_stemmer {
+    struct SN_env * (*create)(void);
+    void (*close)(struct SN_env *);
+    int (*stem)(struct SN_env *);
+
+    struct SN_env * env;
+};
+
+extern const char **
+sb_stemmer_list(void)
+{
+    return algorithm_names;
+}
+
+static stemmer_encoding_t
+sb_getenc(const char * charenc)
+{
+    struct stemmer_encoding * encoding;
+    if (charenc == NULL) return ENC_UTF_8;
+    for (encoding = encodings; encoding->name != 0; encoding++) {
+	if (strcmp(encoding->name, charenc) == 0) break;
+    }
+    if (encoding->name == NULL) return ENC_UNKNOWN;
+    return encoding->enc;
+}
+
+extern struct sb_stemmer *
+sb_stemmer_new(const char * algorithm, const char * charenc)
+{
+    stemmer_encoding_t enc;
+    struct stemmer_modules * module;
+    struct sb_stemmer * stemmer =
+	    (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
+    if (stemmer == NULL) return NULL;
+    enc = sb_getenc(charenc);
+    if (enc == ENC_UNKNOWN) return NULL;
+
+    for (module = modules; module->name != 0; module++) {
+	if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
+    }
+    if (module->name == NULL) return NULL;
+    
+    stemmer->create = module->create;
+    stemmer->close = module->close;
+    stemmer->stem = module->stem;
+
+    stemmer->env = stemmer->create();
+    if (stemmer->env == NULL)
+    {
+        sb_stemmer_delete(stemmer);
+        return NULL;
+    }
+
+    return stemmer;
+}
+
+void
+sb_stemmer_delete(struct sb_stemmer * stemmer)
+{
+    if (stemmer == 0) return;
+    if (stemmer->close == 0) return;
+    stemmer->close(stemmer->env);
+    stemmer->close = 0;
+    free(stemmer);
+}
+
+const sb_symbol *
+sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
+{
+    int ret;
+    if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
+    {
+        stemmer->env->l = 0;
+        return NULL;
+    }
+    ret = stemmer->stem(stemmer->env);
+    if (ret < 0) return NULL;
+    stemmer->env->p[stemmer->env->l] = 0;
+    return (const sb_symbol *)(stemmer->env->p);
+}
+
+int
+sb_stemmer_length(struct sb_stemmer * stemmer)
+{
+    return stemmer->env->l;
+}

data/libstemmer_c/libstemmer/libstemmer_utf8.c

@@ -0,0 +1,93 @@
+
+#include <stdlib.h>
+#include <string.h>
+#include "../include/libstemmer.h"
+#include "../runtime/api.h"
+#include "modules_utf8.h"
+
+struct sb_stemmer {
+    struct SN_env * (*create)(void);
+    void (*close)(struct SN_env *);
+    int (*stem)(struct SN_env *);
+
+    struct SN_env * env;
+};
+
+extern const char **
+sb_stemmer_list(void)
+{
+    return algorithm_names;
+}
+
+static stemmer_encoding_t
+sb_getenc(const char * charenc)
+{
+    struct stemmer_encoding * encoding;
+    if (charenc == NULL) return ENC_UTF_8;
+    for (encoding = encodings; encoding->name != 0; encoding++) {
+	if (strcmp(encoding->name, charenc) == 0) break;
+    }
+    if (encoding->name == NULL) return ENC_UNKNOWN;
+    return encoding->enc;
+}
+
+extern struct sb_stemmer *
+sb_stemmer_new(const char * algorithm, const char * charenc)
+{
+    stemmer_encoding_t enc;
+    struct stemmer_modules * module;
+    struct sb_stemmer * stemmer =
+	    (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
+    if (stemmer == NULL) return NULL;
+    enc = sb_getenc(charenc);
+    if (enc == ENC_UNKNOWN) return NULL;
+
+    for (module = modules; module->name != 0; module++) {
+	if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
+    }
+    if (module->name == NULL) return NULL;
+    
+    stemmer->create = module->create;
+    stemmer->close = module->close;
+    stemmer->stem = module->stem;
+
+    stemmer->env = stemmer->create();
+    if (stemmer->env == NULL)
+    {
+        sb_stemmer_delete(stemmer);
+        return NULL;
+    }
+
+    return stemmer;
+}
+
+void
+sb_stemmer_delete(struct sb_stemmer * stemmer)
+{
+    if (stemmer == 0) return;
+    if (stemmer->close == 0) return;
+    stemmer->close(stemmer->env);
+    stemmer->close = 0;
+    free(stemmer);
+}
+
+const sb_symbol *
+sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
+{
+    int ret;
+    if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
+    {
+        stemmer->env->l = 0;
+        return NULL;
+    }
+    ret = stemmer->stem(stemmer->env);
+    if (ret < 0) return NULL;
+    stemmer->env->p[stemmer->env->l] = 0;
+    return (const sb_symbol *)(stemmer->env->p);
+}
+
+int
+sb_stemmer_length(struct sb_stemmer * stemmer)
+{
+    return stemmer->env->l;
+}