chipper 0.4.2

data/README.rdoc

@@ -0,0 +1,51 @@
+= Twitter Text Extraction
+
+A fast screen name, hashtag, url extraction and tokenizer for tweets.
+
+
+== API
+
+    Chipper
+      #users     => [Array]
+      #hashtags  => [Array]
+      #urls      => [Array]
+      #tokens    => [Array]
+
+      #skip_users
+      #skip_hashtags
+      #skip_tokens
+      #skip_token_pattern
+
+== Usage
+
+    require 'chipper'
+
+    Chipper.skip_users(%w(youtube msn))
+    Chipper.skip_hashtags(%w(abc24 cnn))
+    Chipper.skip_tokens(%w(story tv why that get from your))
+    Chipper.skip_token_pattern '^vimeo$'
+
+    tweet = "hi @youtube, could we get #cnn videos so i can #watch it on my @apple tv http://t.co/HM7XoimM"
+    Chipper.users(tweet)    #=> ["@apple"]
+    Chipper.hashtags(tweet) #=> ["#watch"]
+    Chipper.urls(tweet)     #=> ["http://t.co/HM7XoimM"]
+
+    # n-gram tokenizer, returns a list of tokens partitioned by stop words, punctuation, urls and hashtags.
+    Chipper.tokens(tweet)   #=> [["could"], ["get"], ["videos"], ["can"]]
+
+    # single method that does all of the above and returns a hash.
+    Chipper.entities(tweet)
+
+== Gotchas
+
+* skips tokens shorter than 3 characters
+* only handles t.co urls
+
+== Updating version
+
+* update ext/src/version.h
+* rake gemspec
+
+== License
+
+{Creative Commons Attribution - CC BY}[http://creativecommons.org/licenses/by/3.0]

data/ext/extconf.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'mkmf'
+require 'fileutils'
+
+puts 'compiling libstemmer_c'
+logs = IO.popen(File.join(File.dirname(__FILE__), 'stemmer.rb') + ' 2>&1') {|io| io.read }
+
+File.open('mkmf.log', 'w') {|fh| fh.write(logs)}
+
+raise "unable to compile libstemmer_c" if $?.exitstatus != 0
+
+EXTCONFIG        = Object.const_get(defined?(RbConfig) ? :RbConfig : :Config)::CONFIG
+EXTCONFIG['CC']  = 'g++'
+EXTCONFIG['CPP'] = 'g++'
+
+version  = %x{g++ --version | head -n1}.strip.sub(%r{^.* ([\d\.]+)$}, '\1')
+maj, min = version.split('.').values_at(0, 1).map(&:to_i)
+
+$CFLAGS  = '-pthread -Wno-sign-compare '
+$CFLAGS += (maj >= 4 && min > 5 || maj > 4)  ? '-Ofast ' : '-O3 '
+$CFLAGS += '-I/usr/include -I/opt/local/include -I/usr/local/include'
+
+snowball  = File.join(File.dirname(__FILE__), 'libstemmer_c')
+$CFLAGS  += " -I#{File.expand_path(File.join(snowball, 'include'))} "
+$libs     = 'libstemmer_c/libstemmer.o'
+
+def apt_install_hint pkg
+  "sudo apt-get install #{pkg}"
+end
+
+def library_installed? name, hint
+  if find_library(name, 'main', *%w(/usr/lib /usr/local/lib /opt/lib /opt/local/lib /sw/lib))
+    true
+  else
+    $stderr.puts <<-ERROR
+
+      Unable to find required library: #{name}.
+      On debian systems, it can be installed as,
+
+      #{hint}
+
+    ERROR
+    false
+  end
+end
+
+# ugly mkmf hack: manually assign source and object directories.
+$srcs = Dir["{re2/*.cc,src/*.cc}"]
+$objs = $srcs.map {|name| File.join(File.dirname(name), File.basename(name, ".cc") + ".o")}
+
+class File
+  def self.basename name
+    name
+  end
+end
+
+create_makefile 'chipper'

data/ext/libstemmer_c/Makefile

@@ -0,0 +1,10 @@
+include mkinc.mak
+CC=g++
+CFLAGS=-Iinclude -fPIC -O3
+all: libstemmer.o stemwords
+libstemmer.o: $(snowball_sources:.c=.o)
+	$(AR) -cru $@ $^
+stemwords: examples/stemwords.o libstemmer.o
+	$(CC) -o $@ $^
+clean:
+	rm -f stemwords *.o src_c/*.o runtime/*.o libstemmer/*.o

data/ext/libstemmer_c/examples/stemwords.c

@@ -0,0 +1,209 @@
+/* This is a simple program which uses libstemmer to provide a command
+ * line interface for stemming using any of the algorithms provided.
+ */
+
+#include <stdio.h>
+#include <stdlib.h> /* for malloc, free */
+#include <string.h> /* for memmove */
+#include <ctype.h>  /* for isupper, tolower */
+
+#include "libstemmer.h"
+
+const char * progname;
+static int pretty = 1;
+
+static void
+stem_file(struct sb_stemmer * stemmer, FILE * f_in, FILE * f_out)
+{
+#define INC 10
+    int lim = INC;
+    sb_symbol * b = (sb_symbol *) malloc(lim * sizeof(sb_symbol));
+
+    while(1) {
+        int ch = getc(f_in);
+        if (ch == EOF) {
+            free(b); return;
+        }
+        {
+            int i = 0;
+	    int inlen = 0;
+            while(1) {
+                if (ch == '\n' || ch == EOF) break;
+                if (i == lim) {
+                    sb_symbol * newb;
+		    newb = (sb_symbol *)
+			    realloc(b, (lim + INC) * sizeof(sb_symbol));
+		    if (newb == 0) goto error;
+		    b = newb;
+                    lim = lim + INC;
+                }
+		/* Update count of utf-8 characters. */
+		if (ch < 0x80 || ch > 0xBF) inlen += 1;
+                /* force lower case: */
+                if (isupper(ch)) ch = tolower(ch);
+
+                b[i] = ch;
+		i++;
+                ch = getc(f_in);
+            }
+
+	    {
+		const sb_symbol * stemmed = sb_stemmer_stem(stemmer, b, i);
+                if (stemmed == NULL)
+                {
+                    fprintf(stderr, "Out of memory");
+                    exit(1);
+                }
+                else
+		{
+		    if (pretty == 1) {
+			fwrite(b, i, 1, f_out);
+			fputs(" -> ", f_out);
+		    } else if (pretty == 2) {
+			fwrite(b, i, 1, f_out);
+			if (sb_stemmer_length(stemmer) > 0) {
+			    int j;
+			    if (inlen < 30) {
+				for (j = 30 - inlen; j > 0; j--)
+				    fputs(" ", f_out);
+			    } else {
+				fputs("\n", f_out);
+				for (j = 30; j > 0; j--)
+				    fputs(" ", f_out);
+			    }
+			}
+		    }
+
+		    fputs((char *)stemmed, f_out);
+		    putc('\n', f_out);
+		}
+            }
+        }
+    }
+error:
+    if (b != 0) free(b);
+    return;
+}
+
+/** Display the command line syntax, and then exit.
+ *  @param n The value to exit with.
+ */
+static void
+usage(int n)
+{
+    printf("usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-p[2]] [-h]\n"
+	  "\n"
+	  "The input file consists of a list of words to be stemmed, one per\n"
+	  "line. Words should be in lower case, but (for English) A-Z letters\n"
+	  "are mapped to their a-z equivalents anyway. If omitted, stdin is\n"
+	  "used.\n"
+	  "\n"
+	  "If -c is given, the argument is the character encoding of the input\n"
+          "and output files.  If it is omitted, the UTF-8 encoding is used.\n"
+	  "\n"
+	  "If -p is given the output file consists of each word of the input\n"
+	  "file followed by \"->\" followed by its stemmed equivalent.\n"
+	  "If -p2 is given the output file is a two column layout containing\n"
+	  "the input words in the first column and the stemmed eqivalents in\n"
+	  "the second column.\n"
+	  "Otherwise, the output file consists of the stemmed words, one per\n"
+	  "line.\n"
+	  "\n"
+	  "-h displays this help\n",
+	  progname);
+    exit(n);
+}
+
+int
+main(int argc, char * argv[])
+{
+    char * in = 0;
+    char * out = 0;
+    FILE * f_in;
+    FILE * f_out;
+    struct sb_stemmer * stemmer;
+
+    char * language = "english";
+    char * charenc = NULL;
+
+    char * s;
+    int i = 1;
+    pretty = 0;
+
+    progname = argv[0];
+
+    while(i < argc) {
+	s = argv[i++];
+	if (s[0] == '-') {
+	    if (strcmp(s, "-o") == 0) {
+		if (i >= argc) {
+		    fprintf(stderr, "%s requires an argument\n", s);
+		    exit(1);
+		}
+		out = argv[i++];
+	    } else if (strcmp(s, "-i") == 0) {
+		if (i >= argc) {
+		    fprintf(stderr, "%s requires an argument\n", s);
+		    exit(1);
+		}
+		in = argv[i++];
+	    } else if (strcmp(s, "-l") == 0) {
+		if (i >= argc) {
+		    fprintf(stderr, "%s requires an argument\n", s);
+		    exit(1);
+		}
+		language = argv[i++];
+	    } else if (strcmp(s, "-c") == 0) {
+		if (i >= argc) {
+		    fprintf(stderr, "%s requires an argument\n", s);
+		    exit(1);
+		}
+		charenc = argv[i++];
+	    } else if (strcmp(s, "-p2") == 0) {
+		pretty = 2;
+	    } else if (strcmp(s, "-p") == 0) {
+		pretty = 1;
+	    } else if (strcmp(s, "-h") == 0) {
+		usage(0);
+	    } else {
+		fprintf(stderr, "option %s unknown\n", s);
+		usage(1);
+	    }
+	} else {
+	    fprintf(stderr, "unexpected parameter %s\n", s);
+	    usage(1);
+	}
+    }
+
+    /* prepare the files */
+    f_in = (in == 0) ? stdin : fopen(in, "r");
+    if (f_in == 0) {
+	fprintf(stderr, "file %s not found\n", in);
+	exit(1);
+    }
+    f_out = (out == 0) ? stdout : fopen(out, "w");
+    if (f_out == 0) {
+	fprintf(stderr, "file %s cannot be opened\n", out);
+	exit(1);
+    }
+
+    /* do the stemming process: */
+    stemmer = sb_stemmer_new(language, charenc);
+    if (stemmer == 0) {
+        if (charenc == NULL) {
+            fprintf(stderr, "language `%s' not available for stemming\n", language);
+            exit(1);
+        } else {
+            fprintf(stderr, "language `%s' not available for stemming in encoding `%s'\n", language, charenc);
+            exit(1);
+        }
+    }
+    stem_file(stemmer, f_in, f_out);
+    sb_stemmer_delete(stemmer);
+
+    if (in != 0) (void) fclose(f_in);
+    if (out != 0) (void) fclose(f_out);
+
+    return 0;
+}
+

data/ext/libstemmer_c/include/libstemmer.h

@@ -0,0 +1,79 @@
+
+/* Make header file work when included from C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sb_stemmer;
+typedef unsigned char sb_symbol;
+
+/* FIXME - should be able to get a version number for each stemming
+ * algorithm (which will be incremented each time the output changes). */
+
+/** Returns an array of the names of the available stemming algorithms.
+ *  Note that these are the canonical names - aliases (ie, other names for
+ *  the same algorithm) will not be included in the list.
+ *  The list is terminated with a null pointer.
+ *
+ *  The list must not be modified in any way.
+ */
+const char ** sb_stemmer_list(void);
+
+/** Create a new stemmer object, using the specified algorithm, for the
+ *  specified character encoding.
+ *
+ *  All algorithms will usually be available in UTF-8, but may also be
+ *  available in other character encodings.
+ *
+ *  @param algorithm The algorithm name.  This is either the english
+ *  name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
+ *  language.  Note that case is significant in this parameter - the
+ *  value should be supplied in lower case.
+ *
+ *  @param charenc The character encoding.  NULL may be passed as
+ *  this value, in which case UTF-8 encoding will be assumed. Otherwise,
+ *  the argument may be one of "UTF_8", "ISO_8859_1" (ie, Latin 1),
+ *  "CP850" (ie, MS-DOS Latin 1) or "KOI8_R" (Russian).  Note that
+ *  case is significant in this parameter.
+ *
+ *  @return NULL if the specified algorithm is not recognised, or the
+ *  algorithm is not available for the requested encoding.  Otherwise,
+ *  returns a pointer to a newly created stemmer for the requested algorithm.
+ *  The returned pointer must be deleted by calling sb_stemmer_delete().
+ *
+ *  @note NULL will also be returned if an out of memory error occurs.
+ */
+struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
+
+/** Delete a stemmer object.
+ *
+ *  This frees all resources allocated for the stemmer.  After calling
+ *  this function, the supplied stemmer may no longer be used in any way.
+ *
+ *  It is safe to pass a null pointer to this function - this will have
+ *  no effect.
+ */
+void                sb_stemmer_delete(struct sb_stemmer * stemmer);
+
+/** Stem a word.
+ *
+ *  The return value is owned by the stemmer - it must not be freed or
+ *  modified, and it will become invalid when the stemmer is called again,
+ *  or if the stemmer is freed.
+ *
+ *  The length of the return value can be obtained using sb_stemmer_length().
+ *
+ *  If an out-of-memory error occurs, this will return NULL.
+ */
+const sb_symbol *   sb_stemmer_stem(struct sb_stemmer * stemmer,
+				    const sb_symbol * word, int size);
+
+/** Get the length of the result of the last stemmed word.
+ *  This should not be called before sb_stemmer_stem() has been called.
+ */
+int                 sb_stemmer_length(struct sb_stemmer * stemmer);
+
+#ifdef __cplusplus
+}
+#endif
+

data/ext/libstemmer_c/libstemmer/libstemmer.c

@@ -0,0 +1,95 @@
+
+#include <stdlib.h>
+#include <string.h>
+#include "../include/libstemmer.h"
+#include "../runtime/api.h"
+#include "modules.h"
+
+struct sb_stemmer {
+    struct SN_env * (*create)(void);
+    void (*close)(struct SN_env *);
+    int (*stem)(struct SN_env *);
+
+    struct SN_env * env;
+};
+
+extern const char **
+sb_stemmer_list(void)
+{
+    return algorithm_names;
+}
+
+static stemmer_encoding_t
+sb_getenc(const char * charenc)
+{
+    struct stemmer_encoding * encoding;
+    if (charenc == NULL) return ENC_UTF_8;
+    for (encoding = encodings; encoding->name != 0; encoding++) {
+	if (strcmp(encoding->name, charenc) == 0) break;
+    }
+    if (encoding->name == NULL) return ENC_UNKNOWN;
+    return encoding->enc;
+}
+
+extern struct sb_stemmer *
+sb_stemmer_new(const char * algorithm, const char * charenc)
+{
+    stemmer_encoding_t enc;
+    struct stemmer_modules * module;
+    struct sb_stemmer * stemmer;
+
+    enc = sb_getenc(charenc);
+    if (enc == ENC_UNKNOWN) return NULL;
+
+    for (module = modules; module->name != 0; module++) {
+	if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
+    }
+    if (module->name == NULL) return NULL;
+    
+    stemmer = (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
+    if (stemmer == NULL) return NULL;
+
+    stemmer->create = module->create;
+    stemmer->close = module->close;
+    stemmer->stem = module->stem;
+
+    stemmer->env = stemmer->create();
+    if (stemmer->env == NULL)
+    {
+        sb_stemmer_delete(stemmer);
+        return NULL;
+    }
+
+    return stemmer;
+}
+
+void
+sb_stemmer_delete(struct sb_stemmer * stemmer)
+{
+    if (stemmer == 0) return;
+    if (stemmer->close == 0) return;
+    stemmer->close(stemmer->env);
+    stemmer->close = 0;
+    free(stemmer);
+}
+
+const sb_symbol *
+sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
+{
+    int ret;
+    if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
+    {
+        stemmer->env->l = 0;
+        return NULL;
+    }
+    ret = stemmer->stem(stemmer->env);
+    if (ret < 0) return NULL;
+    stemmer->env->p[stemmer->env->l] = 0;
+    return (const sb_symbol *)(stemmer->env->p);
+}
+
+int
+sb_stemmer_length(struct sb_stemmer * stemmer)
+{
+    return stemmer->env->l;
+}

data/ext/libstemmer_c/libstemmer/libstemmer_utf8.c

@@ -0,0 +1,95 @@
+
+#include <stdlib.h>
+#include <string.h>
+#include "../include/libstemmer.h"
+#include "../runtime/api.h"
+#include "modules_utf8.h"
+
+struct sb_stemmer {
+    struct SN_env * (*create)(void);
+    void (*close)(struct SN_env *);
+    int (*stem)(struct SN_env *);
+
+    struct SN_env * env;
+};
+
+extern const char **
+sb_stemmer_list(void)
+{
+    return algorithm_names;
+}
+
+static stemmer_encoding_t
+sb_getenc(const char * charenc)
+{
+    struct stemmer_encoding * encoding;
+    if (charenc == NULL) return ENC_UTF_8;
+    for (encoding = encodings; encoding->name != 0; encoding++) {
+	if (strcmp(encoding->name, charenc) == 0) break;
+    }
+    if (encoding->name == NULL) return ENC_UNKNOWN;
+    return encoding->enc;
+}
+
+extern struct sb_stemmer *
+sb_stemmer_new(const char * algorithm, const char * charenc)
+{
+    stemmer_encoding_t enc;
+    struct stemmer_modules * module;
+    struct sb_stemmer * stemmer;
+
+    enc = sb_getenc(charenc);
+    if (enc == ENC_UNKNOWN) return NULL;
+
+    for (module = modules; module->name != 0; module++) {
+	if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
+    }
+    if (module->name == NULL) return NULL;
+    
+    stemmer = (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
+    if (stemmer == NULL) return NULL;
+
+    stemmer->create = module->create;
+    stemmer->close = module->close;
+    stemmer->stem = module->stem;
+
+    stemmer->env = stemmer->create();
+    if (stemmer->env == NULL)
+    {
+        sb_stemmer_delete(stemmer);
+        return NULL;
+    }
+
+    return stemmer;
+}
+
+void
+sb_stemmer_delete(struct sb_stemmer * stemmer)
+{
+    if (stemmer == 0) return;
+    if (stemmer->close == 0) return;
+    stemmer->close(stemmer->env);
+    stemmer->close = 0;
+    free(stemmer);
+}
+
+const sb_symbol *
+sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
+{
+    int ret;
+    if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
+    {
+        stemmer->env->l = 0;
+        return NULL;
+    }
+    ret = stemmer->stem(stemmer->env);
+    if (ret < 0) return NULL;
+    stemmer->env->p[stemmer->env->l] = 0;
+    return (const sb_symbol *)(stemmer->env->p);
+}
+
+int
+sb_stemmer_length(struct sb_stemmer * stemmer)
+{
+    return stemmer->env->l;
+}