ruletagger 0.0.1

data/ext/word_tagger/porter_stemmer.h

@@ -0,0 +1,19 @@
+#ifndef PORTER_STEMMER_H
+#define PORTER_STEMMER_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct stemmer;
+
+extern struct stemmer * porter_stemmer_new(void);
+extern void porter_stemmer_free(struct stemmer * z);
+
+extern int porter_stem(struct stemmer * z, const char * b, int k);
+
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif

data/ext/word_tagger/rtagger.cc

@@ -0,0 +1,83 @@
+/** 
+ * Copyright (c) 2008 Todd A. Fisher
+ * see LICENSE
+ */
+#include "ruby.h"
+#include "tagger.h"
+
+#define DEBUG
+#ifdef DEBUG
+#define TRACE()  fprintf(stderr, "> %s:%d:%s\n", __FILE__, __LINE__, __FUNCTION__)
+#else
+#define TRACE() 
+#endif
+
+/* ruby 1.9 compat */
+#ifndef RSTRING_PTR
+#define RSTRING_PTR(str) RSTRING(str)->ptr
+#endif
+
+#ifndef RSTRING_LEN
+#define RSTRING_LEN(str) RSTRING(str)->len
+#endif
+
+static VALUE rb_Tagger;
+static VALUE rb_NWordTagger;
+
+VALUE Tagger_execute( VALUE self, VALUE text )
+{
+  NWordTagger *tagger;
+  Data_Get_Struct( self, NWordTagger, tagger );
+  std::vector<std::string> tags = tagger->execute( RSTRING_PTR(text) );
+  VALUE results = rb_ary_new2(tags.size());
+  for( size_t i = 0; i < tags.size(); ++i ){
+    rb_ary_push( results, rb_str_new( tags[i].c_str(), tags[i].length() ) );
+  }
+  return results;
+}
+VALUE Tagger_set_words( VALUE self, VALUE words )
+{
+  NWordTagger *tagger;
+  Data_Get_Struct( self, NWordTagger, tagger );
+  tagger->setNWords( NUM2INT(words) );
+  return Qnil;
+}
+VALUE Tagger_load_tags( VALUE self, VALUE tagarr )
+{
+  NWordTagger *tagger;
+  Data_Get_Struct( self, NWordTagger, tagger );
+  std::set<std::string> tags;
+  int len = RARRAY(tagarr)->len;
+  for( int i = 0; i < len; ++i ){
+    std::string tag = RSTRING_PTR( rb_ary_entry( tagarr, i ) );
+    tags.insert(tag);
+  }
+  tagger->loadTags(tags);
+  return Qnil;
+}
+
+static void Tagger_free( NWordTagger *tagger )
+{
+  delete tagger;
+}
+
+VALUE Tagger_alloc(VALUE klass)
+{
+  VALUE object;
+  NWordTagger *tagger = new NWordTagger();
+  object = Data_Wrap_Struct( klass, NULL, Tagger_free, tagger );
+
+  return object;
+}
+
+extern "C" void Init_rtagger()
+{
+  rb_Tagger = rb_define_module( "Tagger" );
+  rb_NWordTagger = rb_define_class_under( rb_Tagger, "NWordTagger", rb_cObject );
+
+  rb_define_alloc_func( rb_NWordTagger, Tagger_alloc );
+
+  rb_define_method( rb_NWordTagger, "load_tags", (VALUE (*)(...))Tagger_load_tags, 1 );
+  rb_define_method( rb_NWordTagger, "execute", (VALUE (*)(...))Tagger_execute, 1 );
+  rb_define_method( rb_NWordTagger, "set_words", (VALUE (*)(...))Tagger_set_words, 1 );
+}

data/ext/word_tagger/tagger.cc

@@ -0,0 +1,153 @@
+#include <ctype.h>
+#include "tagger.h"
+#include <set>
+#include <algorithm>
+#include <sstream>
+#include <iterator>
+#include <string>
+#include <vector>
+#include "porter_stemmer.h"
+
+struct WordComparitor
+{
+  bool operator()(const std::pair<std::string,int> &s1, const std::pair<std::string,int> &s2) const
+  {
+    return s1.second < s2.second;
+  }
+};
+
+
+// from http://www.thescripts.com/forum/thread167600.html
+// split words by ' '
+static std::vector<std::string> word_split(const std::string& s)
+{
+  std::string words = s;
+  // convert all non alpha characters to spaces
+  for( size_t i = 0; i < words.length(); ++i ) { 
+    if( !isalpha( words[i] ) ) {
+      words[i] = ' '; // convert to space
+    }
+  }
+
+  std::istringstream is(words);
+  return std::vector<std::string>(std::istream_iterator<std::string>(is), std::istream_iterator<std::string>());
+}
+
+NWordTagger::NWordTagger() 
+ : nwords(2), stemmer(porter_stemmer_new()){
+}
+NWordTagger::~NWordTagger(){
+  porter_stemmer_free(stemmer);
+}
+void NWordTagger::loadTags( const std::set<std::string> &tags )
+{
+  for( std::set<std::string>::iterator i = tags.begin(); i != tags.end(); ++i ){
+    std::string stemmed, word = std::string(*i);
+    std::vector<std::string> words = word_split( *i );
+    //printf( "word: %s\n", word.c_str() );
+
+    if( words.size() > 1 ){
+      for( size_t j = 0; j < words.size(); ++j ){
+        stemmed += this->stemWord(words[j]) + " ";
+      }
+      stemmed = stemmed.substr(0,stemmed.length()-1);
+      this->tags[stemmed] = word;
+      //printf( "word: %s -> %s\n", word.c_str(), stemmed.c_str() );
+    }
+    else{
+      stemmed = this->stemWord(*i);
+      //printf( "word: %s -> %s\n", word.c_str(), stemmed.c_str() );
+      this->tags[stemmed] = word;
+    }
+
+  }
+}
+std::string NWordTagger::stemWord( const std::string &word )const
+{
+  std::string stemmed;
+  char *transition_buffer = strdup( word.c_str() );
+  stemmed = word.substr(0,porter_stem(this->stemmer, transition_buffer, word.length()-1 )+1);
+  free( transition_buffer );
+  return stemmed;
+}
+  
+std::vector<std::string> NWordTagger::execute( const char *text, short max )const
+{
+  int max_count = 0;
+  std::vector<std::string> words = word_split( text );
+  std::map<std::string, int> matched_tags; // stores tags and frequency
+  std::string match_word;
+  std::map<std::string,std::string>::const_iterator matched;
+
+  // loop over the words stemming each word
+  for( size_t i = 0; i < words.size(); ++i ) {
+
+    // get the stemmed word at position i
+    match_word = this->stemWord(words[i]);
+
+    // now scan ahead nwords positions searching our tags table for matches
+    for( short j = 1; (j <= this->nwords) && ((i+j) < words.size()); ++j ) {
+      matched = this->tags.find( match_word );
+      if( matched != this->tags.end() ){
+        //printf( "word: %d:(%s->%s)\n", i, match_word.c_str(), matched->second.c_str() );
+        std::map<std::string, int>::iterator mloc = matched_tags.find( matched->second );
+        if( mloc == matched_tags.end() ) {
+          matched_tags[matched->second] = 1; // count 1
+        }
+        else {
+          mloc->second++;
+          if( max_count < mloc->second ) { max_count = mloc->second; }
+        }
+      }
+      // stem each word and compare against our tag bank
+      //printf( "window: %ld:%lu\n", i,(i+j) );
+      match_word += " " + this->stemWord(words[i+j]);
+    }
+
+    matched = this->tags.find( match_word );
+    if( matched != this->tags.end() ) {
+      //printf( "word: %ld:(%s->%s)\n", i, words[i].c_str(), match_word.c_str() );
+      std::map<std::string, int>::iterator mloc = matched_tags.find( matched->second );
+      if( mloc == matched_tags.end() ) {
+        matched_tags[matched->second] = 1; // count 1
+      }
+      else {
+        mloc->second++;
+        if( max_count < mloc->second ) { max_count = mloc->second; }
+      }
+    }
+  }
+
+  // now that we have all the matched tags reduce to max using the tag frequency as a reduction measure
+  std::vector< std::pair<std::string,int> > sorted_tags;
+
+  //printf( "max frequency: %d\n", max_count );
+  for( std::map<std::string, int>::iterator mloc = matched_tags.begin(); mloc != matched_tags.end(); ++mloc ){
+    //printf( "word: %s, frequency: %d\n", mloc->first.c_str(), mloc->second ); 
+    sorted_tags.push_back(*mloc);
+  }
+
+  // sort the tags in frequency order
+  std::sort( sorted_tags.begin(), sorted_tags.end(), WordComparitor() );
+
+  std::vector< std::string > reduced_tags;
+
+  std::vector< std::pair<std::string, int> >::iterator mloc;
+  do {
+    for(mloc = sorted_tags.begin(); mloc != sorted_tags.end(); ++mloc ) {
+      std::pair< std::string, int > word_freq = *mloc;
+     // printf( "word: %s, frequency: %d\n", word_freq.first.c_str(), word_freq.second );
+      //printf( "word: %s, frequency: %d\n", mloc->first.c_str(), mloc->second ); 
+      if( word_freq.second < max_count ) { 
+        sorted_tags.erase( mloc );
+        break;
+      }
+    }
+  } while( sorted_tags.size() > (size_t)max && mloc != sorted_tags.end() );
+
+  for( size_t i = 0; i < sorted_tags.size(); ++i ) {
+    reduced_tags.push_back( sorted_tags[i].first );
+  }
+
+  return reduced_tags;
+}

data/ext/word_tagger/tagger.h

@@ -0,0 +1,27 @@
+#ifndef NWORD_TAGGER_H
+#define NWORD_TAGGER_H
+#include <set>
+#include <map>
+#include <string>
+#include <vector>
+
+struct NWordTagger {
+  NWordTagger();
+  ~NWordTagger();
+
+  void loadTags( const std::set<std::string> &tags );
+
+  short getNWords()const{ return nwords; }
+  void setNWords( short words ){ nwords = words; }
+
+  std::vector<std::string> execute( const char *text, short max = 10 )const;
+private:
+  short nwords;
+  struct stemmer *stemmer;
+  std::map<std::string,std::string> tags;
+  std::vector<std::string> words;
+
+  std::string stemWord( const std::string &word )const;
+};
+
+#endif

data/ext/word_tagger/tagger.rb

@@ -0,0 +1,8 @@
+module Tagger
+  require 'rtagger'
+  class SimpleTagger < Tagger::NWordTagger
+    def execute( text )
+      super( text.gsub(/[^\w]/,' ') )
+    end
+  end
+end

data/ext/word_tagger/test/Makefile

@@ -0,0 +1,22 @@
+CFLAGS=-g -Wall #-fprofile-arcs -ftest-coverage
+
+all: porter_stemmer.o tagger.o test.o
+	g++ $(CFLAGS) -o testit $^
+	time ./testit doc.txt
+#	gcov -b tagger.cc
+#	valgrind --leak-check=full ./testit
+test.o: test.cc tagger.h
+	g++ $(CFLAGS) test.cc -c
+tagger.o: tagger.cc tagger.h porter_stemmer.h
+	g++ $(CFLAGS) tagger.cc -c
+porter_stemmer.o: porter_stemmer.c porter_stemmer.h
+	gcc $(CFLAGS) porter_stemmer.c -c
+
+setup:
+	ln -s ../tagger.h
+	ln -s ../tagger.cc
+	ln -s ../porter_stemmer.h
+	ln -s ../porter_stemmer.c
+
+clean:
+	rm -f tagger.o test.o testit

data/ext/word_tagger/test/doc.txt

@@ -0,0 +1,87 @@
+Allergies are diseases of the immune system that cause an overreaction to
+substances called "allergens." Allergies are grouped by the kind of trigger,
+time of year or where symptoms appear on the body: indoor andoutdoor allergies
+(also called "hay fever," "seasonal," "perennial" or "nasal" allergies), food
+and drug allergies, latex allergies, insect allergies, skin allergies and eye
+allergies. People who have allergies can live healthy and active lives.
+
+movieMultimedia Allergy Library
+GlossaryGlossary of Allergy Terms
+
+What are Allergies  |   What Causes Allergies  |   Diagnosis  |   Treatment  |
+Prevention  |  
+
+What Causes Allergies
+
+The substances that cause allergic disease in people are known as allergens.
+"Antigens," or protein particles like pollen, food or dander enter our bodies
+through a variety of ways. If the antigen causes an allergic reaction, that
+particle is considered an "allergen" – and antigen that triggers an allergic
+reaction. These allergens can get into our body in several ways:
+
+    * Inhaled into the nose and the lungs. Examples are airborne pollens of
+    * certain trees, grasses and weeds; house dust that include dust mite
+    * particles, mold spores, cat and dog dander and latex dust.
+        * Ingested by mouth. Frequent culprits include shrimp, peanuts and
+        * other nuts.
+            * Injected. Such as medications delivered by needle like
+            * penicillin or other injectable drugs, and venom from insect
+            * stings and bites.
+                * Absorbed through the skin. Plants such as poison ivy, sumac
+                * and oak and latex are examples.
+
+                What Makes Some Pollen Cause Allergies, and Not Others?
+
+                Plant pollens that are carried by the wind cause most
+                allergies of the nose, eyes and lungs. These plants (including
+                certain weeds, trees and grasses) are natural pollutants
+                produced at various times of the year when their small,
+                inconspicuous flowers discharge literally billions of pollen
+                particles.
+
+                Because the particles can be carried significant distances, it
+                is important for you not only to understand local
+                environmental conditions, but also conditions over the broader
+                area of the state or region in which you live. Unlike the
+                wind-pollinated plants, conspicuous wild flowers or flowers
+                used in most residential gardens are pollinated by bees,
+                wasps, and other insects and therefore are not widely capable
+                of producing allergic disease.
+
+                What is the Role of Heredity in Allergy?
+
+                Like baldness, height and eye color, the capacity to become
+                allergic is an inherited characteristic. Yet, although you may
+                be born with the genetic capability to become allergic, you
+                are not automatically allergic to specific allergens. Several
+                factors must be present for allergic sensitivity to be
+                developed:
+
+                    * The specific genes acquired from parents.
+                        * The exposure to one or more allergens to which you
+                        * have a genetically programmed response.
+                            * The degree and length of exposure.
+
+                            A baby born with the tendency to become allergic
+                            to cow's milk, for example, may show allergic
+                            symptoms several months after birth. A genetic
+                            capability to become allergic to cat dander may
+                            take three to four years of cat exposure before
+                            the person shows symptoms. These people may also
+                            become allergic to other environmental substances
+                            with age.
+
+                            On the other hand, poison ivy allergy (contact
+                            dermatitis) is an example of an allergy in which
+                            hereditary background does not play a part. The
+                            person with poison ivy allergy first has to be
+                            exposed to the oil from the plant. This usually
+                            occurs during youth, when a rash does not always
+                            appear. However, the first exposure may sensitize
+                            or cause the person to become allergic and, when
+                            subsequent exposure takes place, a contact
+                            dermatitis rash appears and can be quite severe.
+                            Many plants are capable of producing this type of
+                            rash. Substances other than plants, such as dyes,
+                            metals, and chemicals in deodorants and cosmetics,
+                            can also cause a similar dermatitis.

data/ext/word_tagger/test/test.cc

@@ -0,0 +1,107 @@
+#include <stdio.h>
+#include <ctype.h>
+#include <sys/stat.h>
+#include <assert.h>
+#include "tagger.h"
+
+struct TestBase {
+  TestBase();
+  void overflow();
+  void small();
+  void large_file();
+
+  std::set<std::string> tags;
+  NWordTagger tagger;
+};
+
+
+TestBase::TestBase()
+{
+  tags.insert("fitness");
+  tags.insert("delightful");
+  tags.insert("dreaming");
+  tags.insert("dreaming of their world");
+  tags.insert("names");
+  tags.insert("places");
+  tags.insert("diabetes");
+  tags.insert("sugars");
+  tags.insert("allergy");
+  tags.insert("dermatitis");
+  
+  tagger.setNWords( 4 );
+
+  tagger.loadTags( tags );
+}
+
+void TestBase::overflow()
+{
+  // input passed to the filter should be processed to downcase, and remove all punctionation
+  std::string words( "hello fitness fitness fitness party dreaming dreaming of their world how are you all doing today so many times I ve seen or heard a delightful story or tales" );
+  std::vector<std::string> matched_tags = tagger.execute( words.c_str(), 2 ); 
+  
+  printf( "%ld tags\n", (long int)matched_tags.size() );
+  for( size_t i = 0; i < matched_tags.size(); ++i ){
+    printf( "tagged: %s\n", matched_tags[i].c_str() ); 
+  }
+
+  assert( matched_tags.size() == 2 );
+  assert( matched_tags[0] == "dreaming" );
+  assert( matched_tags[1] == "fitness" );
+}
+
+void TestBase::small()
+{
+  // input passed to the filter should be processed to downcase, and remove all punctionation
+  std::string words( "nothing to see here" );
+  std::vector<std::string> matched_tags = tagger.execute( words.c_str(), 2 ); 
+
+  assert( matched_tags.size() == 0 );
+}
+
+void TestBase::large_file()
+{
+  // input passed to the filter should be processed to downcase, and remove all punctionation
+  std::string words;
+  FILE *in = fopen("doc.txt","r");
+  struct stat s;
+  char *buffer = NULL;
+  fstat( fileno(in), &s );
+  buffer = (char*)malloc(sizeof(char)*(s.st_size+1));
+  memset(buffer,'\0',s.st_size+1);
+  fread( buffer, sizeof(char), s.st_size, in );
+  words = buffer;
+  free(buffer);
+  fclose(in);
+
+  std::vector<std::string> matched_tags = tagger.execute( words.c_str(), 10 ); 
+  
+  printf( "%ld tags\n", (long int)matched_tags.size() );
+  for( size_t i = 0; i < matched_tags.size(); ++i ){
+    printf( "tagged: %s\n", matched_tags[i].c_str() ); 
+  }
+
+  assert( matched_tags.size() == 2 );
+  assert( matched_tags[0] == "allergy" );
+  assert( matched_tags[1] == "dermatitis" );
+}
+
+static void test_run()
+{
+  TestBase test;
+
+  for( int i = 0; i < 10; ++i ) {
+    test.overflow();
+    test.small();
+    test.large_file();
+  }
+
+}
+
+int main()
+{
+  // running multiple iterations to test for memory leaks
+  for( int i = 0; i < 2; ++i ) {
+    test_run();
+  }
+  return 0;
+}

data/ext/word_tagger/test.rb

@@ -0,0 +1,31 @@
+if $0 == __FILE__
+require 'test/unit'
+require 'tagger'
+
+class NWordTest < Test::Unit::TestCase
+
+  def setup
+    if !defined?($tagger)
+      $tagger = Tagger::SimpleTagger.new
+      $tagger.load_tags( File.read('../../tags.txt').split("\n") )
+      $tagger.set_words( 4 );
+    end
+  end
+
+  def test_basic
+    timer = Time.now
+    text = "This is a sa'mple doc[]ument lets see how cancer ngrams 4 works out for this interesting text!"
+    tags = $tagger.execute( text )
+    assert_equal ['cancer','work'], tags
+    puts "Duration: #{Time.now - timer} sec"
+  end
+
+  def test_ngram_size3
+    timer = Time.now
+    text = "This body of text contains something like ventricular septal defect"
+    tags = $tagger.execute( text )
+    assert_equal ['ventricular septal defect'], tags
+    puts "Duration: #{Time.now - timer} sec"
+  end
+end
+end