aurelian-ruby-ahocorasick 0.2.2

data/ext/ac.h

@@ -0,0 +1,36 @@
+
+#ifndef _AC_H_
+#define _AC_H_
+
+typedef struct actreenode {
+  char ch;
+  int matchid;
+  struct actreenode *outlink, *faillink;
+  struct actreenode *children, *sibling;
+} ACTREE_NODE, *AC_TREE;
+
+typedef struct {
+  AC_TREE tree;
+  int ispreprocessed, errorflag;
+
+  int Psize;
+  int *Plengths;
+
+  char *T;
+  int N, c, initflag, endflag;
+  AC_TREE w, output;
+#ifdef STATS
+  int prep_new_edges, prep_old_edges, prep_fail_compares;
+  int num_compares, num_failures, edges_traversed, outlinks_traversed;
+#endif
+} AC_STRUCT;
+
+AC_STRUCT *ac_alloc(void);
+int ac_add_string(AC_STRUCT *node, char *P, int M, int id);
+// int ac_del_string(AC_STRUCT *node, char *P, int M, int id);
+int ac_prep(AC_STRUCT *node);
+void ac_search_init(AC_STRUCT *node, char *T, int N);
+char *ac_search(AC_STRUCT *node, int *length_out, int *id_out, int *ends_at);
+void ac_free(AC_STRUCT *node);
+
+#endif

data/ext/extconf.rb

@@ -0,0 +1,4 @@
+require "mkmf"
+
+create_makefile("ahocorasick")
+

data/ext/ruby-ahocorasick.c

@@ -0,0 +1,329 @@
+
+//
+// (c) 2008, Aurelian Oancea < aurelian at locknet . ro >
+//
+// Released under MIT-LICENSE
+//
+
+//
+// TODO: new methods?
+//  * kwt[id] = word
+//  * kwt.from_file (class instance method)
+//
+
+#include <ruby.h>
+#include "ac.h"
+
+static VALUE sym_id, sym_value, sym_ends_at, sym_starts_at;
+
+VALUE rb_mAhoCorasick;
+VALUE rb_cKeywordTree;
+
+#define KeywordTree(obj, kwt_data) {\
+  Data_Get_Struct(obj, struct kwt_struct_data, kwt_data);\
+}
+
+struct kwt_struct_data {
+  AC_STRUCT * tree;
+  int last_id;
+  int dictionary_size;
+  int is_frozen;
+};
+
+/*
+ * call-seq: initialize
+ *
+ * Creates a new KeywordTree
+ *
+ *   require 'ahocorasick'
+ *   kwt = Ahocorasick::KeywordTree.new
+ *
+ */
+static VALUE 
+rb_kwt_init(VALUE self)
+{ 
+  AC_STRUCT * tree;
+  struct kwt_struct_data *kwt_data;
+
+  kwt_data = ALLOC(struct kwt_struct_data);
+  tree     = ac_alloc();
+  DATA_PTR(self) = kwt_data;
+  kwt_data->tree            = tree;
+  kwt_data->last_id         = 1;
+  kwt_data->dictionary_size = 0;
+  kwt_data->is_frozen       = 0;
+  return self;
+}
+
+/*
+ * Document-method: make
+ * call-seq: make
+ *
+ * It freezes the current KeywordTree. After this point, the tree will not accept any new entries.
+ *
+ * ==== Note: This method is called internally by search
+ *
+ *   require 'ahocorasick'
+ *   
+ *   kwt = Ahocorasick::KeywordTree.new
+ *
+ *   kwt.add_string("one")
+ *   kwt.add_string("two")
+ *   kwt.make()
+ */
+static VALUE 
+rb_kwt_make(VALUE self)
+{ 
+  struct kwt_struct_data *kwt_data;
+  KeywordTree(self, kwt_data);
+
+  ac_prep( kwt_data->tree );
+  kwt_data->is_frozen = 1;
+  return self;
+}
+
+/*
+ * Document-method: search
+ * call-seq: search
+ *
+ * Search the current tree.
+ *
+ * It returns an array on hashes, e.g.
+ *
+ *   [ { :id => int, :value => int, :starts_at => int, :ends_at => int}, { ... } ]
+ * 
+ * Returns an empty array when the search didn't return any result.
+ *
+ *   # assuming a valid KeywordTree kwt object:
+ *   kwt.add_string("one")
+ *   kwt.add_string("two")
+ *
+ *   kwt.search( "moved two times already" ).each  do | result |
+ *     result[:id] # => 2
+ *     result[:ends_at] # => 9
+ *     result[:starts_at] # => 6
+ *     result[:value] # => two
+ *   end # => 1
+ *
+ */
+static VALUE
+rb_kwt_search(int argc, VALUE *argv, VALUE self)
+{
+  char * result;        // itermediate result
+  char * remain;        // returned by ac_search, the remaing text to search
+  int lgt, id, ends_at; // filled in by ac_search, the id, length and ends_at position
+  int starts_at;
+  VALUE v_result;  // one result, as hash
+  VALUE v_results; // all the results, an array
+  VALUE v_search;  // search string, function argument
+  struct kwt_struct_data *kwt_data;
+  
+  // one mandatory argument.
+  rb_scan_args(argc, argv, "1", &v_search);
+  // it should be string.
+  Check_Type(v_search, T_STRING);
+  // get the structure
+  KeywordTree(self, kwt_data);
+  // freeze the tree, if not already
+  if(kwt_data->is_frozen == 0) {
+    ac_prep( kwt_data->tree );
+    kwt_data->is_frozen = 1;
+  }
+  // prepare the return value
+  // v_results= rb_block_given_p()? Qnil : rb_ary_new();
+  v_results= rb_ary_new();
+  // fail quickly and return the empty array
+  if(kwt_data->dictionary_size == 0) 
+    return v_results;
+  // prepare the search
+  ac_search_init(kwt_data->tree, RSTRING( v_search )->ptr, RSTRING( v_search )->len);
+  // loop trought the results
+  while((remain= ac_search(kwt_data->tree, &lgt, &id, &ends_at)) != NULL) {
+    // this is an individual result as a hash
+    v_result= rb_hash_new();
+    rb_hash_aset( v_result, sym_id, INT2FIX(id) );
+    rb_hash_aset( v_result, sym_starts_at, INT2FIX( ends_at - lgt - 1 ) );
+    rb_hash_aset( v_result, sym_ends_at, INT2FIX( ends_at - 1 ) );
+    result = (char*) malloc (sizeof(char)*lgt);
+    sprintf( result, "%.*s", lgt, remain);
+    rb_hash_aset( v_result, sym_value, rb_str_new(result, lgt) );
+
+    // yield this hash or, add it to the results
+    // if(rb_block_given_p())
+    //   rb_yield(v_result);
+    // else
+    rb_ary_push( v_results, v_result );
+    free(result);
+  }
+  
+  // TODO: maybe the Tree can be re-opened to add new items to dictionary
+
+  // return the results or nil if none
+  // if( v_results != Qnil && RARRAY(v_results)->len > 0 ) {
+  return v_results;
+  // } else {
+  //   return Qnil;
+  // }
+}
+
+
+/*
+ * Document-method: size
+ * call-seq: size
+ *
+ * Returns the size of this KeywordTree
+ *
+ *    kwt.add_string("foo")
+ *    kwt.add_string("bar")
+ *    kwt.size #=> 2
+ *
+ */ 
+static VALUE 
+rb_kwt_size(VALUE self)
+{ 
+  struct kwt_struct_data *kwt_data;
+  KeywordTree(self, kwt_data);
+
+  return INT2FIX(kwt_data->dictionary_size);
+}
+
+
+/*
+ * Document-method: add_string
+ * call-seq: add_string
+ *
+ * Adds a sequence to this KeywordTree.
+ *
+ *    kwt.add_string("foo1$21^ 98N3 ba>Z")
+ *    kwt << "bar" # using the alias
+ *
+ * ==== Note: you can also specify the id, a number between 1 and k
+ *
+ *    kwt.add_string "bar", 123
+ *
+ * This id should be unique in the context of the current tree.
+ *
+ */ 
+static VALUE
+rb_kwt_add_string(int argc, VALUE *argv, VALUE self)
+{ 
+  VALUE v_string, v_id;
+  struct kwt_struct_data *kwt_data;
+  char * string;
+  int id;
+
+  rb_scan_args(argc, argv, "11", &v_string, &v_id);
+ 
+  Check_Type(v_string, T_STRING);
+  string= RSTRING(v_string)->ptr;
+
+  KeywordTree(self, kwt_data);
+
+  if(kwt_data->is_frozen == 1)
+    rb_raise(rb_eRuntimeError, "Cannot add `%s\" into a frozen tree.", string);
+
+  if(v_id == Qnil) {
+    id = kwt_data->last_id;
+  } else if(TYPE(v_id) != T_FIXNUM) {
+    rb_raise(rb_eRuntimeError, "Please use a number from 1 to K as id, or leave nil to auto-generate one. `%s\" given.", RSTRING(v_id)->ptr);
+  } else if(NUM2INT(v_id) <= 0) {
+    rb_raise(rb_eRuntimeError, "Please use a number from 1 to K as id, or leave nil to auto-generate one. `%d\" given.", NUM2INT(v_id));
+  } else {
+    id= NUM2INT(v_id);
+  }
+  
+  if( ac_add_string(kwt_data->tree, string, strlen(string), id) == 0 ) {
+    rb_raise(rb_eRuntimeError, "Failed to add `%s\", duplicate id `%d\"?", string, id);
+  }
+
+  kwt_data->last_id= id + 1;
+  kwt_data->dictionary_size++;
+  return self;
+}
+
+/*
+ * call-seq: from_file
+ *
+ * Creates a new KeywordTree and loads the dictionary from a file
+ * 
+ *    % cat dict0.txt
+ *    foo
+ *    bar
+ *    base
+ *     
+ *    k= AhoCorasick::KeywordTree.from_file "dict0.txt"
+ *    k.search("basement").size # => 1
+ *
+ */
+static VALUE
+rb_kwt_new_from_file(int argc, VALUE *argv, VALUE klass)
+{ 
+
+  // TODO: 
+  //  * use rb_kwt_add_string
+  //  * use rb_io* to handle the file
+
+  struct kwt_struct_data *kwt_data;
+  char word[1024];
+  int id;
+  VALUE self;
+  VALUE f_string;
+  FILE *dictionary;
+
+  rb_scan_args(argc, argv, "10", &f_string);
+ 
+  id = 0;
+  SafeStringValue( f_string );
+  self= rb_class_new_instance( 0, NULL, klass );
+  KeywordTree( self, kwt_data );
+
+  dictionary = fopen( RSTRING( f_string )->ptr, "r" );
+  if(dictionary == NULL) {
+    rb_raise(rb_eRuntimeError, "Cannot open `%s\". No such file?", RSTRING(f_string)->ptr);
+  }
+
+  while(fgets(word, 1024, dictionary) != NULL) {
+    ac_add_string(kwt_data->tree, word, strlen(word)-1, id++);
+    kwt_data->dictionary_size++;
+  }
+  kwt_data->last_id= id+1;
+  fclose(dictionary);
+  return self;
+}
+
+static void
+rb_kwt_struct_free(struct kwt_struct_data * kwt_data)
+{
+  ac_free(kwt_data->tree);
+}
+
+static VALUE
+rb_kwt_struct_alloc(VALUE klass)
+{
+  return Data_Wrap_Struct(klass, 0, rb_kwt_struct_free, 0);
+}
+
+/*
+ * Blump.
+ */
+void Init_ahocorasick() {
+  rb_mAhoCorasick = rb_define_module("AhoCorasick");
+  rb_cKeywordTree = rb_define_class_under(rb_mAhoCorasick, "KeywordTree", rb_cObject);
+  
+  rb_define_alloc_func(rb_cKeywordTree, rb_kwt_struct_alloc);
+
+  rb_define_method(rb_cKeywordTree, "initialize", rb_kwt_init, 0);
+  rb_define_method(rb_cKeywordTree, "size", rb_kwt_size, 0);
+  rb_define_method(rb_cKeywordTree, "make", rb_kwt_make, 0);
+  rb_define_method(rb_cKeywordTree, "add_string", rb_kwt_add_string, -1);
+  rb_define_method(rb_cKeywordTree, "search", rb_kwt_search, -1);
+  rb_define_alias(rb_cKeywordTree, "<<", "add_string");
+  rb_define_singleton_method(rb_cKeywordTree, "from_file", rb_kwt_new_from_file, -1);
+
+  sym_id= ID2SYM(rb_intern("id"));
+  sym_value= ID2SYM(rb_intern("value"));
+  sym_ends_at= ID2SYM( rb_intern("ends_at") );
+  sym_starts_at= ID2SYM( rb_intern("starts_at") );
+
+}
+

data/spec/ahocorasick_spec.rb

@@ -0,0 +1,183 @@
+require 'ext/ahocorasick'
+
+include AhoCorasick
+
+describe KeywordTree do
+
+  describe "How to create a new KeywordTree" do
+    it "should create a new KeywordTree" do
+      KeywordTree.new.class.should == KeywordTree
+    end
+    it "should create a new KeywordTree" do
+      KeywordTree.from_file( File.dirname(__FILE__) + "/data/dict0.txt").class.should == KeywordTree
+    end
+  end
+
+  describe "How to search" do
+
+    before(:each) do
+      @kwt= KeywordTree.new
+    end
+
+    # XXX: is this usefull?
+    after(:each) do
+      @kwt= nil
+    end
+    it "should return an array" do
+      @kwt << "foo"
+      @kwt.search("bar").class.should == Array
+    end
+
+    it "the array should contain hashes" do
+      @kwt << "bar" << "foo"
+      @kwt.search("foo")[0].class.should == Hash
+    end
+    
+    # XXX: this is subject of ...talks. no yield at this point
+    # it "should return nil if block_given?" do
+    #   @kwt.search("foo"){|r| r[:id]}.should == nil
+    # end
+
+    it "should return empty array if no results" do
+      @kwt.search("baba").should == []
+    end
+
+    it "each hash should have the required symbols values" do
+      @kwt << "bar" << "foo"
+      @kwt.search("foo").each do | r |
+        r[:id].class.should == Fixnum
+        r[:starts_at].class.should == Fixnum
+        r[:ends_at].class.should == Fixnum
+        r[:value].should == "foo"
+      end
+    end
+
+    it "should match position" do
+      #        0123
+      #        |  |
+      @kwt << "data"
+      q= "data moved"
+      @kwt.search(q).each do | result |
+        result[:starts_at].should == 0
+        result[:ends_at].should   == 4
+      end
+    end
+
+    it "should match position with unicode" do
+      #        012345689
+      #        |       |
+      @kwt << "bucurești"
+      #   01234567890123456789023
+      #                 |       |
+      q= "data moved to bucurești"
+      @kwt.search(q).each do | result |
+        result[:starts_at].should == 14
+        result[:ends_at].should   == 24
+      end
+    end
+
+    it "more unicode" do
+      @kwt << "expected"
+      #    012345678901234578901234567890
+      q = "moved to bucurești as expected"
+      @kwt.search(q).each do | r |
+        r[:starts_at].should == 23
+        r[:ends_at].should   == q.size
+        (r[:ends_at]-r[:starts_at]).should == r[:value].size
+      end
+    end
+
+    it "checks for result length" do
+      @kwt << "foo"
+      result= @kwt.search("foo").first
+      #          4                 0
+      (result[:ends_at]-result[:starts_at]).should == result[:value].size
+      "foo"[result[:ends_at]].should == nil
+      "foo"[result[:ends_at]-1].chr.should == "o"
+    end
+
+  end
+
+  describe "Context Match vs. Exact Word Match" do
+
+    before(:each) do 
+      # data, base, database
+      @kwt= KeywordTree.from_file File.dirname(__FILE__) + "/data/dict0.txt"
+    end
+
+    it "should match on context" do
+      @kwt.search("I've moved the data to a new database").size.should == 4
+    end
+
+  end
+
+  describe "How to add strings" do
+    it "should add 2 strings" do
+      kwt= KeywordTree.new
+      kwt.add_string "foo"
+      kwt << "bar"
+      kwt.size.should == 2
+    end
+    it "should add 2 strings with id" do
+      kwt= KeywordTree.new
+      kwt.add_string "foo", 1
+      kwt.add_string "bar", 2
+      kwt.size.should == 2
+    end
+
+    it "should rise an error when adding same id twice" do
+      kwt= KeywordTree.new
+      kwt.add_string "foo", 1
+      lambda{kwt.add_string("bar", 1)}.should raise_error(RuntimeError)
+    end
+
+    it "should raise an error when not using id's > 0" do
+      kwt= KeywordTree.new
+      lambda{kwt.add_string("bar", -1)}.should raise_error(RuntimeError)
+      lambda{kwt.add_string("bar", "a")}.should raise_error(RuntimeError)
+      lambda{kwt.add_string("bar", 0)}.should raise_error(RuntimeError)
+    end
+
+    it "should work to add a random id" do
+      kwt= KeywordTree.new
+      kwt << "baz"
+      kwt.add_string "foo", 1990
+      kwt << "bar"
+      kwt.size.should == 3
+    end
+
+    it "should add strings from file and manually" do
+      kwt= KeywordTree.from_file File.dirname(__FILE__) + "/data/dict0.txt"
+      kwt << "foo"
+      kwt.size.should == File.readlines( File.dirname(__FILE__) + "/data/dict0.txt" ).size + 1
+    end
+
+    it "should raise an error when adding new strings after the tree is frozen" do
+      kwt= KeywordTree.from_file File.dirname(__FILE__) + "/data/dict0.txt"
+      kwt.make
+      lambda{kwt << "foo"}.should raise_error(RuntimeError)
+    end
+
+  end
+
+  describe "Benchmarks. Loading from a file" do
+
+    it "should be fast to load a bunch of english words" do
+      start= Time.now
+      k= KeywordTree.from_file File.dirname(__FILE__) + "/data/en.words"
+      puts "\n%d words loaded in %s seconds" % [k.size, (Time.now - start)]
+      (Time.now-start).should < 0.2
+    end
+
+    it "should be fast to find" do
+      start= Time.now
+      k= KeywordTree.from_file File.dirname(__FILE__) + "/data/en.words"
+      load_time= Time.now
+      results= k.search( File.read( File.dirname(__FILE__) + "/data/melville-moby_dick.txt" ) )
+      puts "\n%d words re-loaded in %s seconds.\nGot %d results in %s seconds" % [k.size, (load_time - start), results.size, (Time.now-load_time)]
+      (Time.now-load_time).should < 1.2
+    end
+  end
+
+
+end