aurelian-ruby-ahocorasick 0.2.2 → 0.3.1

data/README.textile

@@ -1,28 +1,61 @@
+
+h1. This is a work in progress.
+
 h3. Introduction
 
 This library is a ruby extension, a wrapper around the "Aho-Corasick":http://en.wikipedia.org/wiki/Aho-Corasick_algorithm implementation in C, found in "Strmat":http://www.cs.ucdavis.edu/~gusfield/strmat.html package.
 
 The source code (ac.c and ac.h) was "adapted" from Strmat. In fact, I've changed only 3-4 lines of code from the original implementation so it will feat my needs: search needed to return the current position in the searched string.
 
-h3. What's the idea?
+h3. Okay, so what's the idea?
+
+Having a dictionary of known sentences (note: not *words*!), this kick ass algorithm can find individual patterns in an incoming stream of data. Kinda Fast.
+
+The algorithm has 2 stages: one where an internal tree in being build from the given dictionary leaving the search to the second step.
+
+h3. Okay, so where can I use this?
 
-Having a dictionary of known sentences, how can I find individual patterns in an incoming stream of data? Fast.
+Well, you can do some crazy things with it, like, you can lookup for DNA patterns or maybe analyze network sequences (read: strange and maybe proprietary network protocols), or domestic stuff like building contextual links on your blog posts to enrich your users experience.
 
-h1. TBD
+h3. Okay, so how can I install it?
+
+If you don't have github surces, type:
 
 <pre>
-[aurelian@stalingrad ext]$ time ./dict.rb 
-110196
-711
+gem sources -a http://gems.github.com
+</pre>
+
+then,
 
-real    0m0.538s
-user    0m0.435s
-sys     0m0.036s
+<pre>
+gem install aurelian-ruby-ahocorasick
 </pre>
 
-h3. Additional Reading / Implementations
+h5. Note
+
+It's known to work - compile - install on Ubuntu 8.04 and Mac OS 10.4.*. It should work out of the box if you have gcc around.
 
-Other suffix - tree 
+h3. Okay, so how do I use it?
+
+<pre>
+  require 'ahocorasick'
+
+  keyword_tree= AhoCorasick::KeywordTree.new # creates a new tree
+  keyword_tree.add_string( "foo-- Z@!bar" ) # add's a keyword to the tree
+  keyword_tree.add_string( "cervantes" )    # even more
+
+  results= keyword_tree.find_all( "1011000129 foo-- Z@!bar761 ! 001211 6xU" ).each do | result |
+    result[:value]     # => "foo-- Z@!bar"
+    result[:starts_at] # => 11
+    result[:ends_at]   # => 23
+    result[:id]        # => 1
+  end
+
+</pre>
+
+h3. Additional Reading 
+
+Other suffix - tree implementations:
 
  * "Strmat":http://www.cs.ucdavis.edu/~gusfield/strmat.html
  * Pytst / Ruby-Pytst
@@ -30,7 +63,8 @@ Other suffix - tree
  * "Keyword Prospector":http://latimes.rubyforge.org/keyword_prospector/rdoc/
  * "libstree":http://www.cl.cam.ac.uk/~cpk25/libstree/
 
---
+h3. License
+
+(c) 2008 - Aurelian Oancea, < oancea at gmail dot com > 
 
-(c) 2008 - Aurelian Oancea, < aurelian at locknet . ro > 
 released under MIT-LICENCE

data/examples/dict.rb

@@ -1,13 +1,24 @@
 #!/usr/bin/env ruby
 
+require 'time'
+
 require File.dirname(__FILE__) + '/../ext/ahocorasick'
 
+t= Time.now
+
 k= AhoCorasick::KeywordTree.from_file(File.dirname(__FILE__) + "/../spec/data/en.words")
 
-query = File.read( File.dirname(__FILE__) + "/../spec/data/news.txt")
+t1= Time.now
+
+puts "%d words added in %s seconds" % [k.size, (t1-t)]
+
+query = File.read( File.dirname(__FILE__) + "/../spec/data/news.txt" )
 
 results= k.search query
 
+puts "took %s seconds to find %d results in a streem with %d charachters" % [(Time.now-t1), results.size, query.size]
+
+exit
 results.each do | r |
   puts query[r[:starts_at]].chr + ".." + query[r[:ends_at]-1].chr + " => " + r[:value]
 end

data/ext/ruby-ahocorasick.c

@@ -1,15 +1,22 @@
 
 //
-// (c) 2008, Aurelian Oancea < aurelian at locknet . ro >
+// (c) 2008, Aurelian Oancea < oancea at gmail dot com >
 //
 // Released under MIT-LICENSE
 //
 
 //
 // TODO: new methods?
+//
 //  * kwt[id] = word
 //  * kwt.from_file (class instance method)
 //
+//  * kwt.find_each ("str") {|r| .. }
+//  * kwt.find_first("str")
+//  * kwt.find_all  ("str")
+//
+// TODO: rename search to find_all
+//
 
 #include <ruby.h>
 #include "ac.h"
@@ -196,13 +203,18 @@ rb_kwt_size(VALUE self)
  *
  *    kwt.add_string("foo1$21^ 98N3 ba>Z")
  *    kwt << "bar" # using the alias
- *
+ * 
  * ==== Note: you can also specify the id, a number between 1 and k
  *
- *    kwt.add_string "bar", 123
+ *    kwt.add_string "bar", 123 # => 123
  *
  * This id should be unique in the context of the current tree.
  *
+ * Returns the id of the inserted object.
+ *
+ *    kwt.add_string("test", 18) # => 18
+ *    kwt.add_string("baz") # => 19
+ *
  */ 
 static VALUE
 rb_kwt_add_string(int argc, VALUE *argv, VALUE self)
@@ -232,13 +244,16 @@ rb_kwt_add_string(int argc, VALUE *argv, VALUE self)
     id= NUM2INT(v_id);
   }
   
+  // printf("[internal]==> %d\n", id);
+  
   if( ac_add_string(kwt_data->tree, string, strlen(string), id) == 0 ) {
     rb_raise(rb_eRuntimeError, "Failed to add `%s\", duplicate id `%d\"?", string, id);
   }
 
   kwt_data->last_id= id + 1;
   kwt_data->dictionary_size++;
-  return self;
+  // printf("[internal]==> %d\n", id);
+  return id;
 }
 
 /*
@@ -316,7 +331,10 @@ void Init_ahocorasick() {
   rb_define_method(rb_cKeywordTree, "size", rb_kwt_size, 0);
   rb_define_method(rb_cKeywordTree, "make", rb_kwt_make, 0);
   rb_define_method(rb_cKeywordTree, "add_string", rb_kwt_add_string, -1);
+
   rb_define_method(rb_cKeywordTree, "search", rb_kwt_search, -1);
+  rb_define_alias(rb_cKeywordTree, "find_all", "search");
+
   rb_define_alias(rb_cKeywordTree, "<<", "add_string");
   rb_define_singleton_method(rb_cKeywordTree, "from_file", rb_kwt_new_from_file, -1);
 

data/spec/ahocorasick_spec.rb

@@ -23,28 +23,25 @@ describe KeywordTree do
     after(:each) do
       @kwt= nil
     end
-    it "should return an array" do
+    it "find_all should return an array" do
       @kwt << "foo"
-      @kwt.search("bar").class.should == Array
+      @kwt.find_all("foo").class.should == Array
     end
 
     it "the array should contain hashes" do
-      @kwt << "bar" << "foo"
-      @kwt.search("foo")[0].class.should == Hash
+      @kwt << "bar"
+      @kwt << "foo"
+      @kwt.find_all("foo")[0].class.should == Hash
     end
     
-    # XXX: this is subject of ...talks. no yield at this point
-    # it "should return nil if block_given?" do
-    #   @kwt.search("foo"){|r| r[:id]}.should == nil
-    # end
-
     it "should return empty array if no results" do
-      @kwt.search("baba").should == []
+      @kwt.find_all("1a4a").should == []
     end
 
     it "each hash should have the required symbols values" do
-      @kwt << "bar" << "foo"
-      @kwt.search("foo").each do | r |
+      @kwt << "bar"
+      @kwt << "foo"
+      @kwt.find_all("foo").each do | r |
         r[:id].class.should == Fixnum
         r[:starts_at].class.should == Fixnum
         r[:ends_at].class.should == Fixnum
@@ -57,7 +54,7 @@ describe KeywordTree do
       #        |  |
       @kwt << "data"
       q= "data moved"
-      @kwt.search(q).each do | result |
+      @kwt.find_all(q).each do | result |
         result[:starts_at].should == 0
         result[:ends_at].should   == 4
       end
@@ -86,6 +83,14 @@ describe KeywordTree do
         (r[:ends_at]-r[:starts_at]).should == r[:value].size
       end
     end
+    
+    it "even more unicode" do
+      @kwt << "șșt"
+      #                    0124789
+      result= @kwt.search("mușștar").first
+      result[:starts_at].should == 2
+      result[:ends_at].should == result[:starts_at] + "șșt".size
+    end
 
     it "checks for result length" do
       @kwt << "foo"
@@ -101,7 +106,6 @@ describe KeywordTree do
   describe "Context Match vs. Exact Word Match" do
 
     before(:each) do 
-      # data, base, database
       @kwt= KeywordTree.from_file File.dirname(__FILE__) + "/data/dict0.txt"
     end
 
@@ -118,6 +122,7 @@ describe KeywordTree do
       kwt << "bar"
       kwt.size.should == 2
     end
+
     it "should add 2 strings with id" do
       kwt= KeywordTree.new
       kwt.add_string "foo", 1
@@ -146,6 +151,13 @@ describe KeywordTree do
       kwt.size.should == 3
     end
 
+    it "should return the id" do
+      kwt= KeywordTree.new
+      kwt.add_string("foo").should == 1
+      kwt.add_string("bar", 2008).should == 2008
+      kwt.add_string("kwt").should == 2009
+    end
+
     it "should add strings from file and manually" do
       kwt= KeywordTree.from_file File.dirname(__FILE__) + "/data/dict0.txt"
       kwt << "foo"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: aurelian-ruby-ahocorasick
 version: !ruby/object:Gem::Version 
-  version: 0.2.2
+  version: 0.3.1
 platform: ruby
 authors: 
 - Aurelian Oancea