ruby-ahocorasick 0.4.5

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2008 Aurelian Oancea
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.textile

@@ -0,0 +1,101 @@
+h1. This is a work in progress.
+
+
+h2. Introduction
+
+This library is a "Ruby":http://ruby-lang.org extension, a wrapper around the "Aho-Corasick":http://en.wikipedia.org/wiki/Aho-Corasick_algorithm implementation in C, found in "Strmat":http://www.cs.ucdavis.edu/~gusfield/strmat.html package.
+
+The source code (ac.c and ac.h) was "adapted" from Strmat. In fact, I've changed only 3-4 lines of code from the original implementation so it will feat my needs: search needed to return the current position in the searched string.
+
+
+h2. Okay, so what's the idea?
+
+Having a dictionary of known sentences (note: not *words*!), this kick ass algorithm can find individual patterns in an incoming stream of data. Kinda Fast.
+
+The algorithm has 2 stages: one where an internal tree in being build from the given dictionary leaving the search to the second step.
+
+
+h2. Okay, so where can I use this?
+
+Well, you can do some crazy things with it, like, you can lookup for DNA patterns or maybe analyze network sequences (read: strange and maybe proprietary network protocols), or domestic stuff like building contextual links on your blog posts to enrich your users experience.
+
+
+h2. Okay, so how can I install it?
+
+
+h3. Rubygems - Development Version
+
+<pre>
+gem install aurelian-ruby-ahocorasick --source=http://gems.github.com
+</pre>
+
+
+h3. Build it from source
+
+<pre>
+$ git clone git://github.com/aurelian/ruby-ahocorasick.git
+$ cd ruby-ahocorasick
+</pre>
+
+To build and install the gem on your machine (run with sudo if needed):
+
+<pre>
+$ rake install
+</pre>
+
+@rake -T@ will list other cool tasks.
+
+
+h3. Rubygems - Stable Version
+
+There's no stable version right now.
+
+
+h4. Notes
+
+It's known to work / compile / install on Ubuntu 8.04 and Mac OS 10.4.*. It should work out of the box if you have gcc around.
+Unfortunately I don't have a Windows PC around nor required knowledge about Microsoft compliers.
+
+
+h2. Okay, so how do I use it?
+
+<pre>
+  require 'ahocorasick'
+
+  keyword_tree= AhoCorasick::KeywordTree.new # creates a new tree
+  keyword_tree.add_string( "foo-- Z@!bar" ) # add's a keyword to the tree
+  keyword_tree.add_string( "cervantes" )    # even more
+
+  results= keyword_tree.find_all( "1011000129 foo-- Z@!bar761 ! 001211 6xU" ).each do | result |
+    result[:value]     # => "foo-- Z@!bar"
+    result[:starts_at] # => 11
+    result[:ends_at]   # => 23
+    result[:id]        # => 1
+  end
+
+</pre>
+
+You can get some API reference on the "wiki":http://github.com/aurelian/ruby-ahocorasick/wikis.
+
+
+h2. Bugs? Suggestions? Ideas? Patches?
+
+For now, just use the email address.
+
+
+h2. Additional Reading 
+
+Other suffix - tree implementations:
+
+ * "Strmat":http://www.cs.ucdavis.edu/~gusfield/strmat.html
+ * "Pytst":http://nicolas.lehuen.com/download/pytst / "Ruby-Pytst":http://rubyforge.org/projects/ruby-pytst/
+ * "Aho-Corasick extension":http://hkn.eecs.berkeley.edu/~dyoo/python/ahocorasick/
+ * "Keyword Prospector":http://latimes.rubyforge.org/keyword_prospector/rdoc/
+ * "libstree":http://www.cl.cam.ac.uk/~cpk25/libstree/
+
+
+h2. License
+
+(c) 2008 - Aurelian Oancea, < oancea at gmail dot com > 
+
+released under MIT-LICENCE

data/examples/dict.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'time'
+
+require File.dirname(__FILE__) + '/../ext/ahocorasick'
+
+t= Time.now
+
+k= AhoCorasick::KeywordTree.from_file(File.dirname(__FILE__) + "/../spec/data/en.words")
+
+t1= Time.now
+
+puts "%d words added in %s seconds" % [k.size, (t1-t)]
+
+query = File.read( File.dirname(__FILE__) + "/../spec/data/news.txt" )
+
+results= k.search query
+
+puts "took %s seconds to find %d results in a streem with %d charachters" % [(Time.now-t1), results.size, query.size]
+
+exit
+results.each do | r |
+  puts query[r[:starts_at]].chr + ".." + query[r[:ends_at]-1].chr + " => " + r[:value]
+end
+

data/examples/elev.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+$kcode='UTF-8'
+
+require File.dirname(__FILE__) + '/../ext/ahocorasick'
+
+k= AhoCorasick::KeywordTree.new
+
+k << "I've"
+k << "data"
+k << "base"
+k << "database"
+
+query= "I've moved my data to a database"
+
+k.search(query).each do | r |
+  puts "-> [ " + r[:id].to_s + " ] " + r[:value] + " / " + query[r[:starts_at]].chr + ".." + query[r[:ends_at]-1].chr
+end
+

data/examples/sample.c

@@ -0,0 +1,94 @@
+//
+// Getting started with Aho-Corasick from Strmat
+//
+// lasick Makefile:
+//
+// --
+// ac.o :
+//   gcc -c -fPIC -shared ac.c
+// libasick : ac.o
+//   gcc -shared -Wl,-soname,libasick.so -o libasick.so.1.0.1
+//   ar rcs libasick.a ac.o
+// clean : 
+//   rm -rf *.o *.a *.so* *.dylib*
+// --
+//
+// Compile this stuff - asick is the library name, generated with the above Makefile :)
+//
+//   gcc sample.c -o ac-sample -I../ext/ -L../ext/ -lasick
+//
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "ac.h"
+
+int main(int argc, char *argv[]) {
+  char * search;
+  char * remain;
+  char * result;
+  char word[1024];
+
+  FILE *dictionary;
+  FILE *input;
+  int lgt, id, n, i;
+
+  AC_STRUCT * tree;
+  
+  input= fopen(argv[1], "r");
+
+  if(input == NULL) {
+    search= argv[1];
+  } else {
+    long lSize;
+    fseek (input , 0 , SEEK_END);
+    lSize= ftell (input);
+    rewind(input);
+    search = (char*) malloc (sizeof(char)*lSize);
+    if (search == NULL) { fputs ("Error: Memory error",stderr); exit(-2); }
+    fread(search, 1, lSize-1, input);
+  }
+
+  dictionary= fopen("../spec/data/dictionary.txt", "r");
+
+  if(dictionary == NULL) {
+    printf("Error: can't open file.\n");
+    return -1;
+  }
+
+  tree= ac_alloc();
+
+  // start counting from 1
+  n= 1;
+  
+  printf("==> building dictionary ...");
+
+  while(fgets(word, 1024, dictionary) != NULL) {
+    // strip \n
+    ac_add_string(tree, word, strlen(word)-1, n++);
+  }
+
+  printf("%d entries added.\n",n);
+
+  ac_prep(tree);
+
+  printf("==> input text [%d]:\n--\n%s\n--\n", strlen(search), search);
+
+  ac_search_init(tree, search, strlen(search) );
+  
+  while((remain= ac_search(tree, &lgt, &id)) != NULL) {
+    printf("`%d'", remain[lgt+1]);
+    result = (char*) malloc (sizeof(char)*lgt);
+    sprintf( result, "%.*s", lgt, remain);
+    // result: should read first lgt chars from remain.
+    printf("==> result: lenght=> %d, id=> %d [%s]\n", lgt, id, result);
+    free(result);
+  }
+
+  ac_free(tree);
+  fclose(dictionary);
+  free(search);
+
+  return 0;
+}
+

data/examples/test.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../ext/ahocorasick'
+
+k= AhoCorasick::KeywordTree.new
+
+puts k.size
+k.add_string("foo");
+
+puts k.size
+k.add_string("bar", 1991);
+
+puts k.size
+k.add_string("fomz");
+
+begin
+  k.add_string("foo", -1);
+rescue RuntimeError => err
+  puts "[ok]==> got " + err.class.name + ": " + err.message
+end
+
+begin
+  k.add_string("foo", "bar");
+rescue RuntimeError => err
+  puts "[ok]==> got " + err.class.name + ": " + err.message
+end
+
+k.add_string("timisoara", 22);
+
+puts k.size
+begin
+  k.add_string("bucuresti", 22);
+rescue RuntimeError => err
+  puts "[ok]==> got " + err.class.name + ": " + err.message
+end
+
+k << "bacau"
+
+k.search('am fost la bacau').each do | result |
+  puts result.inspect
+end
+
+k.search( 'din foo in foo' ).each do | q |
+  puts q.inspect
+end
+

data/ext/ac.c

@@ -0,0 +1,623 @@
+/*
+ * ac.c
+ *
+ * Implementation of the Aho-Corasick algorithm.
+ *
+ * NOTES:
+ *    8/94  -  Original Implementation  (Sean Davis)
+ *    9/94  -  Redid Implementation  (James Knight)
+ *    3/96  -  Modularized the code  (James Knight)
+ *    7/96  -  Finished the modularization  (James Knight)
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "ac.h"
+
+/*
+ * ac_alloc
+ *
+ * Creates a new AC_STRUCT structure and initializes its fields.
+ *
+ * Parameters:    none.
+ *
+ * Returns:  A dynamically allocated AC_STRUCT structure.
+ */
+AC_STRUCT *ac_alloc(void)
+{
+  AC_STRUCT *node;
+
+  if ((node = malloc(sizeof(AC_STRUCT))) == NULL)
+    return NULL;
+  memset(node, 0, sizeof(AC_STRUCT));
+
+  if ((node->tree = malloc(sizeof(ACTREE_NODE))) == NULL) {
+    free(node);
+    return NULL;
+  }
+  memset(node->tree, 0, sizeof(ACTREE_NODE));
+
+  return node;
+}
+
+
+/*
+ * ac_add_string
+ *
+ * Adds a string to the AC_STRUCT structure's keyword tree.
+ *
+ * NOTE:  The `id' value given must be unique to any of the strings
+ *        added to the tree, and must be a small integer greater than
+ *        0 (since it is used to index an array holding information
+ *        about each of the strings).
+ *
+ *        The best id's to use are to number the strings from 1 to K.
+ *
+ * Parameters:   node      -  an AC_STRUCT structure
+ *               P         -  the sequence
+ *               M         -  the sequence length
+ *               id        -  the sequence identifier
+ *
+ * Returns:  non-zero on success, zero on error.
+ */
+int ac_add_string(AC_STRUCT *node, char *P, int M, int id)
+{
+  int i, j, newsize;
+  AC_TREE tnode, child, back, newnode, list, tail;
+
+  /*
+   * Return a zero if a previous error had occurred, or if the
+   * given id equals zero.  An id value of zero is used by the 
+   * algorithm to signal that no pattern ends at a node in the
+   * keyword tree.  So, it can't be used as a pattern's id.
+   */
+  if (node->errorflag || id == 0)
+    return 0;
+
+  P--;            /* Shift to make sequence be P[1],...,P[M] */
+
+  /*
+   * Allocate space for the new string's information.
+   */
+  if (node->Psize <= id) {
+    if (node->Psize == 0) {
+      newsize = (id >= 16 ? id + 1 : 16);
+      node->Plengths = malloc(newsize * sizeof(int));
+    }
+    else {
+      newsize = node->Psize + id + 1;
+      node->Plengths = realloc(node->Plengths, newsize * sizeof(int));
+    }
+    if (node->Plengths == NULL) {
+      node->errorflag = 1;
+      return 0;
+    }
+
+    for (i=node->Psize; i < newsize; i++)
+      node->Plengths[i] = 0;
+    node->Psize = newsize;
+  }
+  
+  // duplicate id
+  if (node->Plengths[id] != 0)
+    return 0;
+
+  /*
+   * Add the string to the keyword tree.
+   */
+  tnode = node->tree;
+  for (i=1; i <= M; i++) {
+    /*
+     * Find the child whose character is P[i].
+     */
+    back = NULL;
+    child = tnode->children;
+    while (child != NULL && child->ch < P[i]) {
+      back = child;
+      child = child->sibling;
+    }
+
+    if (child == NULL || child->ch != P[i])
+      break;
+
+    tnode = child;
+
+#ifdef STATS
+    node->prep_old_edges++;
+#endif
+
+  }
+
+  /*
+   * If only part of the pattern exists in the tree, add the
+   * rest of the pattern to the tree.
+   */
+  if (i <= M) {
+    list = tail = NULL;
+    for (j=i; j <= M; j++) {
+      if ((newnode = malloc(sizeof(ACTREE_NODE))) == NULL)
+        break;
+      memset(newnode, 0, sizeof(ACTREE_NODE));
+      newnode->ch = P[j];
+
+      if (list == NULL)
+        list = tail = newnode;
+      else
+        tail = tail->children = newnode;
+
+#ifdef STATS
+      node->prep_new_edges++;
+#endif
+
+    }
+    if (j <= M) {
+      while (list != NULL) {
+        tail = list->children;
+        free(list);
+        list = tail;
+      }
+      return 0;
+    }
+
+    list->sibling = child;
+    if (back == NULL)
+      tnode->children = list;
+    else
+      back->sibling = list;
+
+    tnode = tail;
+  }
+
+  tnode->matchid = id;
+  node->Plengths[id] = M;
+  node->ispreprocessed = 0;
+
+  return 1;
+}
+
+
+/*
+ * ac_del_string
+ *
+ * Deletes a string from the keyword tree.
+ *
+ * Parameters:   node  -  an AC_STRUCT structure
+ *               P     -  the sequence to be deleted
+ *               M     -  its length
+ *               id    -  its identifier
+ *
+ * Returns:  non-zero on success, zero on error.
+ */
+int ac_del_string(AC_STRUCT *node, char *P, int M, int id)
+{
+  int i, flag;
+  AC_TREE tnode, tlast, tback, child, back;
+
+  if (node->errorflag || id > node->Psize || node->Plengths[id] == 0)
+    return 0;
+
+  P--;            /* Shift to make sequence be P[1],...,P[M] */
+
+  /*
+   * Scan the tree for the path corresponding to the keyword to be deleted.
+   */
+  flag = 1;
+  tlast = tnode = node->tree;
+  tback = NULL;
+
+  for (i=1; i <= M; i++) {
+    /*
+     * Find the child matching P[i].  It must be there.
+     */
+    child = tnode->children;
+    back = NULL;
+    while (child != NULL && child->ch != P[i]) {
+      back = child;
+      child = child->sibling;
+    }
+
+    if (child == NULL) {
+      fprintf(stderr, "Error in Aho-Corasick preprocessing.  String to be "
+              "deleted is not in tree.\n");
+      return 0;
+    }
+
+    /*
+     * Try to find the point where the pattern to be deleted branches off
+     * from the paths of the other patterns in the tree.  This point must
+     * be at the latest node which satisfies one of these two conditions:
+     *
+     *    1) Another pattern ends at that node (and so
+     *       `child->matchid != 0').  In this case, the branch point is
+     *       just below this node and so the children of this node
+     *       should be removed.
+     *    2) A node has other siblings.  In this case, the node itself
+     *       is the branch point, and it and its children should be
+     *       removed.
+     */
+    if (i < M && child->matchid != 0) {
+      flag = 1;
+      tlast = child;
+    }
+    else if (back != NULL || child->sibling != NULL) {
+      flag = 2;
+      tlast = child;
+      tback = (back == NULL ? tnode : back);
+    }
+
+    tnode = child;
+  }
+
+  /*
+   * If the node corresponding to the end of the keyword has children,
+   * then the tree should not be altered, except to remove the keyword's
+   * identifier from the tree.
+   *
+   * Otherwise, apply the appropriate removal, as described above.
+   */
+  if (tnode->children != NULL) {
+    tnode->matchid = 0;
+  }
+  else {
+    if (flag == 1) {
+      child = tlast->children;
+      tlast->children = NULL;
+      tlast = child;
+    }
+    else {
+      if (tback->children == tlast)
+        tback->children = tlast->sibling;
+      else
+        tback->sibling = tlast->sibling;
+    }
+
+    while (tlast != NULL) {
+      child = tlast->children;
+      free(tlast);
+      tlast = child;
+    }
+  }
+
+  node->Plengths[id] = 0;
+  node->ispreprocessed = 0;
+
+  return 1;
+}
+
+
+/*
+ * ac_prep
+ *
+ * Compute the failure and output links for the keyword tree.
+ *
+ * Parameters:  node  -  an AC_STRUCT structure
+ *
+ * Returns: non-zero on success, zero on error.
+ */
+int ac_prep(AC_STRUCT *node)
+{
+  char x;
+  AC_TREE v, vprime, w, wprime, root, front, back, child;
+
+  if (node->errorflag)
+    return 0;
+
+  /*
+   * The failure link and output link computation requires a breadth-first
+   * traversal of the keyword tree.  And, to do that, we need a queue of
+   * the nodes yet to be processed.
+   *
+   * The `faillink' fields will be used as the pointers for the queue
+   * of nodes to be computed (since the failure link is only set after
+   * the node is removed from the queue).
+   * 
+   * The `outlink' fields will be used as the pointers to a node's parent
+   * for nodes in the queue (since the output link is also only set after
+   * the node is removed from the queue).
+   */
+  root = node->tree;
+
+  front = back = root;
+  front->faillink = NULL;
+  front->outlink = NULL;
+
+  while (front != NULL) {
+    v = front;
+    x = v->ch;
+    vprime = v->outlink;
+
+    /*
+     * Add the node's children to the queue.
+     */
+    for (child=v->children; child != NULL; child=child->sibling) {
+      child->outlink = v;
+      back->faillink = child;
+      back = child;
+    }
+    back->faillink = NULL;
+
+    front = front->faillink;
+    v->faillink = v->outlink = NULL;
+    
+    /*
+     * Set the failure and output links.
+     */
+    if (v == root)
+      ;
+    else if (vprime == root)
+      v->faillink = root;
+    else {
+      /*
+       * Find the find link in the failure link chain which has a child
+       * labeled with x.
+       */
+      wprime = NULL;
+      w = vprime->faillink;
+
+      while (1) {
+        wprime = w->children;
+        while (wprime != NULL && wprime->ch < x)
+          wprime = wprime->sibling;
+
+        if ((wprime != NULL && wprime->ch == x) || w == root)
+          break;
+
+        w = w->faillink;
+
+#ifdef STATS
+        node->prep_fail_compares++;
+#endif
+      }
+#ifdef STATS
+      node->prep_fail_compares++;
+#endif
+
+      if (wprime != NULL && wprime->ch == x)
+        v->faillink = wprime;
+      else
+        v->faillink = root;
+
+      if (v->matchid != 0) {
+        if (v->faillink->matchid != 0)
+          v->outlink = v->faillink;
+        else
+          v->outlink = v->faillink->outlink;
+      }
+    }
+  }
+
+  node->ispreprocessed = 1;
+  node->initflag = 0;
+
+  return 1;
+}
+
+
+/*
+ * ac_search_init
+ *
+ * Initializes the variables used during an Aho-Corasick search.
+ * See ac_search for an example of how it should be used.
+ *
+ * Parameters:  node  -  an AC_STRUCT structure
+ *              T     -  the sequence to be searched
+ *              N     -  the length of the sequence
+ *
+ * Returns:  nothing.
+ */
+void ac_search_init(AC_STRUCT *node, char *T, int N)
+{
+  if (node->errorflag)
+    return;
+  else if (!node->ispreprocessed) {
+    fprintf(stderr, "Error in Aho-Corasick search.  The preprocessing "
+            "has not been completed.\n");
+    return;
+  }
+
+  node->T = T - 1;          /* Shift to make sequence be T[1],...,T[N] */
+  node->N = N;
+  node->c = 1;
+  node->w = node->tree;
+  node->output = NULL;
+  node->initflag = 1;
+  node->endflag = 0;
+}
+
+
+/*
+ * ac_search
+ *
+ * Scans a text to look for the next occurrence of one of the patterns
+ * in the text.  An example of how this search should be used is the
+ * following:
+ *
+ *    s = T;
+ *    len = N; 
+ *    contflag = 0;
+ *    ac_search_init(node, T, N);
+ *    while ((s = ac_search(node, &matchlen, &matchid) != NULL) {
+ *      >>> Pattern `matchid' matched from `s' to `s + matchlen - 1'. <<<
+ *    }
+ *
+ * where `node', `T' and `N' are assumed to be initialized appropriately.
+ *
+ * Parameters:  node           -  a preprocessed AC_STRUCT structure
+ *              length_out     -  where to store the new match's length
+ *              id_out         -  where to store the identifier of the
+ *                                pattern that matched
+ *              ends_at        -  where to store the n-th matched char                  
+ *
+ * Returns:  the left end of the text that matches a pattern, or NULL
+ *           if no match occurs.  (It also stores values in `*length_out',
+ *           and `*id_out' giving the match's length and pattern identifier.
+ */
+char *ac_search(AC_STRUCT *node, int *length_out, int *id_out, int *ends_at)
+{
+  int c, N, id;
+  char *T;
+  AC_TREE w, wprime, root;
+
+  if (node->errorflag)
+    return NULL;
+  else if (!node->ispreprocessed) {
+    fprintf(stderr, "Error in Aho-Corasick search.  The preprocessing "
+            "has not been completed.\n");
+    return NULL;
+  }
+  else if (!node->initflag) {
+    fprintf(stderr, "Error in Aho-Corasick search.  ac_search_init was not "
+            "called.\n");
+    return NULL;
+  }
+  else if (node->endflag)
+    return NULL;
+
+  T = node->T;
+  N = node->N;
+  c = node->c;
+  w = node->w;
+  root = node->tree;
+
+  /*
+   * If the last call to ac_search returned a match, check for another
+   * match ending at the same right endpoint (denoted by a non-NULL
+   * output link).
+   */
+  if (node->output != NULL) {
+    node->output = node->output->outlink;
+
+#ifdef STATS
+  node->outlinks_traversed++;
+#endif
+
+    if (node->output != NULL) {
+      id = node->output->matchid;
+      if (id_out)
+        *id_out = id;
+      if (length_out)
+        *length_out = node->Plengths[id];
+      if (ends_at)
+        *ends_at= c;
+      return &T[c] - node->Plengths[id];
+    }
+
+  }
+
+  /*
+   * Run the search algorithm, stopping at the first position where a
+   * match to one of the patterns occurs.
+   */
+  while (c <= N) {
+    /*
+     * Try to match the next input character to a child in the tree.
+     */
+    wprime = w->children;
+    while (wprime != NULL && wprime->ch != T[c])
+      wprime = wprime->sibling;
+
+#ifdef STATS
+    node->num_compares++;
+#endif
+
+    /*
+     * If the match fails, then either use the failure link (if not
+     * at the root), or move to the next character since no prefix
+     * of any pattern ends with character T[c].
+     */
+    if (wprime == NULL) {
+      if (w == root)
+        c++;
+      else {
+        w = w->faillink;
+
+#ifdef STATS
+        node->num_failures++;
+#endif
+
+      }
+    }
+    else {
+      /*
+       * If we could match the input, move down the tree and to the
+       * next input character, and see if that match completes the
+       * match to a pattern (when matchid != 0 or outlink != NULL).
+       */
+      c++;
+      w = wprime;
+
+#ifdef STATS
+      node->edges_traversed++;
+#endif
+
+      if (w->matchid != 0)
+        node->output = w;
+      else if (w->outlink != NULL) {
+        node->output = w->outlink;
+
+#ifdef STATS
+        node->outlinks_traversed++;
+#endif
+
+      }
+
+      if (node->output != NULL) {
+        id = node->output->matchid;
+        if (id_out)
+          *id_out = id;
+        if (length_out)
+          *length_out= node->Plengths[id];
+        if(ends_at)
+          *ends_at= c;
+
+        node->w = w;
+        node->c = c; // ends_at - length_out;
+
+        return &T[c] - node->Plengths[id];
+      }
+    }
+  }
+
+  node->c = c;
+  node->endflag = 1;
+
+  return NULL;
+}
+
+
+/*
+ * ac_free
+ *
+ * Free up the allocated AC_STRUCT structure.
+ *
+ * Parameters:   node  -  a AC_STRUCT structure
+ *
+ * Returns:  nothing.
+ */
+void ac_free(AC_STRUCT *node)
+{
+  AC_TREE front, back, next;
+
+  if (node == NULL)
+    return;
+
+  if (node->tree != NULL) {
+    front = back = node->tree;
+    while (front != NULL) {
+      back->sibling = front->children;
+      while (back->sibling != NULL)
+        back = back->sibling;
+
+      next = front->sibling;
+      free(front);
+      front = next;
+    }
+  }
+
+  if (node->Plengths != NULL)
+    free(node->Plengths);
+
+  free(node);
+}
+

data/ext/ac.h

@@ -0,0 +1,36 @@
+
+#ifndef _AC_H_
+#define _AC_H_
+
+typedef struct actreenode {
+  char ch;
+  int matchid;
+  struct actreenode *outlink, *faillink;
+  struct actreenode *children, *sibling;
+} ACTREE_NODE, *AC_TREE;
+
+typedef struct {
+  AC_TREE tree;
+  int ispreprocessed, errorflag;
+
+  int Psize;
+  int *Plengths;
+
+  char *T;
+  int N, c, initflag, endflag;
+  AC_TREE w, output;
+#ifdef STATS
+  int prep_new_edges, prep_old_edges, prep_fail_compares;
+  int num_compares, num_failures, edges_traversed, outlinks_traversed;
+#endif
+} AC_STRUCT;
+
+AC_STRUCT *ac_alloc(void);
+int ac_add_string(AC_STRUCT *node, char *P, int M, int id);
+// int ac_del_string(AC_STRUCT *node, char *P, int M, int id);
+int ac_prep(AC_STRUCT *node);
+void ac_search_init(AC_STRUCT *node, char *T, int N);
+char *ac_search(AC_STRUCT *node, int *length_out, int *id_out, int *ends_at);
+void ac_free(AC_STRUCT *node);
+
+#endif

data/ext/extconf.rb

@@ -0,0 +1,6 @@
+require "mkmf"
+
+dir_config("ahocorasick")
+
+create_makefile("ahocorasick")
+

data/ext/ruby-ahocorasick.c

@@ -0,0 +1,339 @@
+
+//
+// (c) 2008, Aurelian Oancea < oancea at gmail dot com >
+//
+// Released under MIT-LICENSE
+//
+
+//
+// TODO: new methods?
+//
+//  * kwt[id] = word
+//  * kwt.from_file (class instance method)
+//
+//  * kwt.find_each ("str") {|r| .. }
+//  * kwt.find_first("str")
+//  * kwt.find_all  ("str")
+//
+// TODO: change last_id and dictionary_size to long
+//
+
+#include <ruby.h>
+#include "ac.h"
+
+static VALUE sym_id, sym_value, sym_ends_at, sym_starts_at;
+
+VALUE rb_mAhoCorasick;
+VALUE rb_cKeywordTree;
+
+#define KeywordTree(obj, kwt_data) {\
+  Data_Get_Struct(obj, struct kwt_struct_data, kwt_data);\
+}
+
+struct kwt_struct_data {
+  AC_STRUCT * tree;
+  int last_id;
+  int dictionary_size;
+  int is_frozen;
+};
+
+// int
+// rb_add_string(struct kwt_struct_data *kwt, char *word, int size, int id) {
+//   if(ac_add_string( kwt->tree, word, size, id ) == 0)
+//     return 0;
+//   kwt->dictionary_size++;
+//   kwt->last_id= id+1;
+//   return 1;
+// }
+
+/*
+ * call-seq: initialize
+ *
+ * Creates a new KeywordTree
+ *
+ *   require 'ahocorasick'
+ *   kwt = Ahocorasick::KeywordTree.new
+ *
+ */
+static VALUE 
+rb_kwt_init(VALUE self)
+{ 
+  AC_STRUCT * tree;
+  struct kwt_struct_data *kwt_data;
+
+  kwt_data = ALLOC(struct kwt_struct_data);
+  tree     = ac_alloc();
+  DATA_PTR(self) = kwt_data;
+  kwt_data->tree            = tree;
+  kwt_data->last_id         = 1;
+  kwt_data->dictionary_size = 0;
+  kwt_data->is_frozen       = 0;
+  return self;
+}
+
+/*
+ * Document-method: make
+ * call-seq: make
+ *
+ * It freezes the current KeywordTree.
+ *
+ * ==== Note: This method is called internally by search
+ *
+ *   require 'ahocorasick'
+ *   
+ *   kwt = Ahocorasick::KeywordTree.new
+ *
+ *   kwt.add_string("one")
+ *   kwt.add_string("two")
+ *   kwt.make()
+ */
+static VALUE 
+rb_kwt_make(VALUE self)
+{ 
+  struct kwt_struct_data *kwt_data;
+  KeywordTree(self, kwt_data);
+
+  if(kwt_data->is_frozen == 1)
+    return Qtrue;
+  
+  if(ac_prep( kwt_data->tree ) == 1) {
+    kwt_data->is_frozen = 1;
+    return Qtrue;
+  }
+
+  rb_raise(rb_eRuntimeError, "Cannot freeze the tree");
+}
+
+/*
+ * Document-method: find_all
+ * call-seq: find_all
+ *
+ * Search the current tree.
+ *
+ * It returns an array on hashes, e.g.
+ *
+ *   [ { :id => int, :value => int, :starts_at => int, :ends_at => int}, { ... } ]
+ * 
+ * Returns an empty array when the search didn't return any result.
+ *
+ *   # assuming a valid KeywordTree kwt object:
+ *   kwt.add_string("one")
+ *   kwt.add_string("two")
+ *
+ *   kwt.search( "moved two times already" ).each  do | result |
+ *     result[:id] # => 2
+ *     result[:ends_at] # => 9
+ *     result[:starts_at] # => 6
+ *     result[:value] # => two
+ *   end # => 1
+ *
+ */
+static VALUE
+rb_kwt_find_all(int argc, VALUE *argv, VALUE self)
+{
+  char * remain;        // returned by ac_search, the remaing text to search
+  int lgt, id, ends_at, starts_at; // filled in by ac_search: the length of the result, the id, and starts_at/ends_at position
+  VALUE v_result;  // one result, as hash
+  VALUE v_results; // all the results, an array
+
+  VALUE v_search;  // search string, function argument
+  struct kwt_struct_data *kwt_data;
+  
+  // one mandatory argument.
+  rb_scan_args(argc, argv, "1", &v_search);
+  // it should be string.
+  Check_Type(v_search, T_STRING);
+  // get the structure
+  KeywordTree(self, kwt_data);
+  // freeze the tree, if not already
+  if(kwt_data->is_frozen == 0) {
+    if(ac_prep( kwt_data->tree ) == 0) 
+      rb_raise(rb_eRuntimeError, "Cannot freeze the tree");
+    kwt_data->is_frozen = 1;
+  }
+  // prepare the return value
+  v_results= rb_ary_new();
+  // fail quickly and return the empty array
+  if(kwt_data->dictionary_size == 0) 
+    return v_results;
+  // prepare the search
+  ac_search_init(kwt_data->tree, RSTRING( v_search )->ptr, RSTRING( v_search )->len);
+  // loop trought the results
+  while((remain= ac_search(kwt_data->tree, &lgt, &id, &ends_at)) != NULL) {
+    // this is an individual result as a hash
+    v_result= rb_hash_new();
+    rb_hash_aset( v_result, sym_id,        INT2FIX(id) );
+    rb_hash_aset( v_result, sym_starts_at, INT2FIX( ends_at - lgt - 1 ) );
+    rb_hash_aset( v_result, sym_ends_at,   INT2FIX( ends_at - 1 ) );
+    rb_hash_aset( v_result, sym_value, rb_str_new(remain, lgt) );
+    rb_ary_push( v_results, v_result );
+  }
+  // reopen the tree
+  kwt_data->is_frozen= 0;
+  return v_results;
+}
+
+/*
+ * Document-method: size
+ * call-seq: size
+ *
+ * Returns the size of this KeywordTree
+ *
+ *    kwt.add_string("foo")
+ *    kwt.add_string("bar")
+ *    kwt.size #=> 2
+ *
+ */ 
+static VALUE 
+rb_kwt_size(VALUE self)
+{ 
+  struct kwt_struct_data *kwt_data;
+  KeywordTree(self, kwt_data);
+
+  return INT2FIX(kwt_data->dictionary_size);
+}
+
+/*
+ * Document-method: add_string
+ * call-seq: add_string
+ *
+ * Adds a sequence to this KeywordTree.
+ *
+ *    kwt.add_string("foo1$21^ 98N3 ba>Z")
+ *    kwt << "bar" # using the alias
+ * 
+ * ==== Note: you can also specify the id, a number between 1 and k
+ *
+ *    kwt.add_string "bar", 123 # => 123
+ *
+ * This id should be unique in the context of the current tree.
+ *
+ * Returns the id of the inserted object.
+ *
+ *    kwt.add_string("test", 18) # => 18
+ *    kwt.add_string("baz") # => 19
+ *
+ */ 
+static VALUE
+rb_kwt_add_string(int argc, VALUE *argv, VALUE self)
+{ 
+  VALUE v_string, v_id;
+  struct kwt_struct_data *kwt_data;
+  char * string;
+  int id;
+
+  rb_scan_args(argc, argv, "11", &v_string, &v_id);
+ 
+  Check_Type(v_string, T_STRING);
+  string= RSTRING(v_string)->ptr;
+  KeywordTree(self, kwt_data);
+
+  if(kwt_data->is_frozen == 1)
+    rb_raise(rb_eRuntimeError, "Cannot add `%s\" into a frozen tree.", string);
+
+  if(v_id == Qnil) {
+    id = kwt_data->last_id;
+  } else if(TYPE(v_id) != T_FIXNUM) {
+    rb_raise(rb_eRuntimeError, "Please use a number from 1 to K as id, or leave nil to auto-generate one. `%s\" given.", RSTRING(v_id)->ptr);
+  } else if(NUM2INT(v_id) <= 0) {
+    rb_raise(rb_eRuntimeError, "Please use a number from 1 to K as id, or leave nil to auto-generate one. `%d\" given.", NUM2INT(v_id));
+  } else {
+    id= NUM2INT(v_id);
+  }
+  
+  if(ac_add_string(kwt_data->tree, string, strlen(string), id) == 0)
+    rb_raise(rb_eRuntimeError, "Failed to add `%s\", duplicate id `%d\"?", string, id);
+
+  kwt_data->last_id= id + 1;
+  kwt_data->dictionary_size++;
+  return INT2FIX(id);
+}
+
+/*
+ * call-seq: from_file
+ *
+ * Creates a new KeywordTree and loads the dictionary from a file
+ * 
+ *    % cat dict0.txt
+ *    foo
+ *    bar
+ *    base
+ *     
+ *    k= AhoCorasick::KeywordTree.from_file "dict0.txt"
+ *    k.search("basement").size # => 1
+ *
+ */
+static VALUE
+rb_kwt_new_from_file(int argc, VALUE *argv, VALUE klass)
+{ 
+
+  // TODO: 
+  //  * use rb_kwt_add_string
+  //  * use rb_io* to handle the file
+
+  struct kwt_struct_data *kwt_data;
+  char word[1024];
+  int id;
+  VALUE self;
+  VALUE f_string;
+  FILE *dictionary;
+
+  rb_scan_args(argc, argv, "10", &f_string);
+ 
+  id = 0;
+  SafeStringValue( f_string );
+  self= rb_class_new_instance( 0, NULL, klass );
+  KeywordTree( self, kwt_data );
+
+  dictionary = fopen( RSTRING( f_string )->ptr, "r" );
+  if(dictionary == NULL)
+    rb_raise(rb_eRuntimeError, "Cannot open `%s\". No such file?", RSTRING(f_string)->ptr);
+
+  while(fgets(word, 1024, dictionary) != NULL) {
+    ac_add_string(kwt_data->tree, word, strlen(word)-1, id++);
+    kwt_data->dictionary_size++;
+  }
+
+  kwt_data->last_id= id+1;
+  fclose(dictionary);
+  return self;
+}
+
+static void
+rb_kwt_struct_free(struct kwt_struct_data * kwt_data)
+{
+  ac_free(kwt_data->tree);
+}
+
+static VALUE
+rb_kwt_struct_alloc(VALUE klass)
+{
+  return Data_Wrap_Struct(klass, 0, rb_kwt_struct_free, 0);
+}
+
+/*
+ * Blump.
+ */
+void Init_ahocorasick() {
+  rb_mAhoCorasick = rb_define_module("AhoCorasick");
+  rb_cKeywordTree = rb_define_class_under(rb_mAhoCorasick, "KeywordTree", rb_cObject);
+  
+  rb_define_alloc_func(rb_cKeywordTree, rb_kwt_struct_alloc);
+
+  rb_define_method(rb_cKeywordTree, "initialize", rb_kwt_init, 0);
+  rb_define_method(rb_cKeywordTree, "size", rb_kwt_size, 0);
+  rb_define_method(rb_cKeywordTree, "make", rb_kwt_make, 0);
+  rb_define_method(rb_cKeywordTree, "add_string", rb_kwt_add_string, -1);
+  rb_define_alias(rb_cKeywordTree, "<<", "add_string");
+
+  rb_define_method(rb_cKeywordTree, "find_all", rb_kwt_find_all, -1);
+  rb_define_alias(rb_cKeywordTree, "search", "find_all");
+
+  rb_define_singleton_method(rb_cKeywordTree, "from_file", rb_kwt_new_from_file, -1);
+
+  sym_id       = ID2SYM(rb_intern("id"));
+  sym_value    = ID2SYM(rb_intern("value"));
+  sym_ends_at  = ID2SYM( rb_intern("ends_at") );
+  sym_starts_at= ID2SYM( rb_intern("starts_at") );
+}
+

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification 
+name: ruby-ahocorasick
+version: !ruby/object:Gem::Version 
+  version: 0.4.5
+platform: ruby
+authors: 
+- Aurelian Oancea
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-11-19 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Expose Aho-Corasick implementation from Strmat to Ruby.
+email: oancea at gmail dot com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- ext/extconf.rb
+- ext/ruby-ahocorasick.c
+- ext/ac.h
+- ext/ac.c
+- examples/dict.rb
+- examples/test.rb
+- examples/elev.rb
+- examples/sample.c
+- MIT-LICENSE
+- README.textile
+has_rdoc: true
+homepage: http://www.locknet.ro
+post_install_message: 
+rdoc_options: 
+- --title
+- Ruby-AhoCorasick
+- --inline-source
+- ext/ruby-ahocorasick.c
+- README.textile
+- --main
+- README.textile
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: ruby-ahocorasick
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: Aho-Corasick alghorithm implementation to Ruby using Strmat lib.
+test_files: []
+