tyler-trie 0.3.1 → 0.3.3

data/README.textile

@@ -44,7 +44,7 @@ Or if you have some integer data to store along with the words, such as weights
   end
 </code></pre>
 
-Great, so we've populated our trie with some words.  (Note the limitations section below.)  Let's make sure those words are really there.
+Great, so we've populated our trie with some words. Let's make sure those words are really there.
 
 <pre><code>
   trie.has_key?('widget')  #=> true
@@ -88,4 +88,45 @@ There are, of course, some more interesting and advanced ways to use a trie.  Fo
 By calling <code>root</code> on a Trie object, you get a TrieNode, pointed at the root of the trie.  You can then use this node to walk the trie and perceive things about each word.
 
 
+h2. Performance Characteristics
+
+Here are some quick benchmarks on my 2.4ghz Intel Core 2 Duo MacBook Pro:
+
+For keys that are 5 characters long:
+31,344 adds/second
+1,827,408 searches/second
+38,453 prefixes searches/second
+
+For keys that are 10 characters long:
+30,653 adds/second
+1,802,649 searches/second
+13,553 prefix searches/second
+
+For keys that are 20 characters long:
+30,488 adds/second
+1,851,461 searches/second
+5,855 prefix searches/second
+
+For keys that are 40 characters long:
+30,710 adds/second
+1,838,380 searches/second
+2,762 prefix searches/second
+
+
+There are a few takeaways from this. First, there is no strong correlation between length of keys and insert or retrieve time. They stay fairly constant as the length of keys increase. Secondly, doing prefix searches with this trie gets slower linearly with the length of the keys in the trie.
+
+This points to a limitation of this type of trie.  It is based on libdatrie, which is a dual-array trie.  When finding branches from a particular node, we must query all possible branches to determine whether or not they exist.  So for each node we do 255 of these queries.
+
+There may be some tricks to speed this up, but for now it is simply a limitation of this trie.
+
+Now, let's look at the effect of the size of the trie itself on query and insertion time.  For this test I inserted 100, 1000, 10000, 100000, and 1000000 words in the trie.  We measure the insertion and retrieval time in each.  The graph below shows the results.
+
+!http://codehallow.com/effect_of_size.png!
+
+So, keeping in mind that we're increasing by orders of magnitude, you can see that the insertion time does take a signifcant hit.  Retrieval also goes down but at a very gradual rate.  (It decreases by about 50% in total, despite the size increasing by 1,000,000%.)
+
+The reason the insertion times takes such a beating is due, again, to a limitation of the trie.  Storing a trie in the dual array setup that is used is excellent for memory usage and retrieval time.  Best in class, in fact. However, the more things are added into the trie the more complicated it gets to insert things.  It often requires shuffling large pieces of the arrays.  There may be room for optimization here, but ultimately insertion time will increase with the size of the trie.
+
+
+
 Copyright (c) 2008 Tyler McMullen. See LICENSE for details.

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-patch: 1
+patch: 3
 major: 0
 minor: 3

data/ext/trie/trie-private.c

@@ -102,6 +102,34 @@ Bool trie_store (Trie *trie, const TrieChar *key, TrieData data) {
 }
 
 
+Bool trie_has_key (const Trie *trie, const TrieChar *key) {
+    TrieIndex        s;
+    short            suffix_idx;
+    const TrieChar *p;
+
+    /* walk through branches */
+    s = da_get_root (trie->da);
+    for (p = key; !trie_da_is_separate (trie->da, s); p++) {
+        if (!da_walk (trie->da, &s, *p))
+            return FALSE;
+        if (0 == *p)
+            break;
+    }
+
+    /* walk through tail */
+    s = trie_da_get_tail_index (trie->da, s);
+    suffix_idx = 0;
+    for ( ; ; p++) {
+        if (!tail_walk_char (trie->tail, s, &suffix_idx, *p))
+            return FALSE;
+        if (0 == *p)
+            break;
+    }
+
+    return TRUE;
+}
+
+
 Bool trie_retrieve (const Trie *trie, const TrieChar *key, TrieData *o_data) {
     TrieIndex        s;
     short            suffix_idx;

data/ext/trie/trie.c

@@ -6,33 +6,64 @@
 
 VALUE cTrie, cTrieNode;
 
+/*
+ * Document-class: Trie
+ * 
+ * A key-value data structure for string keys which is efficient memory usage and fast retrieval time.
+ *
+ */
+
 static VALUE rb_trie_alloc(VALUE klass) {
 	VALUE obj;
 	obj = Data_Wrap_Struct(klass, 0, trie_free, trie_new());
 	return obj;
 }
 
+/*
+ * call-seq:
+ *   has_key?(key) -> true/false
+ *
+ * Determines whether or not a key exists in the Trie.  Use this if you don't care about the value, as it
+ * is marginally faster than Trie#get.
+ *
+ */
 static VALUE rb_trie_has_key(VALUE self, VALUE key) {
     Trie *trie;
     Data_Get_Struct(self, Trie, trie);
 
-    if(trie_retrieve(trie, (TrieChar*)RSTRING(key)->ptr, NULL))
+    if(trie_has_key(trie, (TrieChar*)RSTRING(key)->ptr))
 		return Qtrue;
     else
 		return Qnil;
 }
 
+/*
+ * call-seq:
+ *   get(key) -> value
+ *   [key]    -> value
+ *
+ * Retrieves the value for a particular key (or nil) from the Trie.
+ *
+ */
 static VALUE rb_trie_get(VALUE self, VALUE key) {
     Trie *trie;
     Data_Get_Struct(self, Trie, trie);
 
 	TrieData data;
     if(trie_retrieve(trie, (TrieChar*)RSTRING(key)->ptr, &data))
-		return INT2FIX(data);
+		return (VALUE)data;
     else
 		return Qnil;
 }
 
+/*
+ * call-seq:
+ *   add(key)
+ *   add(key,value)
+ *
+ * Add a key, or a key and value to the Trie.  If you add a key without a value it assumes true for the value. 
+ *
+ */
 static VALUE rb_trie_add(VALUE self, VALUE args) {
 	Trie *trie;
     Data_Get_Struct(self, Trie, trie);
@@ -43,7 +74,7 @@ static VALUE rb_trie_add(VALUE self, VALUE args) {
 
     VALUE key;
     key = RARRAY(args)->ptr[0];
-    TrieData value = size == 2 ? NUM2INT(RARRAY(args)->ptr[1]) : TRIE_DATA_ERROR;
+    TrieData value = size == 2 ? RARRAY(args)->ptr[1] : TRIE_DATA_ERROR;
     
     if(trie_store(trie, (TrieChar*)RSTRING(key)->ptr, value))
 		return Qtrue;
@@ -51,6 +82,13 @@ static VALUE rb_trie_add(VALUE self, VALUE args) {
 		return Qnil;
 }
 
+/*
+ * call-seq:
+ *   delete(key)
+ *
+ * Delete a key from the Trie.  Returns true if it deleted a key, nil otherwise.
+ *
+ */
 static VALUE rb_trie_delete(VALUE self, VALUE key) {
 	Trie *trie;
     Data_Get_Struct(self, Trie, trie);
@@ -61,14 +99,6 @@ static VALUE rb_trie_delete(VALUE self, VALUE key) {
 		return Qnil;
 }
 
-char* append_char(char* existing, int size, char c) {
-	char *new = (char*) malloc(size + 2);
-	memcpy(new, existing, size);
-	new[size] = c;
-	new[size + 1] = 0;
-	return new;
-}
-
 static VALUE walk_all_paths(Trie *trie, VALUE children, TrieState *state, char *prefix, int prefix_size) {
 	int c;
     for(c = 1; c < 256; c++) {
@@ -93,6 +123,13 @@ static VALUE walk_all_paths(Trie *trie, VALUE children, TrieState *state, char *
     }
 }
 
+/*
+ * call-seq:
+ *   children(prefix) -> [ key, ... ]
+ *
+ * Finds all keys in the Trie beginning with the given prefix. 
+ *
+ */
 static VALUE rb_trie_children(VALUE self, VALUE prefix) {
     if(NIL_P(prefix))
 		return rb_ary_new();
@@ -148,7 +185,7 @@ static VALUE walk_all_paths_with_values(Trie *trie, VALUE children, TrieState *s
 				rb_ary_push(tuple, rb_str_new2(word));
 
 				TrieData trie_data = trie_state_get_data(end_state);
-				rb_ary_push(tuple, INT2FIX(trie_data));
+				rb_ary_push(tuple, (VALUE)trie_data);
 				rb_ary_push(children, tuple);
  
 				trie_state_free(end_state);
@@ -162,9 +199,13 @@ static VALUE walk_all_paths_with_values(Trie *trie, VALUE children, TrieState *s
     }
 }
 
-
-
-
+/*
+ * call-seq:
+ *   children_with_values(key) -> [ [key,value], ... ]
+ *
+ * Finds all keys with their respective values in the Trie beginning with the given prefix. 
+ * 
+ */
 static VALUE rb_trie_children_with_values(VALUE self, VALUE prefix) {
     if(NIL_P(prefix))
 		return rb_ary_new();
@@ -194,7 +235,7 @@ static VALUE rb_trie_children_with_values(VALUE self, VALUE prefix) {
 		VALUE tuple = rb_ary_new();
 		rb_ary_push(tuple, prefix);
 		TrieData trie_data = trie_state_get_data(end_state);
-		rb_ary_push(tuple, INT2FIX(trie_data));
+		rb_ary_push(tuple, (VALUE)trie_data);
 		rb_ary_push(children, tuple);
 
 		trie_state_free(end_state);
@@ -210,12 +251,15 @@ static VALUE rb_trie_children_with_values(VALUE self, VALUE prefix) {
     return children;
 }
 
-static VALUE rb_trie_node_alloc(VALUE klass) {
-    VALUE obj;
-    obj = Data_Wrap_Struct(klass, 0, trie_state_free, NULL);
-    return obj;
-}
+static VALUE rb_trie_node_alloc(VALUE klass);
 
+/*
+ * call-seq:
+ *   root -> TrieNode
+ *
+ * Returns a TrieNode representing the root of the Trie.
+ *
+ */
 static VALUE rb_trie_root(VALUE self) {
     Trie *trie;
     Data_Get_Struct(self, Trie, trie);
@@ -230,13 +274,61 @@ static VALUE rb_trie_root(VALUE self) {
     return trie_node;
 }
 
+
+/*
+ * Document-class: TrieNode
+ * 
+ * Represents a single node in the Trie. It can be used as a cursor to walk around the Trie.
+ * You can grab a TrieNode for the root of the Trie by using Trie#root.
+ *
+ */
+
+static VALUE rb_trie_node_alloc(VALUE klass) {
+    VALUE obj;
+    obj = Data_Wrap_Struct(klass, 0, trie_state_free, NULL);
+    return obj;
+}
+
+/* nodoc */
+static VALUE rb_trie_node_initialize_copy(VALUE self, VALUE from) {
+	RDATA(self)->data = trie_state_clone(RDATA(from)->data);
+    
+    rb_iv_set(self, "@state", rb_iv_get(from, "@state"));
+    rb_iv_set(self, "@full_state", rb_iv_get(from, "@full_state"));
+
+    return self;
+}
+
+/*
+ * call-seq:
+ *   state -> single character
+ *
+ * Returns the letter that the TrieNode instance points to. So, if the node is pointing at the "e" in "monkeys", the state is "e".
+ *
+ */
 static VALUE rb_trie_node_get_state(VALUE self) {
     return rb_iv_get(self, "@state");
 }
+
+/*
+ * call-seq:
+ *   full_state -> string
+ *
+ * Returns the full string from the root of the Trie up to this node.  So if the node pointing at the "e" in "monkeys",
+ * the full_state is "monke".
+ *
+ */
 static VALUE rb_trie_node_get_full_state(VALUE self) {
     return rb_iv_get(self, "@full_state");
 }
 
+/*
+ * call-seq:
+ *   walk!(letter) -> TrieNode
+ *
+ * Tries to walk down a particular branch of the Trie.  It modifies the node it is called on.
+ *
+ */
 static VALUE rb_trie_node_walk_bang(VALUE self, VALUE rchar) {
     TrieState *state;
     Data_Get_Struct(self, TrieState, state);
@@ -256,6 +348,43 @@ static VALUE rb_trie_node_walk_bang(VALUE self, VALUE rchar) {
 		return Qnil;
 }
 
+/*
+ * call-seq:
+ *   walk(letter) -> TrieNode
+ *
+ * Tries to walk down a particular branch of the Trie.  It clones the node it is called on and 
+ * walks with that one, leaving the original unchanged.
+ *
+ */
+static VALUE rb_trie_node_walk(VALUE self, VALUE rchar) {
+	VALUE new_node = rb_funcall(self, rb_intern("dup"), 0);
+
+    TrieState *state;
+    Data_Get_Struct(new_node, TrieState, state);
+
+    if(RSTRING(rchar)->len != 1)
+		return Qnil;
+
+    Bool result = trie_state_walk(state, *RSTRING(rchar)->ptr);
+    
+    if(result) {
+		rb_iv_set(new_node, "@state", rchar);
+		VALUE full_state = rb_iv_get(new_node, "@full_state");
+		rb_str_append(full_state, rchar);
+		rb_iv_set(new_node, "@full_state", full_state);
+		return self;
+    } else
+		return Qnil;
+}
+
+/*
+ * call-seq:
+ *   value
+ *
+ * Attempts to get the value at this node of the Trie.  This only works if the node is a terminal 
+ * (i.e. end of a key), otherwise it returns nil.
+ *
+ */
 static VALUE rb_trie_node_value(VALUE self) {
     TrieState *state;
 	TrieState *dup;
@@ -267,9 +396,17 @@ static VALUE rb_trie_node_value(VALUE self) {
     TrieData trie_data = trie_state_get_data(dup);
     trie_state_free(dup);
 
-    return TRIE_DATA_ERROR == trie_data ? Qnil : INT2FIX(trie_data);
+    return TRIE_DATA_ERROR == trie_data ? Qnil : (VALUE)trie_data;
 }
 
+/*
+ * call-seq:
+ *   terminal? -> true/false
+ *
+ * Returns true if this node is at the end of a key.  So if you have two keys in your Trie, "he" and
+ * "hello", and you walk all the way to the end of "hello", the "e" and the "o" will return true for terminal?.
+ *
+ */
 static VALUE rb_trie_node_terminal(VALUE self) {
     TrieState *state;
     Data_Get_Struct(self, TrieState, state);
@@ -277,6 +414,12 @@ static VALUE rb_trie_node_terminal(VALUE self) {
     return trie_state_is_terminal(state) ? Qtrue : Qnil;
 }
 
+/*
+ * call-seq:
+ *   leaf? -> true/false
+ *
+ * Returns true if there are no branches at this node.
+ */
 static VALUE rb_trie_node_leaf(VALUE self) {
     TrieState *state;
     Data_Get_Struct(self, TrieState, state);
@@ -284,28 +427,10 @@ static VALUE rb_trie_node_leaf(VALUE self) {
     return trie_state_is_leaf(state) ? Qtrue : Qnil;
 }
 
-static VALUE rb_trie_node_clone(VALUE self) {
-    TrieState *state;
-    Data_Get_Struct(self, TrieState, state);
-    
-    VALUE new_node = rb_trie_node_alloc(cTrieNode);
-
-	TrieState *new_state = trie_state_clone(state);
-
-    RDATA(new_node)->data = new_state;
-    
-    rb_iv_set(new_node, "@state", rb_iv_get(self, "@state"));
-    rb_iv_set(new_node, "@full_state", rb_iv_get(self, "@full_state"));
-
-    return new_node;
-}
-
  
 void Init_trie() {
     cTrie = rb_define_class("Trie", rb_cObject);
     rb_define_alloc_func(cTrie, rb_trie_alloc);
-    //rb_define_method(cTrie, "initialize", rb_trie_initialize, -2);
-    //rb_define_method(cTrie, "path", rb_trie_get_path, 0);
     rb_define_method(cTrie, "has_key?", rb_trie_has_key, 1);
     rb_define_method(cTrie, "get", rb_trie_get, 1);
     rb_define_method(cTrie, "add", rb_trie_add, -2);
@@ -313,15 +438,15 @@ void Init_trie() {
     rb_define_method(cTrie, "children", rb_trie_children, 1);
     rb_define_method(cTrie, "children_with_values", rb_trie_children_with_values, 1);
     rb_define_method(cTrie, "root", rb_trie_root, 0);
-    //rb_define_method(cTrie, "save", rb_trie_save, 0);
 
     cTrieNode = rb_define_class("TrieNode", rb_cObject);
     rb_define_alloc_func(cTrieNode, rb_trie_node_alloc);
+	rb_define_method(cTrieNode, "initialize_copy", rb_trie_node_initialize_copy, 1);
     rb_define_method(cTrieNode, "state", rb_trie_node_get_state, 0);
     rb_define_method(cTrieNode, "full_state", rb_trie_node_get_full_state, 0);
     rb_define_method(cTrieNode, "walk!", rb_trie_node_walk_bang, 1);
+    rb_define_method(cTrieNode, "walk", rb_trie_node_walk, 1);
     rb_define_method(cTrieNode, "value", rb_trie_node_value, 0);
     rb_define_method(cTrieNode, "terminal?", rb_trie_node_terminal, 0);
     rb_define_method(cTrieNode, "leaf?", rb_trie_node_leaf, 0);
-    rb_define_method(cTrieNode, "clone", rb_trie_node_clone, 0);
 }

data/ext/trie/triedefs.h

@@ -8,7 +8,7 @@
 #ifndef __TRIEDEFS_H
 #define __TRIEDEFS_H
 
-#include <datrie/typedefs.h>
+#include "typedefs.h"
 
 /**
  * @file triedefs.h
@@ -60,7 +60,7 @@ typedef int32          TrieIndex;
 /**
  * @brief Type of value associated to trie entries
  */
-typedef int32          TrieData;
+typedef unsigned long TrieData;
 /**
  * @brief Trie error data
  */

data/ext/trie/typedefs.h

@@ -0,0 +1,113 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * typedefs.h - general types
+ * Created : 11 Aug 2006
+ * Author  : Theppitak Karoonboonyanan <thep@linux.thai.net>
+ */
+
+#ifndef __TYPEDEFS_H
+#define __TYPEDEFS_H
+
+#include <limits.h>
+
+typedef enum { FALSE = 0, TRUE = 1 } Bool;
+
+# if UCHAR_MAX == 0xff
+#   ifndef UINT8_TYPEDEF
+#     define UINT8_TYPEDEF
+      typedef unsigned char  uint8;
+#   endif /* UINT8_TYPEDEF */
+# endif /* UCHAR_MAX */
+
+# if SCHAR_MAX == 0x7f
+#   ifndef INT8_TYPEDEF
+#     define INT8_TYPEDEF
+      typedef signed char    int8;
+#   endif /* INT8_TYPEDEF */
+# endif /* SCHAR_MAX */
+
+# if UINT_MAX == 0xffff
+#   ifndef UINT16_TYPEDEF
+#     define UINT16_TYPEDEF
+      typedef unsigned int   uint16;
+#   endif /* UINT16_TYPEDEF */
+# endif /* UINT_MAX */
+
+# if INT_MAX == 0x7fff
+#   ifndef INT16_TYPEDEF
+#     define INT16_TYPEDEF
+      typedef int            int16;
+#   endif /* INT16_TYPEDEF */
+# endif /* INT_MAX */
+
+# if USHRT_MAX == 0xffff
+#   ifndef UINT16_TYPEDEF
+#     define UINT16_TYPEDEF
+      typedef unsigned short uint16;
+#   endif /* UINT16_TYPEDEF */
+# endif /* USHRT_MAX */
+
+# if SHRT_MAX == 0x7fff
+#   ifndef INT16_TYPEDEF
+#     define INT16_TYPEDEF
+      typedef short          int16;
+#   endif /* INT16_TYPEDEF */
+# endif /* SHRT_MAX */
+
+# if UINT_MAX == 0xffffffff
+#   ifndef UINT32_TYPEDEF
+#     define UINT32_TYPEDEF
+      typedef unsigned int   uint32;
+#   endif /* UINT32_TYPEDEF */
+# endif /* UINT_MAX */
+
+# if INT_MAX == 0x7fffffff
+#   ifndef INT32_TYPEDEF
+#     define INT32_TYPEDEF
+      typedef int            int32;
+#   endif /* INT32_TYPEDEF */
+# endif /* INT_MAX */
+
+# if ULONG_MAX == 0xffffffff
+#   ifndef UINT32_TYPEDEF
+#     define UINT32_TYPEDEF
+      typedef unsigned long  uint32;
+#   endif /* UINT32_TYPEDEF */
+# endif /* ULONG_MAX */
+
+# if LONG_MAX == 0x7fffffff
+#   ifndef INT32_TYPEDEF
+#     define INT32_TYPEDEF
+      typedef long           int32;
+#   endif /* INT32_TYPEDEF */
+# endif /* LONG_MAX */
+
+# ifndef UINT8_TYPEDEF
+#   error "uint8 type is undefined!"
+# endif
+# ifndef INT8_TYPEDEF
+#   error "int8 type is undefined!"
+# endif
+# ifndef UINT16_TYPEDEF
+#   error "uint16 type is undefined!"
+# endif
+# ifndef INT16_TYPEDEF
+#   error "int16 type is undefined!"
+# endif
+# ifndef UINT32_TYPEDEF
+#   error "uint32 type is undefined!"
+# endif
+# ifndef INT32_TYPEDEF
+#   error "int32 type is undefined!"
+# endif
+
+typedef uint8  byte;
+typedef uint16 word;
+typedef uint32 dword;
+
+
+#endif /* __TYPEDEFS_H */
+
+/*
+vi:ts=4:ai:expandtab
+*/

data/spec/trie_spec.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../ext/trie'
+require File.dirname(__FILE__) + '/../ext/trie/trie'
 
 describe Trie do
   before :each do
@@ -49,6 +49,11 @@ describe Trie do
       @trie.add('chicka', 72_000).should == true
       @trie.get('chicka').should == 72_000
     end
+
+    it 'adds a word with a non-numeric value to the trie' do
+      @trie.add('doot', 'Heeey').should == true
+      @trie.get('doot').should == 'Heeey'
+    end
   end
 
   describe :delete do
@@ -194,6 +199,18 @@ describe TrieNode do
     end
   end
 
+  describe :walk do
+    it 'returns a new node object when the walk succeeds' do
+      other = @node.walk('r')
+      other.should != @node
+    end
+
+    it 'returns nil when the walk fails' do
+      @node.walk('q').should be_nil
+    end
+  end
+
+
   describe :value do
     it 'returns nil when the node is not terminal' do
       @node.walk!('r')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tyler-trie
 version: !ruby/object:Gem::Version 
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors: 
 - Tyler McMullen
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-16 00:00:00 -07:00
+date: 2009-04-18 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -40,11 +40,19 @@ files:
 - ext/trie/trie.c
 - ext/trie/trie.h
 - ext/trie/triedefs.h
-has_rdoc: false
+- ext/trie/typedefs.h
+has_rdoc: true
 homepage: http://github.com/tyler/trie
 post_install_message: 
-rdoc_options: []
-
+rdoc_options: 
+- --title
+- Trie
+- --line-numbers
+- --op
+- rdoc
+- --main
+- ext/trie/trie.c
+- README
 require_paths: 
 - ext
 - lib