wordtree 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7bcb5a59a130a24ca1bede26a9f3b8efa651230c
-  data.tar.gz: 03ef94ffd836a11f891065fd7eb3c73d4aa8bdd2
+  metadata.gz: c57c8ec838d33d0f963f3f6316bcb165b65de21f
+  data.tar.gz: 08b411c2499a4a11a5b94cc160e57b4a24f33b96
 SHA512:
-  metadata.gz: cbf2b847f90fdffd1a52a4f7ea2c2fd78ce339ded803cbf30720fba1dced77b2ea2e9b1b84f5fee147d472d0e673437ad0e58ccffcb1a5fd2794fa87ec739384
-  data.tar.gz: 0abb233dd846913d69e13087d84c174136e8a1511c417f5239e3395972f17af71f43091079e8fc605354843c09bb73b6e82ca358801303d77f692c3d8d140ea5
+  metadata.gz: 89a3efcd77faaa357b4240f26a3f70291f9c340f33d83209417b0b12b3324b12280d14fdea786300634b0daa2d2c0c56ad013f00460799da3e82efc46691b41c
+  data.tar.gz: cf66225761d294969e41dc9e2d2d8de7d2ac518553240a40bd5bc4f72c9b488d7d64b507837a8cbf37310fa997d9b29a1d9305797ca00b3adb99eefcc35ec98b

data/ext/wordtree.cc

@@ -112,6 +112,114 @@ static VALUE text_clean(VALUE self, VALUE text) {
     return text;
 }
 
+static inline void _incr_value(
+  VALUE hash,   // Hash
+  VALUE key,    // String
+  VALUE suffix, // Symbol or nil
+  VALUE incr_existing_keys_only) // true/false
+{
+  // rb_funcall(rb_mKernel, rb_intern("p"), 4, hash, key, suffix, incr_existing_keys_only);
+  if (suffix == Qnil) {
+    // We know the hash is shallow, and has just integer values
+    VALUE val = rb_hash_lookup(hash, key);
+    if (val != Qnil) {
+      // Increment the key's value by 1
+      rb_hash_aset(hash, key, INT2FIX(FIX2INT(val) + 1));
+    } else if (!RTEST(incr_existing_keys_only)) {
+      // Add this key and start the value at 1
+      rb_hash_aset(hash, key, INT2FIX(1));
+    }
+  } else {
+    // The hash contains a hash
+    VALUE inner_hash = rb_hash_lookup(hash, key);
+    if (inner_hash != Qnil) {
+      Check_Type(inner_hash, T_HASH);
+      VALUE val = rb_hash_lookup(inner_hash, suffix);
+      if (val == Qnil) {
+        // Start this key.suffix's value at 1
+        rb_hash_aset(inner_hash, suffix, INT2FIX(1));
+      } else {
+        // Increment the key.suffix's value by 1
+        rb_hash_aset(inner_hash, suffix, INT2FIX(FIX2INT(val) + 1));
+      }
+    } else if (!RTEST(incr_existing_keys_only)) {
+      // Create an inner hash for this key (to contain suffixes)
+      inner_hash = rb_hash_new();
+      rb_hash_aset(inner_hash, suffix, INT2FIX(1));
+      // Add suffix inner_hash to this key
+      rb_hash_aset(hash, key, inner_hash);
+    }
+  }
+}
+
+VALUE text_incr_value(VALUE self, VALUE hash, VALUE key, VALUE suffix, VALUE incr_existing_keys_only)
+{
+  Check_Type(hash, T_HASH);
+  Check_Type(key, T_STRING);
+  if (suffix != Qnil) Check_Type(suffix, T_SYMBOL);
+
+  _incr_value(hash, key, suffix, incr_existing_keys_only);
+  return self;
+}
+
+VALUE text_add_ngrams_with_suffix(
+  VALUE self,
+  VALUE text,
+  VALUE hash,
+  VALUE upto_n_value,
+  VALUE suffix,
+  VALUE incr_existing_keys_only)
+{
+  char* head = RSTRING_PTR(text);
+  char* tail = RSTRING_PTR(text);
+  char* next_head = head;
+  char* next_tail = tail;
+  int word_count = 0;
+  int text_len = RSTRING_LEN(text);
+  int incr_existing = RTEST(incr_existing_keys_only);
+  int upto_n = FIX2INT(upto_n_value);
+
+  if (text_len == 0) return self;
+
+  do {
+    if (*tail == ' ' || *tail == '.' || tail >= head+text_len) {
+      word_count++;
+      if (word_count == 1 || upto_n == 1) {
+        next_head = next_tail = tail + 1;
+      } else if (word_count == 2) {
+        next_tail = tail;
+      }
+      if (word_count <= upto_n) {
+        _incr_value(hash, rb_str_new(head, tail - head), suffix, incr_existing_keys_only);
+      }
+      if (word_count == upto_n) {
+        head = next_head;
+        tail = next_tail;
+        word_count = 0;
+      } else {
+        tail++;
+      }
+    } else {
+      tail++;
+    }
+  } while(*tail);
+
+  // add the last ngram of size upto_n
+  _incr_value(hash, rb_str_new(head, tail - head), suffix, incr_existing_keys_only);
+
+  // add the 1..(upto_n-1) sized ngrams at the tail
+  if (upto_n > 1) {
+    while(head < RSTRING_PTR(text)+text_len) {
+      if(*head == ' ' || *head == '.') {
+        _incr_value(hash, rb_str_new(head + 1, tail - head - 1), suffix, incr_existing_keys_only);
+      }
+      head++;
+    }
+  }
+
+  return self;
+}
+
 extern "C"
 void Init_wordtree() {
     VALUE rb_mWordTree = rb_define_module("WordTree");
@@ -122,4 +230,6 @@ void Init_wordtree() {
 
     rb_define_module_function(rb_mText, "clean", RUBY_METHOD_FUNC(text_clean), 1);
     rb_define_module_function(rb_mText, "common_trigrams", RUBY_METHOD_FUNC(text_common_trigrams), 1);
+    rb_define_module_function(rb_mText, "incr_value", RUBY_METHOD_FUNC(text_incr_value), 4);
+    rb_define_module_function(rb_mText, "_add_ngrams_with_suffix", RUBY_METHOD_FUNC(text_add_ngrams_with_suffix), 5);
 }

data/lib/wordtree/text.rb

@@ -33,5 +33,8 @@ module WordTree
       return wrapped_output
     end
 
+    def self.add_ngrams_with_suffix(text, hash, upto_n=4, suffix=nil, incr_existing_keys_only=false)
+      _add_ngrams_with_suffix(text, hash, upto_n, suffix, incr_existing_keys_only)
+    end
   end
 end

data/lib/wordtree/version.rb

@@ -1,3 +1,3 @@
 module WordTree
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/spec/wordtree/text_spec.rb

@@ -78,4 +78,70 @@ describe WordTree::Text do
     end
   end
 
+  describe "#incr_value" do
+    context "existing keys only" do
+      it "does not add keys" do
+        hash = {"hello" => 1}
+        WordTree::Text.incr_value(hash, "goodbye", nil, true)
+        expect(hash.size).to eq 1
+        expect(hash).to_not have_key("goodbye")
+      end
+
+      it "creates suffixes to existing keys" do
+        hash = {"hello" => {}}
+        WordTree::Text.incr_value(hash, "hello", :greeting, true)
+        expect(hash.size).to eq 1
+        expect(hash["hello"]).to be_a(Hash)
+        expect(hash["hello"][:greeting]).to eq 1
+      end
+
+      it "adds values for suffixes to existing keys" do
+        hash = {"hello" => {:greeting => 1}}
+        WordTree::Text.incr_value(hash, "hello", :greeting, true)
+        WordTree::Text.incr_value(hash, "hello", :other, true)
+        expect(hash.size).to eq 1
+        expect(hash["hello"]).to eq(:greeting => 2, :other => 1)
+      end
+    end
+
+    context "open ended keys" do
+      it "adds keys" do
+        hash = {}
+        WordTree::Text.incr_value(hash, "hello", nil, false)
+        expect(hash).to eq("hello" => 1)
+      end
+
+      it "adds key and suffix" do
+        hash = {}
+        WordTree::Text.incr_value(hash, "hello", :greeting, false)
+        expect(hash).to eq("hello" => {:greeting => 1})
+      end
+    end
+  end
+
+  describe "#add_ngrams_with_suffix" do
+    it "adds ngrams to a hash" do
+      hash = {}
+      text = "some text.text"
+      WordTree::Text.add_ngrams_with_suffix(text, hash, 2)
+      expect(hash).to eq(
+        "some" => 1,
+        "text" => 2,
+        "some text" => 1,
+        "text.text" => 1)
+    end
+
+    it "adds suffixes to hash of hashes" do
+      hash = {}
+      text = "some text.text"
+      WordTree::Text.add_ngrams_with_suffix(text, hash, 1, :a)
+      WordTree::Text.add_ngrams_with_suffix(text, hash, 2, :b)
+      expect(hash).to eq(
+        "some" => {:a => 1, :b => 1},
+        "text" => {:a => 2, :b => 2},
+        "some text" => {:b => 1},
+        "text.text" => {:b => 1}
+      )
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wordtree
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Duane Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-26 00:00:00.000000000 Z
+date: 2014-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: virtus