wordtriez 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d703337bf6fdd17a5ac29691d7cc22bc0aa94c2
-  data.tar.gz: 0ae1078629cf2e13ddf365575c5b271f32de2e32
+  metadata.gz: 2cf93aec498957ae648674251b2051ab4a08381c
+  data.tar.gz: 8fe39195a015704d6afee86762d304f84d485456
 SHA512:
-  metadata.gz: 1356429e081fed8cdf8bc1cf893d9acc845e512c2a6a6bd444d180e0c9c15b2179b48aff350da11eb3f42b81f93a838d910c68c1f5218eeeb1ed19ef29917a50
-  data.tar.gz: c3eb416caf1b40b860c622300c81fae29430eeb576f1ae26bfca68189dbc3be812ee4a66f52a7e8784c3fe10cbaf5dfbd65dcce545e9873389c5ccc8b224e6fc
+  metadata.gz: 23f5c58246dec87b35d3c11476c4e8d82c6fdc013b1f534eaa8dc47881608c74a33b002dae5a2d744a223b38de59233eddf50fef8ac239a07f9fff64d03559f3
+  data.tar.gz: 1033906d59479f13f251af341905315ae637e34663cee293a785614299baf0e883715d9d61d0bab3b3e2f68e945e298c50b2861f6191c2918fd6cf308605218d

data/ext/hat-trie/text.c

@@ -20,7 +20,7 @@ size_t text_clean(char* text)
   char* read;
   char* write = text;
   uint8_t join_lines = false,
-          just_added_space = false,
+          just_added_space = true,   // prevent prefix spaces
           just_added_period = false;
   for (read=text; *read; read++) {
     char c = *read;
@@ -60,17 +60,15 @@ size_t text_clean(char* text)
   }
   // erase space at end of text
   if (just_added_space) write--;
-  // terminate the string at its new length
-  *write = '\0';
 
   // Return the new length of the string
   return (size_t)(write - text);
 }
 
-void add_ngrams(hattrie_t* trie, int upto_n, char* text, uint8_t incr_existing_keys_only)
+void add_ngrams(hattrie_t* trie, int upto_n, char* text, size_t text_len, uint8_t incr_existing_keys_only)
 {
   char blank_suffix[] = "\0";
-  add_ngrams_with_suffix(trie, upto_n, text, blank_suffix, incr_existing_keys_only);
+  add_ngrams_with_suffix(trie, upto_n, text, text_len, blank_suffix, incr_existing_keys_only);
 }
 
 static inline void incr_value(
@@ -101,7 +99,7 @@ static inline void incr_value(
 
 }
 
-void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, char* suffix, uint8_t incr_existing_keys_only)
+void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, size_t text_len, char* suffix, uint8_t incr_existing_keys_only)
 {
   char* head = text;
   char* tail = text;
@@ -109,7 +107,7 @@ void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, char* suffi
   char* next_tail = text;
   int word_count = 0;
 
-  if (*text == '\0') return;
+  if (text_len == 0) return;
 
   char buffer[NGRAM_BUFFER_SIZE];
   size_t suffix_len = strlen(suffix);
@@ -117,11 +115,8 @@ void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, char* suffi
   char* buffer_pre = buffer + buffer_offset;
   strcpy(buffer_pre, suffix);
 
-  // skip any spaces at beginning
-  while(*head == ' ') head++;
-
   do {
-    if (*tail == ' ' || *tail == '.' || *tail == '\0') {
+    if (*tail == ' ' || *tail == '.' || tail >= head+text_len) {
       word_count++;
       if (word_count == 1 || upto_n == 1) {
         next_head = next_tail = tail + 1;
@@ -152,7 +147,7 @@ void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, char* suffi
 
   // add the 1..(upto_n-1) sized ngrams at the tail
   if (upto_n > 1) {
-    while(*head) {
+    while(head < text+text_len) {
       if(*head == ' ' || *head == '.') {
         incr_value(trie, buffer, buffer_pre,
           head + 1, tail - head - 1, suffix_len,

data/ext/hat-trie/text.h

@@ -12,8 +12,8 @@ extern "C" {
 #define NGRAM_BUFFER_SIZE 4096
 
 size_t text_clean(char* text);
-void add_ngrams(hattrie_t* trie, int upto_n, char* text, uint8_t incr_existing_keys_only);
-void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, char* suffix, uint8_t incr_existing_keys_only);
+void add_ngrams(hattrie_t* trie, int upto_n, char* text, size_t text_len, uint8_t incr_existing_keys_only);
+void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, size_t text_len, char* suffix, uint8_t incr_existing_keys_only);
 
 #ifdef __cplusplus
 }

data/ext/wordtriez.cc

@@ -273,26 +273,33 @@ static VALUE hat_walk(VALUE self, VALUE key) {
     return data.arr;
 }
 
+static VALUE hat_text_clean(VALUE self, VALUE text) {
+    rb_str_modify(text);
+
+    char* ctext = StringValueCStr(text);
+    size_t new_length = text_clean(ctext);
+
+    rb_str_set_len(text, (long)new_length);
+
+    return text;
+}
+
 static VALUE hat_add_text(VALUE self, VALUE text, VALUE ngrams, VALUE suffix, VALUE incr_existing_keys_only) {
-    // rb_str_dup
     hattrie_t* p;
     HatTrie* ht;
     Data_Get_Struct(self, HatTrie, ht);
     p = ht->p;
 
-    char* ctext = StringValueCStr(text);
-    size_t new_length = text_clean(ctext);
+    hat_text_clean(self, text);
 
     add_ngrams_with_suffix(p,
         FIX2INT(ngrams),
-        ctext,
+        RSTRING_PTR(text),
+        RSTRING_LEN(text),
         StringValueCStr(suffix),
         RTEST(incr_existing_keys_only));
 
-    rb_str_resize(text, (long)new_length);
-
     return self;
-    // rb_str_substr
 }
 
 #define DEF(k,n,f,c) rb_define_method(k,n,RUBY_METHOD_FUNC(f),c)
@@ -316,4 +323,5 @@ void Init_wordtriez() {
     DEF(hat_class, "_internal_search", hat_search, 4);
     DEF(hat_class, "_internal_walk", hat_walk, 1);
     DEF(hat_class, "_internal_add_text", hat_add_text, 4);
+    DEF(hat_class, "text_clean", hat_text_clean, 1);
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wordtriez
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Zete Lui