wordtriez 0.1.3 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c086defc8f9d26af3174ab0fe8f6abeec00add65
-  data.tar.gz: 1093097d455ed7b082c7da626c923a221e41853d
+  metadata.gz: 1166b687e599b75c7c6423e177e661e6e01557fc
+  data.tar.gz: 7157ecc077b1be637ceb067bae8e170acef367bf
 SHA512:
-  metadata.gz: 09bbe2b50bda31e775d17906b4a448e505e8f676bfb84e37d36186afaba706d6e3de0567e49f0ab37440eabcfa6a44176c1ee3a6ba173287022849901a1adeef
-  data.tar.gz: 46f914ec077291c2f5d7b749799f0179ab42208894622da5fcb4dde360c992082faa747faec6a56a949d735bfbd78f0c6c21ec7e653c253832479ec5b9ec724a
+  metadata.gz: a674980638b76c5ad7893897d945986b56634b49055b807cded3bc367a05917a207f7ea92a731692a74a4335580b758ac31398466654b01a3a3e291fe13fea1c
+  data.tar.gz: 4c113c405f97f8033a11e0f501c4e3043466ade43717dc762347c5283d5064a00096b1b27f578940ee4a97978a72977a79651755a81c3342c2c5b6059b31cb2c

data/ext/hat-trie/text.c

@@ -15,16 +15,16 @@
  *
  * Spaces indicate word boundaries, while periods indicate sentence boundaries.
  */
-size_t text_clean(char* text)
+size_t text_clean(char* text, size_t len)
 {
-  if (*text == '\0') return 0;
+  if (len == 0) return 0;
 
   char* read;
   char* write = text;
   uint8_t join_lines = false,
           just_added_space = true,   // prevent prefix spaces
           just_added_period = false;
-  for (read=text; *read; read++) {
+  for (read=text; read<text+len; read++) {
     char c = *read;
     if (c >= 'A' && c <= 'Z') {
       // Change upper case to lowercase
@@ -70,7 +70,7 @@ size_t text_clean(char* text)
 void add_ngrams(hattrie_t* trie, int upto_n, char* text, size_t text_len, uint8_t incr_existing_keys_only)
 {
   char blank_suffix[] = "\0";
-  add_ngrams_with_suffix(trie, upto_n, text, text_len, blank_suffix, incr_existing_keys_only);
+  add_ngrams_with_suffix(trie, upto_n, text, text_len, blank_suffix, 0, incr_existing_keys_only);
 }
 
 static inline void incr_value(
@@ -101,7 +101,7 @@ static inline void incr_value(
 
 }
 
-void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, size_t text_len, char* suffix, uint8_t incr_existing_keys_only)
+void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, size_t text_len, char* suffix, size_t suffix_len, uint8_t incr_existing_keys_only)
 {
   char* head = text;
   char* tail = text;
@@ -112,10 +112,9 @@ void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, size_t text
   if (text_len == 0) return;
 
   char buffer[NGRAM_BUFFER_SIZE];
-  size_t suffix_len = strlen(suffix);
   size_t buffer_offset = NGRAM_BUFFER_SIZE - suffix_len - 1;
   char* buffer_pre = buffer + buffer_offset;
-  strcpy(buffer_pre, suffix);
+  memcpy(buffer_pre, suffix, suffix_len);
 
   do {
     if (*tail == ' ' || *tail == '.' || tail >= head+text_len) {

data/ext/hat-trie/text.h

@@ -11,9 +11,9 @@ extern "C" {
 
 #define NGRAM_BUFFER_SIZE 4096
 
-size_t text_clean(char* text);
+size_t text_clean(char* text, size_t len);
 void add_ngrams(hattrie_t* trie, int upto_n, char* text, size_t text_len, uint8_t incr_existing_keys_only);
-void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, size_t text_len, char* suffix, uint8_t incr_existing_keys_only);
+void add_ngrams_with_suffix(hattrie_t* trie, int upto_n, char* text, size_t text_len, char* suffix, size_t suffix_len, uint8_t incr_existing_keys_only);
 
 #ifdef __cplusplus
 }

data/ext/wordtriez.cc

@@ -276,8 +276,7 @@ static VALUE hat_walk(VALUE self, VALUE key) {
 static VALUE hat_text_clean(VALUE self, VALUE text) {
     rb_str_modify(text);
 
-    char* ctext = StringValueCStr(text);
-    size_t new_length = text_clean(ctext);
+    size_t new_length = text_clean(RSTRING_PTR(text), RSTRING_LEN(text));
 
     rb_str_set_len(text, (long)new_length);
 
@@ -296,7 +295,8 @@ static VALUE hat_add_text(VALUE self, VALUE text, VALUE ngrams, VALUE suffix, VA
         FIX2INT(ngrams),
         RSTRING_PTR(text),
         RSTRING_LEN(text),
-        StringValueCStr(suffix),
+        RSTRING_PTR(suffix),
+        RSTRING_LEN(suffix),
         RTEST(incr_existing_keys_only));
 
     return self;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wordtriez
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.6
 platform: ruby
 authors:
 - Zete Lui
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-25 00:00:00.000000000 Z
+date: 2014-09-28 00:00:00.000000000 Z
 dependencies: []
 description: fast, efficient, unicode aware HAT trie with prefix / suffix support.
 email: