multi_string_replace 0.1.0 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 212107102a63c46a717026fbc8c853da01204b73cfda68fa7ff58909d72cade7
-  data.tar.gz: 59cafa3a2e9c934e4824a61f87372524b765ba52c4ddadda1283b42f52d4860a
+  metadata.gz: '03680f4522d98a7a162df95c1151feedabe821febc15a03c7a98a6996dd43cc1'
+  data.tar.gz: 80222e5675b3310fd079e6099026f8fa584d77542a0a34a30a3c6c3c67643cdb
 SHA512:
-  metadata.gz: '073753252667508ac0006fe6b6b9d88d5890aaea4a102a94622dcc1e8d12dee9f3760fa1c98ca8ee343b289b474e6e0a89e8cd81379aa618b11d9096154d50e5'
-  data.tar.gz: c18f648b5a010e926f8d79b219fc67ccd96dd76fe213003988e265c16279e24d0117678f12526be5c54021065da175101546d5fccc88e3f84a3a2dbeee6f7d15
+  metadata.gz: 6bcda8e829e5fd9e747c567a44c09363f304bbc8162d661418fb491f6626019c158d8a60345bb814cab42598c3830d884e6b8b70b20215d9614cb36d6913928e
+  data.tar.gz: 3f2895d64ba1d7560e866104f499a592767fa1dd161c1ff3d975126091d8330f06e85e25f37182de46f05c5dcb51a3f8ed4e38b352b78c8a1955dde1fdca5396

data/.gitignore

@@ -10,6 +10,7 @@
 *.o
 *.log
 *.so
+*.gem
 
 extconf.h
 Makefile

data/Gemfile.lock

@@ -1,22 +1,13 @@
 PATH
   remote: .
   specs:
-    multi_string_replace (0.1.0)
+    multi_string_replace (1.0.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    byebug (10.0.2)
-    coderay (1.1.2)
     diff-lcs (1.3)
-    method_source (0.9.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    pry-byebug (3.6.0)
-      byebug (~> 10.0)
-      pry (~> 0.10)
-    rake (10.5.0)
+    rake (13.0.1)
     rake-compiler (1.0.5)
       rake
     rspec (3.8.0)
@@ -39,10 +30,9 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.16)
   multi_string_replace!
-  pry-byebug
-  rake (~> 10.0)
+  rake
   rake-compiler
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.16.2
+   1.16.5

data/LICENSE.txt

@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.

data/README.md

@@ -1,8 +1,15 @@
+
+[![Gem](https://img.shields.io/gem/v/multi_string_replace.svg)](https://rubygems.org/gems/multi_string_replace)
+
+
 # MultiStringReplace
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/multi_string_replace`. To experiment with that code, run `bin/console` for an interactive prompt.
+A fast multiple string replace library for ruby. Uses a C implementation of the Aho–Corasick Algorithm based
+on https://github.com/morenice/ahocorasick while adding support for a few performance enhancements and on the
+fly multiple string replacement.
 
-TODO: Delete this and the text above, and describe your gem
+If Regex is not needed, this library offers significant performance advantages over String.gsub() for large string
+and with a large number of tokens.
 
 ## Installation
 
@@ -22,7 +29,37 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+MultiStringReplace.match("The quick brown fox jumps over the lazy dog brown", ['brown', 'fox'])
+# { 0 => [10, 44], 1 => [16] }
+MultiStringReplace.replace("The quick brown fox jumps over the lazy dog brown", {'brown' => 'black', 'fox' => 'wolf'})
+# The quick black wolf jumps over the lazy dog black
+```
+
+You can also pass in a Proc, these will only get evaluated when the token is encountered.
+
+```ruby
+MultiStringReplace.replace("The quick brown fox jumps over the lazy dog brown", {'brown' => 'black', 'fox' => ->() { "cat" }})
+```
+
+Also adds a mreplace method to String which does the same thing:
+
+```ruby
+"The quick brown fox jumps over the lazy dog brown".mreplace({'brown' => 'black', 'fox' => ->() { "cat" }})
+```
+
+## Performance
+
+Performing token replacement on a 200K text file repeated 100 times
+
+```
+                         user     system      total        real
+multi gsub           1.322510   0.000000   1.322510 (  1.344405)
+MultiStringReplace   0.196823   0.007979   0.204802 (  0.207219)
+mreplace             0.200593   0.004031   0.204624 (  0.205379)
+```
+
+Benchmark sources can be found here: <https://github.com/jedld/multi_word_replace/blob/master/bin/benchmark.rb>
 
 ## Development
 

data/bin/benchmark.rb

@@ -1,7 +1,6 @@
 require "bundler/setup"
 require 'multi_string_replace'
 require 'benchmark'
-require 'pry-byebug'
 
 class String
   def mgsub(key_value_pairs=[].freeze)
@@ -27,12 +26,14 @@ replace = {
   'Cras' => 'uuuuuuuu',
   'nunc' => 'eeeeeee',
   'cursus' => '乧乨乩乪乫乬乭乮乯买乱乲乳乴乵乶乷乸乹乺乻乼乽乾乿',
+  'Vivamus' => '㐀㐁㐂㐃㐄㐅㐆㐇㐈㐉㐊㐋'
 }
 
 File.write('replaced.txt', body.gsub(/(#{replace.keys.join('|')})/, replace))
 File.write('replaced2.txt', MultiStringReplace.replace(body, replace))
 
 Benchmark.bmbm do |x|
-  x.report "multi gsub" do body.mgsub(replace.map { |k, v| [/#{k}/, v] } ) end
-  x.report "MultiStringReplace" do MultiStringReplace.replace(body, replace) end
+  x.report "multi gsub" do 100.times { body.mgsub(replace.map { |k, v| [/#{k}/, v] } ) } end
+  x.report "MultiStringReplace" do 100.times { MultiStringReplace.replace(body, replace) } end
+  x.report "mreplace" do 100.times { body.mreplace(replace) } end
 end

data/ext/multi_string_replace/aho_trie.c

@@ -29,60 +29,48 @@ bool aho_add_trie_node(struct aho_trie * restrict t, struct aho_text_t * restric
 
     for (int text_idx = 0; text_idx < text->len; text_idx++)
     {
-        unsigned char node_text = text->text[text_idx];
+        unsigned int node_text = text->text[text_idx];
         bool find_node = false;
         int child_idx = 0;
 
         if (travasal_node->child_count == 0)
         {
             /* insert first node to child_list */
-            travasal_node->child_list[0] =
-                         (struct aho_trie_node*) malloc(sizeof(struct aho_trie_node));
-            travasal_node->child_count++;
+            struct aho_trie_node* child = (struct aho_trie_node*) malloc(sizeof(struct aho_trie_node));
 
-            __aho_trie_node_init(travasal_node->child_list[0]);
-            travasal_node->child_list[0]->text = node_text;
-            travasal_node->child_list[0]->parent = travasal_node;
+            travasal_node->child_list[node_text] = child;
+            travasal_node->first_child = child;
+            travasal_node->last_child = child;
+            travasal_node->child_count++;
 
-            travasal_node = travasal_node->child_list[0];
+            __aho_trie_node_init(child);
+            child->text = node_text;
+            child->parent = travasal_node;
+            child->failure_link = &(t->root);
+            travasal_node = child;
             continue;
         }
 
-        if (travasal_node->child_count == MAX_AHO_CHILD_NODE)
-        {
-            return false;
-        }
-
-        for (child_idx=0; child_idx < travasal_node->child_count; child_idx++)
-        {
-            if (travasal_node->child_list[child_idx]->text == node_text )
-            {
-                find_node = true;
-                break;
-            }
-        }
-
-        if (find_node == true)
+        if ( travasal_node->child_list[node_text] != NULL)
         {
-            travasal_node->child_list[child_idx]->ref_count++;
-            travasal_node = travasal_node->child_list[child_idx];
+            travasal_node->child_list[node_text]->ref_count++;
+            travasal_node = travasal_node->child_list[node_text];
         }
         else
         {
             /* push_back to child_list */
-            struct aho_trie_node* child_node = NULL;
-
-            travasal_node->child_list[travasal_node->child_count] =
-                         (struct aho_trie_node*) malloc(sizeof(struct aho_trie_node));
+            struct aho_trie_node* child =  (struct aho_trie_node*) malloc(sizeof(struct aho_trie_node));
 
-            child_node = travasal_node->child_list[travasal_node->child_count];
+            travasal_node->child_list[node_text] = child;
             travasal_node->child_count++;
-
-            __aho_trie_node_init(child_node);
-            child_node->text = node_text;
-            child_node->parent = travasal_node;
-
-            travasal_node = child_node;
+            travasal_node->last_child->next = child;
+            travasal_node->last_child = child;
+
+            __aho_trie_node_init(child);
+            child->text = node_text;
+            child->parent = travasal_node;
+            child->failure_link = &(t->root);
+            travasal_node = child;
         }
     }
 
@@ -99,7 +87,6 @@ bool __aho_connect_link(struct aho_trie_node* p, struct aho_trie_node* q)
 {
     struct aho_trie_node *pf = NULL;
     int i = 0;
-
     /* is root node */
     if (p->parent == NULL)
     {
@@ -108,26 +95,26 @@ bool __aho_connect_link(struct aho_trie_node* p, struct aho_trie_node* q)
     }
 
     pf = p->failure_link;
-    for (i=0; i < pf->child_count; i++)
+
+    /* check child node of failure link(p) */
+    if (pf->child_list[q->text] != NULL)
     {
-        /* check child node of failure link(p) */
-        if (pf->child_list[i]->text == q->text )
-        {
-            /* connect failure link */
-            q->failure_link = pf->child_list[i];
+        struct aho_trie_node *node = pf->child_list[q->text];
+        /* connect failure link */
+        q->failure_link =node;
 
-            /* connect output link */
-            if (pf->child_list[i]->text_end)
-            {
-                q->output_link = pf->child_list[i];
-            }
-            else
-            {
-                q->output_link = pf->child_list[i]->output_link;
-            }
-            return true;
+        /* connect output link */
+        if (node->text_end)
+        {
+            q->output_link = node;
         }
+        else
+        {
+            q->output_link = node->output_link;
+        }
+        return true;
     }
+
     return false;
 }
 
@@ -158,20 +145,20 @@ void aho_connect_link(struct aho_trie * restrict t)
         free(queue_node);
 
         /* get child node list of p */
-        for (i=0; i < p->child_count; i++)
+        struct aho_trie_node *child_ptr = p->first_child;
+        while (child_ptr != NULL)
         {
             struct aho_trie_node *pf = p;
-
-            aho_queue_enqueue(&queue, p->child_list[i]);
-            q = p->child_list[i];
+            aho_queue_enqueue(&queue, child_ptr);
+            q = child_ptr;
 
             while (__aho_connect_link(pf, q) == false)
             {
                 pf = pf->failure_link;
             }
+            child_ptr = child_ptr->next;
         }
     }
-
     aho_queue_destroy(&queue);
 }
 
@@ -180,7 +167,6 @@ void aho_clean_trie_node(struct aho_trie * restrict t)
     struct aho_queue queue;
     aho_queue_init(&queue);
     aho_queue_enqueue(&queue, &(t->root));
-
     /* BFS */
     while (true)
     {
@@ -197,9 +183,11 @@ void aho_clean_trie_node(struct aho_trie * restrict t)
         remove_node = queue_node->data;
         free(queue_node);
 
-        for (i=0; i < remove_node->child_count; i++)
+        struct aho_trie_node *child_ptr = remove_node->first_child;
+        while (child_ptr != NULL)
         {
-            aho_queue_enqueue(&queue, remove_node->child_list[i]);
+            aho_queue_enqueue(&queue, child_ptr);
+            child_ptr = child_ptr->next;
         }
 
         /* is root node */
@@ -218,14 +206,12 @@ bool __aho_find_trie_node(struct aho_trie_node** restrict start, const unsigned
     int i = 0;
 
     search_node = *start;
-    for (i = 0; i < search_node->child_count; i++)
+
+    if (search_node->child_list[(unsigned int)text] != NULL)
     {
-        if (search_node->child_list[i]->text == text)
-        {
-            /* find it! move to find child node! */
-            *start = search_node->child_list[i];
-            return true;
-        }
+        /* find it! move to find child node! */
+        *start = search_node->child_list[(unsigned int)text];
+        return true;
     }
 
     /* not found */
@@ -243,7 +229,6 @@ struct aho_text_t* aho_find_trie_node(struct aho_trie_node** restrict start, con
         {
             return NULL;
         }
-
         /* retry find. move failure link. */
         *start = (*start)->failure_link;
     }
@@ -260,7 +245,6 @@ struct aho_text_t* aho_find_trie_node(struct aho_trie_node** restrict start, con
     {
         return (*start)->output_link->output_text;
     }
-
     /* keep going */
     return NULL;
 }
@@ -286,10 +270,11 @@ void aho_print_trie(struct aho_trie * restrict t)
 
         travasal_node = queue_node->data;
         free(queue_node);
-
-        for (i=0; i < travasal_node->child_count; i++)
+        struct aho_trie_node *child_ptr = travasal_node->first_child;
+        while (child_ptr != NULL)
         {
-            aho_queue_enqueue(&queue, travasal_node->child_list[i]);
+            aho_queue_enqueue(&queue, child_ptr);
+            child_ptr = child_ptr->next;
         }
 
         /* is root node */

data/ext/multi_string_replace/aho_trie.h

@@ -11,6 +11,9 @@ struct aho_trie_node
 
     struct aho_trie_node* parent;
     struct aho_trie_node* child_list[MAX_AHO_CHILD_NODE];
+    struct aho_trie_node* first_child;
+    struct aho_trie_node* last_child;
+    struct aho_trie_node* next;
     unsigned int child_count;
 
     bool text_end;

data/ext/multi_string_replace/ahocorasick.c

@@ -1,3 +1,15 @@
+
+// MIT License
+
+// Copyright (c) 2017 morenice
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
 #include <limits.h>
 #include <string.h>
 #include <stdlib.h>
@@ -20,6 +32,7 @@ void aho_destroy(struct ahocorasick * restrict aho)
 int aho_add_match_text(struct ahocorasick * restrict aho, const char* text, unsigned int len)
 {
     struct aho_text_t* a_text = NULL;
+
     if (aho->accumulate_text_id == AHO_MAX_TEXT_ID)
     {
         return -1;
@@ -30,11 +43,14 @@ int aho_add_match_text(struct ahocorasick * restrict aho, const char* text, unsi
         goto lack_free_mem;
 
     a_text->text = (char*) malloc(sizeof(char)*len);
+
     if (!a_text->text)
         goto lack_free_mem;
 
     a_text->id = aho->accumulate_text_id++;
+
     memcpy(a_text->text, text, len);
+
     a_text->len = len;
     a_text->prev = NULL;
     a_text->next = NULL;
@@ -54,6 +70,7 @@ int aho_add_match_text(struct ahocorasick * restrict aho, const char* text, unsi
     return a_text->id;
 
 lack_free_mem:
+    
     return -1;
 }
 
@@ -110,7 +127,9 @@ void aho_create_trie(struct ahocorasick * restrict aho)
 
     for (iter = aho->text_list_head; iter != NULL; iter = iter->next)
     {
+
         aho_add_trie_node(&(aho->trie), iter);
+
     }
 
     aho_connect_link(&(aho->trie));
@@ -162,7 +181,8 @@ unsigned int aho_findtext(struct ahocorasick * restrict aho, const char* data, u
     return match_count;
 }
 
-VALUE aho_replace_text(struct ahocorasick * restrict aho, const char* data, unsigned long long data_len, char *values[], VALUE ruby_values[])
+VALUE aho_replace_text(struct ahocorasick * restrict aho, const char* data,
+    unsigned long long data_len, char *values[], long value_sizes[], VALUE ruby_values[])
 {
     int i = 0;
     int match_count = 0;
@@ -191,14 +211,18 @@ VALUE aho_replace_text(struct ahocorasick * restrict aho, const char* data, unsi
         }
 
         // concatenate from last_concat_pos
-        rb_str_cat(main_result, &data[last_concat_pos], pos - last_concat_pos);
+        if (pos > last_concat_pos) {
+            rb_str_cat(main_result, &data[last_concat_pos], pos - last_concat_pos);
+        }
+
         // concatenate replace
         if (values[result->id] == NULL) {
             VALUE proc_result = rb_funcall(ruby_values[result->id], rb_intern("call"), 0);
-            values[result->id] = StringValueCStr(proc_result);
+            value_sizes[result->id] = RSTRING_LEN(proc_result);
+            values[result->id] = StringValuePtr(proc_result);
         }
 
-        rb_str_cat2(main_result, values[result->id]); 
+        rb_str_cat(main_result, values[result->id], value_sizes[result->id]); 
         last_concat_pos = i + 1;
     }