ffi-radix_tree 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c2547ed89d9cbedb9bd340a86785aa16f0becd3c
-  data.tar.gz: 737b4483c593146c8c9fef84330f9ed5eac149ee
+  metadata.gz: b44935a7f81323aa80f656cab493c5637d3029a0
+  data.tar.gz: f2854b861c6c3507110328ce5e291f533938f97c
 SHA512:
-  metadata.gz: 79474879d50035f089b2a04b60db2dee547e9aabf4aeffa43c33b430156bac7004e55da050e35bb71cda97eede7bcda077c9ca4a53dbc66809d040a06c04d277
-  data.tar.gz: becb72bae362661101c75aed79d53c42f9d698c19116593b0180971026ba8e59eebcb712d8c65ea7e4e5ffbd71a9de6ef6dcf46b2a22141cc2e64288fdafddb9
+  metadata.gz: 45f6f4eb51181c92fedeb6fbfa577b67a73a921196edfc4ce937c251dd9a660d9ff6888c1f17f2afe67cd5d2858997ce947946588448e4d2e87810138bae4764
+  data.tar.gz: ecb205e7db9b41cf866fce8709d894eebf1b21472fd5546b2fb529dea605bd9a06494c66d83d5d1401e00b601861d72e1108dd34243320d1059595e5b1d4fcb8

data/README.md

@@ -22,7 +22,44 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+# create a new tree
+rtree = ::FFI::RadixTree::Tree.new
+
+# add key/value pairs to the tree
+rtree.push("key1", "value1")
+rtree.push("key2", ["value2", "value3"])
+rtree.push("key3", { :attr1 => "value4", :attr2 => "value5" })
+
+# work with the collection
+if rtree.has_key?("key1")
+  val = rtree.get("key2")
+  rtree.set("key3", "value6")
+end
+
+# search the tree using the keys
+
+# keys based off the root word: "manage"
+rtree.push("manage", "base verb")
+rtree.push("managed", "past tense")
+rtree.push("manager", "noun")
+rtree.push("managers", "plural noun")
+rtree.push("managing", "present tense")
+
+# Find the key that matches the _most_ of the beggining of the search term
+rtree.longest_prefix("managerial")                # returns "manager"
+rtree.longest_prefix_and_value("managerial")      # returns ["manager", "noun"]
+rtree.longest_prefix_value("managerial")          # returns "noun"
+
+# Find all values whose keys match the _most_ of the beginning of the search term
+rtree.greedy_match("managerial")                  # returns ["noun", "plural noun"]
+
+# Find all values whose keys are included _anywhere_ in the search term
+rtree.greedy_substring_match("I managed to jump") # returns ["base verb", "past tense"]
+
+# cleanup
+rtree.destroy!
+```
 
 ## Development
 

data/Rakefile

@@ -7,6 +7,67 @@ namespace :radixtree do
   task :compile do
     Rake::Task[:compile_radixtree].invoke
   end
+
+  desc "run benchmarks for radixtree lib"
+  task :benchmark do
+    require "benchmark/ips"
+    require "./lib/ffi/radix_tree"
+
+    radix_tree = ::FFI::RadixTree::Tree.new
+
+    (1..1000).each do |number|
+      radix_tree.push(number.to_s, "DERP#{number}" * (number % 10))
+      radix_tree.push(number.to_s * 100, "DERP#{number}" * (number % 10))
+      radix_tree.push(number.to_s * 1000, "DERP#{number}" * (number % 10))
+    end
+
+    ::Benchmark.ips do |x|
+      x.config(:warmup => 10)
+
+      x.report("get") do
+        radix_tree.get(rand(1000).to_s * [1, 100, 100].sample)
+      end
+
+      x.report("longest prefix") do
+        radix_tree.longest_prefix(rand(1000).to_s * [1, 100, 100].sample)
+      end
+
+      x.report("get then value") do
+        val = rand(1000).to_s * [1, 100, 100].sample
+
+        radix_tree.longest_prefix(val)
+        radix_tree.longest_prefix_value(val)
+      end
+
+      x.report("prefix and value (combined)") do
+        radix_tree.longest_prefix_and_value(rand(1000).to_s * [1, 100, 100].sample)
+      end
+
+      x.report("longest prefix (miss)") do
+        radix_tree.longest_prefix("DERP DERPIE")
+      end
+
+      x.report("prefix and value (combined/miss)") do
+        radix_tree.longest_prefix_and_value("DERP DERPIE")
+      end
+
+      x.report("greedy match") do
+        radix_tree.greedy_match(rand(1000).to_s * [1, 100, 100].sample)
+      end
+
+      x.report("greedy match (miss)") do
+        radix_tree.greedy_match("DERP DERPIE")
+      end
+
+      x.report("greedy substring match") do
+        radix_tree.greedy_substring_match(rand(1000).to_s * [1, 100, 100].sample)
+      end
+
+      x.report("greedy substring match (miss)") do
+        radix_tree.greedy_substring_match("DERP DERPIE")
+      end
+    end
+  end
 end
 
 Rake::TestTask.new(:spec) do |t|

data/ffi-radix_tree.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "msgpack"
   spec.add_dependency "ffi"
 
+  spec.add_development_dependency "benchmark-ips"
   spec.add_development_dependency "bundler", "~> 1.15"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "mocha"

data/lib/ffi/radix_tree.rb

@@ -77,9 +77,14 @@ module FFI
     attach_function :erase, [:pointer, :string], :void
     attach_function :fetch, [:pointer, :string, :pointer], :pointer
     attach_function :insert, [:pointer, :string, :pointer, :size_t], :void
+    attach_function :update, [:pointer, :string, :pointer, :size_t], :bool
     attach_function :longest_prefix, [:pointer, :string], :string
+    attach_function :longest_prefix_and_value, [:pointer, :string, :pointer, :pointer], :pointer
     attach_function :longest_prefix_value, [:pointer, :string, :pointer], :pointer
+    attach_function :greedy_match, [:pointer, :string, :pointer, :pointer], :int
+    attach_function :greedy_substring_match, [:pointer, :string, :pointer, :pointer], :int
     attach_function :match_free, [:pointer], :void
+    attach_function :multi_match_free, [:pointer, :int], :void
     attach_function :has_key, [:pointer, :string], :bool
 
     class Tree
@@ -115,6 +120,21 @@ module FFI
         push_response
       end
 
+      def push_or_update(key, value)
+        response = nil
+        @first_character_present[key[0]] = true
+        storage_data = ::MessagePack.pack(value)
+        bytesize = storage_data.bytesize
+
+        ::FFI::MemoryPointer.new(:char, bytesize, true) do |memory_buffer|
+          memory_buffer.put_bytes(0, storage_data)
+          response = ::FFI::RadixTree.update(@ptr, key, memory_buffer, bytesize)
+          response ||= ::FFI::RadixTree.insert(@ptr, key, memory_buffer, bytesize)
+        end
+
+        response
+      end
+
       def get(key)
         return nil unless @first_character_present[key[0]]
         byte_pointer = get_response = nil
@@ -122,7 +142,11 @@ module FFI
         ::FFI::MemoryPointer.new(:int) do |byte_length|
           byte_pointer = ::FFI::RadixTree.fetch(@ptr, key, byte_length)
           bytesize = byte_length.read_int
-          get_response = ::MessagePack.unpack(byte_pointer.get_bytes(0, bytesize)) if bytesize && bytesize > 0
+
+          if bytesize && bytesize > 0
+            bytes = byte_pointer.get_bytes(0, bytesize)
+            get_response = ::MessagePack.unpack(bytes)
+          end
         end
 
         get_response
@@ -139,6 +163,29 @@ module FFI
         ::FFI::RadixTree.match_free(p_out) if p_out
       end
 
+      def longest_prefix_and_value(string)
+        return [nil, nil] unless @first_character_present[string[0]]
+        byte_pointer = prefix_response = get_response = nil
+
+        ::FFI::MemoryPointer.new(:int) do |byte_length|
+          ::FFI::MemoryPointer.new(:int) do |prefix_length|
+            byte_pointer = ::FFI::RadixTree.longest_prefix_and_value(@ptr, string, byte_length, prefix_length)
+            bytesize = byte_length.read_int
+
+            if bytesize && bytesize > 0
+              prefix_size = prefix_length.read_int
+              get_response = byte_pointer.get_bytes(0, bytesize)
+              prefix_response = get_response[0..(prefix_size - 1)]
+              get_response = ::MessagePack.unpack(get_response[prefix_size..-1])
+            end
+          end
+        end
+
+        [prefix_response, get_response]
+      ensure
+        ::FFI::RadixTree.match_free(byte_pointer) if byte_pointer
+      end
+
       def longest_prefix_value(string)
         return nil unless @first_character_present[string[0]]
         byte_pointer = get_response = nil
@@ -153,6 +200,61 @@ module FFI
       ensure
         ::FFI::RadixTree.match_free(byte_pointer) if byte_pointer
       end
+
+      def greedy_match(string)
+        return [] unless @first_character_present[string[0]]
+        array_pointer = nil
+        match_sizes_pointer = nil
+        array_size = 0
+        get_response = []
+
+        ::FFI::MemoryPointer.new(:pointer) do |match_array|
+          ::FFI::MemoryPointer.new(:pointer) do |match_sizes_array|
+            array_size = ::FFI::RadixTree.greedy_match(@ptr, string, match_array, match_sizes_array)
+            if array_size > 0
+              array_sizes_pointer = match_sizes_array.read_pointer
+              match_sizes = array_sizes_pointer.get_array_of_int(0, array_size)
+              array_pointer = match_array.read_pointer
+              char_arrays = array_pointer.get_array_of_pointer(0, array_size)
+              char_arrays.each_with_index do |ptr, index|
+                get_response << ::MessagePack.unpack(ptr.get_bytes(0, match_sizes[index]))
+              end
+            end
+          end
+        end
+
+        get_response
+      ensure
+        ::FFI::RadixTree.multi_match_free(array_pointer, array_size) if array_pointer
+        ::FFI::RadixTree.match_free(match_sizes_pointer) if match_sizes_pointer
+      end
+
+      def greedy_substring_match(string)
+        array_pointer = nil
+        match_sizes_pointer = nil
+        array_size = 0
+        get_response = []
+
+        ::FFI::MemoryPointer.new(:pointer) do |match_array|
+          ::FFI::MemoryPointer.new(:pointer) do |match_sizes_array|
+            array_size = ::FFI::RadixTree.greedy_substring_match(@ptr, string, match_array, match_sizes_array)
+            if array_size > 0
+              array_sizes_pointer = match_sizes_array.read_pointer
+              match_sizes = array_sizes_pointer.get_array_of_int(0, array_size)
+              array_pointer = match_array.read_pointer
+              char_arrays = array_pointer.get_array_of_pointer(0, array_size)
+              char_arrays.each_with_index do |ptr, index|
+                get_response << ::MessagePack.unpack(ptr.get_bytes(0, match_sizes[index]))
+              end
+            end
+          end
+        end
+
+        get_response
+      ensure
+        ::FFI::RadixTree.multi_match_free(array_pointer, array_size) if array_pointer
+        ::FFI::RadixTree.match_free(match_sizes_pointer) if match_sizes_pointer
+      end
     end
   end
 end

data/lib/ffi/radix_tree/version.rb

@@ -1,5 +1,5 @@
 module FFI
   module RadixTree
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

data/vendor/radixtree/ffi_radix_tree.cpp

@@ -38,13 +38,23 @@ void match_free(const char* match) {
   }
 }
 
+void multi_match_free(const char** match, int length) {
+  if (match != NULL) {
+    for (int i=0; i<length; ++i) {
+      delete[] match[i];
+      match[i] = NULL;
+    }
+    delete[] match;
+    match = NULL;
+  }
+}
+
 const char* longest_prefix(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key) {
   std::string string_key(key);
   auto iter = map_pointer->longest_match(string_key);
 
   if (iter != map_pointer->end()) {
     char *val  = new char[iter->first.size() + 1]{0};
-    val[iter->first.size()] = '\0';
     memcpy(val, iter->first.c_str(), iter->first.size());
 
     return val;
@@ -53,6 +63,33 @@ const char* longest_prefix(radix_tree<std::string, std::vector<char>>* map_point
   return NULL;
 }
 
+const char* longest_prefix_and_value(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key, int* read_size, int* prefix_size) {
+  std::string string_key(key);
+  auto iter = map_pointer->longest_match(string_key);
+
+  if (iter != map_pointer->end()) {
+    long counter = 0;
+    int size_of_response = iter->second.size() + iter->first.size();
+    char *return_val  = new char[size_of_response]{0};
+
+    strncpy(return_val, iter->first.c_str(), iter->first.size());
+    counter = iter->first.size();
+
+    for( auto& val3 : iter->second) {
+      return_val[counter] = val3;
+      counter++;
+    }
+
+    *prefix_size = iter->first.size();
+    *read_size = size_of_response;
+    return return_val;
+  }
+
+  *prefix_size = 0;
+  *read_size = 0;
+  return NULL;
+}
+
 const char* longest_prefix_value(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key, int* read_size) {
   std::string string_key(key);
   auto iter = map_pointer->longest_match(string_key);
@@ -73,12 +110,12 @@ const char* longest_prefix_value(radix_tree<std::string, std::vector<char>>* map
   return NULL;
 }
 
-const char* fetch(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key, int* read_size) {
+unsigned char* fetch(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key, int* read_size) {
   auto iter = map_pointer->find(std::string(key));
   long counter = 0;
 
   if (iter != map_pointer->end()) {
-    char *return_val  = new char[iter->second.size()]{0};
+    unsigned char *return_val  = new unsigned char[iter->second.size()]{0};
     for( auto& val : iter->second ) {
       return_val[counter] = val;
       counter++;
@@ -92,10 +129,64 @@ const char* fetch(radix_tree<std::string, std::vector<char>>* map_pointer, const
   return NULL;
 }
 
+int greedy_match(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key, const char*** matches, int** match_sizes) {
+  std::string string_key(key);
+  typedef radix_tree<std::string, std::vector<char>>::iterator iterator;
+  std::vector<iterator> vec;
+  map_pointer->greedy_match(string_key, vec);
+  long counter = 0;
+
+  if (vec.size() > 0) {
+    *matches = new const char* [vec.size()]{nullptr};
+    *match_sizes = new int [vec.size()]{0};
+    for (auto& iter : vec) {
+      auto ret_str = new char[iter->second.size()];
+      long char_index = 0;
+      for (auto& val : iter->second) {
+        ret_str[char_index] = val;
+        ++char_index;
+      }
+      (*matches)[counter] = ret_str;
+      (*match_sizes)[counter] = iter->second.size();
+      ++counter;
+    }
+  }
+  return counter;
+}
+
+int greedy_substring_match(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key, const char*** matches, int** match_sizes) {
+  std::string string_key(key);
+  typedef radix_tree<std::string, std::vector<char>>::iterator iterator;
+  std::vector<iterator> vec;
+  map_pointer->greedy_substring_match(string_key, vec);
+  long counter = 0;
+
+  if (vec.size() > 0) {
+    *matches = new const char* [vec.size()]{nullptr};
+    *match_sizes = new int [vec.size()]{0};
+    for (auto& iter : vec) {
+      auto ret_str = new char[iter->second.size()];
+      long char_index = 0;
+      for (auto& val : iter->second) {
+        ret_str[char_index] = val;
+        ++char_index;
+      }
+      (*matches)[counter] = ret_str;
+      (*match_sizes)[counter] = iter->second.size();
+      ++counter;
+    }
+  }
+  return counter;
+}
+
 void insert(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key, char* value, size_t size) {
   map_pointer->insert({std::string(key), std::vector<char>(value, value + size)});
 }
 
+bool update(radix_tree<std::string, std::vector<char>>* map_pointer, const char* key, char* value, size_t size) {
+  return map_pointer->update({std::string(key), std::vector<char>(value, value + size)});
+}
+
 void destroy(radix_tree<std::string, std::vector<char>>* map_pointer) {
   delete map_pointer;
   map_pointer = NULL;

data/vendor/radixtree/radix_tree.hpp

@@ -67,10 +67,12 @@ public:
     iterator end();
 
     std::pair<iterator, bool> insert(const value_type &val);
+    bool update(const value_type &val);
     bool erase(const K &key);
     void erase(iterator it);
     void prefix_match(const K &key, std::vector<iterator> &vec);
     void greedy_match(const K &key,  std::vector<iterator> &vec);
+    void greedy_substring_match(const K &key, std::vector<iterator> &vec);
     iterator longest_match(const K &key);
 
     T& operator[] (const K &lhs);
@@ -81,9 +83,11 @@ private:
 
     radix_tree_node<K, T>* begin(radix_tree_node<K, T> *node);
     radix_tree_node<K, T>* find_node(const K &key, radix_tree_node<K, T> *node, int depth);
+    std::vector<radix_tree_node<K, T>*> find_leaf_nodes_with_substr(const K &key, radix_tree_node<K, T> *node, const K &ancestor_key);
     radix_tree_node<K, T>* append(radix_tree_node<K, T> *parent, const value_type &val);
     radix_tree_node<K, T>* prepend(radix_tree_node<K, T> *node, const value_type &val);
     void greedy_match(radix_tree_node<K, T> *node, std::vector<iterator> &vec);
+    void greedy_substring_match(radix_tree_node<K, T> *node, std::vector<iterator> &vec);
 
     radix_tree(const radix_tree& other); // delete
     radix_tree& operator =(const radix_tree other); // delete
@@ -232,6 +236,23 @@ void radix_tree<K, T>::greedy_match(radix_tree_node<K, T> *node, std::vector<ite
     }
 }
 
+template <typename K, typename T>
+void radix_tree<K, T>::greedy_substring_match(const K &key, std::vector<iterator> &vec)
+{
+    std::vector<radix_tree_node<K, T>*> nodes;
+
+    vec.clear();
+
+    if (m_root == NULL)
+        return;
+
+    nodes = find_leaf_nodes_with_substr(key, m_root, "");
+
+    for (auto& node : nodes) {
+        vec.push_back(node);
+    }
+}
+
 template <typename K, typename T>
 void radix_tree<K, T>::erase(iterator it)
 {
@@ -448,6 +469,22 @@ std::pair<typename radix_tree<K, T>::iterator, bool> radix_tree<K, T>::insert(co
     }
 }
 
+template <typename K, typename T>
+bool radix_tree<K, T>::update(const value_type &val)
+{
+    if (m_root == NULL)
+        return false;
+
+    radix_tree_node<K, T> *node = find_node(val.first, m_root, 0);
+
+    // if the node is a internal node, return false
+    if (! node->m_is_leaf)
+        return false;
+
+    node->m_value = new value_type(val);
+    return true;
+}
+
 template <typename K, typename T>
 typename radix_tree<K, T>::iterator radix_tree<K, T>::find(const K &key)
 {
@@ -495,6 +532,30 @@ radix_tree_node<K, T>* radix_tree<K, T>::find_node(const K &key, radix_tree_node
     return node;
 }
 
+template <typename K, typename T>
+std::vector<radix_tree_node<K, T>*> radix_tree<K, T>::find_leaf_nodes_with_substr(const K &key, radix_tree_node<K, T> *node, const K &ancestor_key)
+{
+    std::vector<radix_tree_node<K, T>*> nodes;
+
+    if (node->m_children.empty())
+        return nodes;
+
+    typename radix_tree_node<K, T>::it_child it;
+    for (it = node->m_children.begin(); it != node->m_children.end(); ++it) {
+        if (it->second->m_is_leaf) {
+            nodes.push_back(it->second);
+        } else {
+            auto child_key = ancestor_key + it->first;
+            if (key.find(child_key) != std::string::npos ) {
+                auto found_nodes = find_leaf_nodes_with_substr(key, it->second, child_key);
+                nodes.insert(nodes.end(), found_nodes.begin(), found_nodes.end());
+            }
+        }
+    }
+
+    return nodes;
+}
+
 /*
 
 (root)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ffi-radix_tree
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Brandon Dewitt
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-23 00:00:00.000000000 Z
+date: 2018-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement