trie_matcher 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be6709d49263fbdc6baf9dcf04766f2e44710216
-  data.tar.gz: 65705c4113f69078469d79a97967cb25ec513329
+  metadata.gz: 0c5273b23bb88d114d2f8f1aae29bd364900718b
+  data.tar.gz: ef5db432e345ee9f8be827ab36447ac15c1059b8
 SHA512:
-  metadata.gz: 61a82d7c000c5c076f240c8ea4221554006de143917a9e1e5210ab820c628ff640d88d3f70ce3a3df35497c03afd25915357fe3304707e67d145aad1abbfd1a5
-  data.tar.gz: f26bbae5d67f77d84665253312b9f1c342bc4523f8d1bbf4cd30c81f94c1e952504f3ec41faee6a67c5a1a98c039bbf6183c68ff70409234e09db24aa2f54985
+  metadata.gz: e12f179872cc28b5190ff69352991a25522d151e5b9604292756395e0017243c5faecaeacf039aa97c673fdb2c48aef6ddb04ce4b17f7ac6292f0ae645330eac
+  data.tar.gz: c23a87b8d7637a8cf9b4fc93f2af2b43fb35bcd21f40904bd91791b4b8b52263212cefe9715ac955b7ee592ed1dd6bf7cf12511288bdab84d97f181316784fd2

data/lib/trie_matcher.rb

@@ -3,11 +3,11 @@ require File.expand_path("trie_matcher/version", __dir__)
 # Trie implementation that acts as a weak mapping
 #
 # Values can be stored for a given prefix, and are returned for the longest prefix.
-# Lookup searches for longer prefixes optimistically, so saturated tries with many lexemes in them will be less efficient
+# Lookup searches based on a fixed prefix size. This can cause extra memory use and performance degredation on saturated tries with many lexemes.
 class TrieMatcher
   # Build an empty trie
   def initialize
-    @root = { nodes: {}, value: nil, longest_node_length: 0, longest_node: nil }
+    @root = { nodes: {}, value: nil, key_length: nil }
   end
 
   # Store a prefix in the trie, and associate a value with it
@@ -35,7 +35,7 @@ class TrieMatcher
     current = @root
     current_prefix = prefix
 
-    while current != nil && current_prefix != ""
+    while !current.nil? && current_prefix != ""
       previous = current
       current, current_prefix = next_node(current, current_prefix)
     end
@@ -95,58 +95,74 @@ class TrieMatcher
   end
 
   private
-  # get the node for insertion, splitting shared prefixes into subnodes if necessary
+  def insert_node(root, key)
+    new_node = {
+      nodes: {},
+      value: nil,
+      key_length: nil,
+    }
+    root[:nodes][key] = new_node
+    return new_node
+  end
+
+  # get the node for insertion, splitting intermediary nodes as necessary
   def find_canididate_insertion_node(current, key)
-    # look for a common prefix
-    current[:nodes].keys.find do |prefix|
+    if current[:key_length].nil?
+      new_node = insert_node(current, key)
+      current[:key_length] = key.length
+      return new_node, ""
+    end
+
+    # check if we have an existing shared prefix already
+    current_key = key[0...current[:key_length]]
+
+    # look for an existing key path
+    if current[:nodes].has_key?(current_key)
+      return current[:nodes][current_key], key[current_key.length..-1]
+    end
+
+    # search for a shared prefix, and split all the nodes if necessary
+    current[:nodes].keys.each do |prefix|
       common_prefix = shared_prefix(key, prefix)
       next unless common_prefix
 
-      if common_prefix == prefix
-        return current[:nodes][prefix], key[common_prefix.length..-1]
-      else
-        old = current[:nodes].delete(prefix)
-        new_suffix = prefix[common_prefix.length..-1]
-        new_node = {
-          nodes: {
-            new_suffix => old
-          },
-          value: nil,
-          longest_node_length: new_suffix.length,
-          longest_node: new_suffix,
-        }
-        current[:nodes][common_prefix] = new_node
-        if current[:longest_node] == prefix
-          longest_prefix = current[:nodes].keys.max_by(&:length)
-          current[:longest_node_length] = longest_prefix.length
-          current[:longest_node] = longest_prefix
-        end
-        return new_node, key[common_prefix.length..-1]
-      end
+      new_key_length = common_prefix.length
+
+      split_nodes(current, new_key_length)
+      return current[:nodes][common_prefix], key[new_key_length..-1]
     end
 
-    new_node = {
-      nodes: {},
-      value: nil,
-      longest_node: nil,
-      longest_node_length: 0,
-    }
-    if key.length > current[:longest_node_length]
-      current[:longest_node_length] = key.length
-      current[:longest_node] = key
+    # potentially split all other keys
+    if current_key.length < current[:key_length]
+      split_nodes(current, current_key.length)
+    end
+
+    new_node = insert_node(current, current_key)
+    return new_node, key[current_key.length..-1]
+  end
+
+  # split all the branches in the given root to the given length
+  def split_nodes(root, new_length)
+    old_nodes = root[:nodes]
+    split_length = root[:key_length] - new_length
+    root[:key_length] = new_length
+    root[:nodes] = {}
+    old_nodes.each do |key, old|
+      new_node = insert_node(root, key[0...new_length])
+      new_node[:nodes][key[new_length..-1]] = old
+      new_node[:key_length] = split_length
     end
-    current[:nodes][key] = new_node
-    return new_node, ""
   end
 
   # find the next node from the current one based on the given key
   def next_node(current, key)
-    ([key.length, current[:longest_node_length]].max).times do |l|
-      if current[:nodes].has_key?(key[0..-l-1])
-        return current[:nodes][key[0..-l-1]], key[-l,l]
-      end
+    return nil, nil unless current[:key_length]
+    next_key = key[0...current[:key_length]]
+    if current[:nodes].has_key?(next_key)
+      return current[:nodes][next_key], key[next_key.length..-1]
+    else
+      return nil, nil
     end
-    return nil, nil
   end
 
   # finds a shared prefix between the two strings, or nil if there isn't any

data/lib/trie_matcher/version.rb

@@ -1,3 +1,3 @@
 class TrieMatcher
-  VERSION = "1.3.0"
+  VERSION = "1.3.1"
 end

data/spec/ips_benchmark.rb

@@ -0,0 +1,32 @@
+require 'benchmark/ips'
+require File.expand_path('../lib/trie_matcher', __dir__)
+
+def run_user_agent_sim
+  trie = TrieMatcher.new
+  uas = <<-UAS.gsub(/^ +/, "").lines
+    Tiny Tiny RSS/1.2
+    Windows-RSS-Platform/1.0 Mozilla compatible
+    RSSOwl/2.1
+    Bloglovin/1.0 (http://www.bloglovin.com;
+    NewsBlur Feed Fetcher - (
+    g2reader-bot/1.0 (+http://www.g2reader.com;
+    Mozilla/5.0 Vienna/3.1.1
+    Mozilla/5.0 (compatible; theoldreader.com; 
+    Feedbin - 
+    Feed Wrangler/1.0 (
+    Mozilla 5.0 (compatible; BazQux/2.4 +http://bazqux.com/fetcher; 
+    livedoor FeedFetcher/0.01 (http://reader.livedoor.com/; 
+    HanRSS/1.1 (http://www.hanrss.com; 
+  UAS
+  uas.each_with_index do |line, i|
+    trie[line] = i
+  end
+  Benchmark.ips do |x|
+    x.report { uas.sample }
+    x.report { trie[uas.sample] }
+
+    x.compare!
+  end
+end
+
+run_user_agent_sim

data/spec/profiling_trie_matcher.rb

@@ -0,0 +1,66 @@
+require 'ruby-prof'
+require 'stringio'
+require File.expand_path('../lib/trie_matcher', __dir__)
+
+# RubyProf.measure_mode = RubyProf::WALL_TIME
+# RubyProf.measure_mode = RubyProf::PROCESS_TIME
+# RubyProf.measure_mode = RubyProf::CPU_TIME
+# RubyProf.measure_mode = RubyProf::ALLOCATIONS
+RubyProf.measure_mode = RubyProf::MEMORY
+# RubyProf.measure_mode = RubyProf::GC_TIME
+# RubyProf.measure_mode = RubyProf::GC_RUNS
+
+def profile
+  RubyProf.start
+  yield
+  return RubyProf.stop
+end
+
+# we investigate two use cases: user agent matching, and t9 prediction.
+# TODO: add third use case - route matching
+
+def run_user_agent_sim
+  trie = TrieMatcher.new
+  uas = <<-UAS.gsub(/^ +/, "").lines
+    Tiny Tiny RSS/1.2
+    Windows-RSS-Platform/1.0 Mozilla compatible
+    RSSOwl/2.1
+    Bloglovin/1.0 (http://www.bloglovin.com;
+    NewsBlur Feed Fetcher - (
+    g2reader-bot/1.0 (+http://www.g2reader.com;
+    Mozilla/5.0 Vienna/3.1.1
+    Mozilla/5.0 (compatible; theoldreader.com; 
+    Feedbin - 
+    Feed Wrangler/1.0 (
+    Mozilla 5.0 (compatible; BazQux/2.4 +http://bazqux.com/fetcher; 
+    livedoor FeedFetcher/0.01 (http://reader.livedoor.com/; 
+    HanRSS/1.1 (http://www.hanrss.com; 
+  UAS
+  uas.each_with_index do |line, i|
+    trie[line] = i
+  end
+  simulation = (uas.size * 10000).times.map do
+    uas.sample
+  end
+  profile do
+    simulation.each do |test_case|
+      trie[test_case]
+    end
+  end
+end
+
+def report_profile(profile, outfile = "profile.html")
+  # print the call graph
+  output = StringIO.new
+
+  printer = RubyProf::CallStackPrinter.new(profile)
+  printer.print(output, min_percent: 2)
+
+  File.write(outfile, output.string)
+
+  # Print the raw report
+  printer = RubyProf::FlatPrinter.new(profile)
+  printer.print(STDOUT)
+end
+
+report_profile(run_user_agent_sim)

data/spec/trie_matcher_spec.rb

@@ -6,7 +6,7 @@ describe TrieMatcher do
   end
 
   it 'has a version number' do
-    expect(TrieMatcher::VERSION).not_to be nil
+    expect(TrieMatcher::VERSION).not_to eq nil
   end
 
   describe "Hash access" do
@@ -17,55 +17,55 @@ describe TrieMatcher do
     it 'should retrieve stored values' do
       value = "bar"
       @t["foo"] = value
-      expect(@t["foo"]).to be value
+      expect(@t["foo"]).to eq value
     end
 
     it 'should return the stored value' do
       value = "bar"
-      expect(@t["foo"] = value).to be value
+      expect(@t["foo"] = value).to eq value
     end
 
     it 'should store values with shared prefixes' do
       @t["cat"] = 1
       @t["car"] = 2
-      expect(@t["cat"]).to be 1
-      expect(@t["car"]).to be 2
+      expect(@t["cat"]).to eq 1
+      expect(@t["car"]).to eq 2
     end
 
     it 'should store keys that are a prefix of other keys' do
       @t["catch"] = 1
       @t["cat"] = 2
-      expect(@t["catch"]).to be 1
-      expect(@t["cat"]).to be 2
+      expect(@t["catch"]).to eq 1
+      expect(@t["cat"]).to eq 2
     end
 
     it 'should store keys that have a prefix of another key' do
       @t["cat"] = 1
       @t["catch"] = 2
-      expect(@t["cat"]).to be 1
-      expect(@t["catch"]).to be 2
+      expect(@t["cat"]).to eq 1
+      expect(@t["catch"]).to eq 2
     end
 
     it 'should do prefix searching' do
       @t["cat"] = 1
-      expect(@t["cats"]).to be 1
+      expect(@t["cats"]).to eq 1
     end
 
     it 'should do partial prefix matching' do
       @t["cat"] = 1
       @t["cats in the cradle"] = 2
-      expect(@t["cats"]).to be 1
+      expect(@t["cats"]).to eq 1
     end
 
     it 'should return the more specific prefix value' do
       @t["cat"] = 1
       @t["catch"] = 2
-      expect(@t["catcher"]).to be 2
+      expect(@t["catcher"]).to eq 2
     end
 
     it 'should not give a longer prefix value' do
       @t["catch"] = 2
-      expect(@t["cat"]).to be nil
+      expect(@t["cat"]).to eq nil
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: trie_matcher
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Steven Karas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-16 00:00:00.000000000 Z
+date: 2015-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -69,6 +69,8 @@ files:
 - lib/trie_matcher.rb
 - lib/trie_matcher/pattern_matcher.rb
 - lib/trie_matcher/version.rb
+- spec/ips_benchmark.rb
+- spec/profiling_trie_matcher.rb
 - spec/spec_helper.rb
 - spec/trie_matcher_spec.rb
 - trie_matcher.gemspec
@@ -97,5 +99,7 @@ signing_key:
 specification_version: 4
 summary: Fast prefix matching
 test_files:
+- spec/ips_benchmark.rb
+- spec/profiling_trie_matcher.rb
 - spec/spec_helper.rb
 - spec/trie_matcher_spec.rb