bk 0.0.1

data/Gemfile

@@ -0,0 +1,8 @@
+# A sample Gemfile
+source "http://rubygems.org"
+
+gem "text"
+
+group :test do
+  gem "test-unit"
+end

data/README.md

@@ -0,0 +1,77 @@
+# BK-Tree implementation in Ruby
+
+If you don’t know what a BK-tree is, these links should provide a good explanation and introduction.
+
+* [Damn Cool Algorithms, Part 1: BK-Trees](http://blog.notdot.net/2007/4/Damn-Cool-Algorithms-Part-1-BK-Trees)
+* [Fast Approximate String Matching in a Dictionary](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.21.3317)
+
+## Usage
+
+    require "bk"
+    tree = BK::Tree.new # Use the default Levenshtein distance algorithm
+
+Add items to the tree:
+
+    tree.add "cat"
+    tree.add "dog"
+    tree.add "monkey"
+    tree.add "donkey"
+
+Find all items within distance 1 of ‘munkey’:
+
+    tree.query("munkey", 1)
+    # => {"monkey"=>1} 
+
+Find all items within distance 2 of ‘munkey’:
+
+    tree.query("munkey", 2)
+    # => {"donkey"=>2, "monkey"=>1}
+
+You can specify a custom distance algorithm by passing an object that responds
+to `call(a, b)` with a number:
+
+    custom_algorithm = lambda{ |a, b|
+      Text::Levenshtein.distance(a, b)
+    }
+
+    tree = BK::Tree.new(custom_algorithm)
+
+Note that the result *must* satisfy the
+_triangle inequality_, i.e. _d(x,z) ≤ d(x,y) + d(y,z)_.
+
+The precomputed tree can be exported to and reimported later from an IO-like object:
+
+    File.open("tree", "wb") do |f|
+      tree.export(f)
+    end
+
+    File.open("tree", "rb") do |f|
+      tree = BK::Tree.import(f)
+    end
+
+## Dependencies
+
+* [text](http://rubygems.org/gems/text) version 0.2.0 or newer.
+
+## Performance
+
+Results of looking for words within distance 1 of ‘alien’ in a 20,000-word dictionary:
+
+    Loading 20000 words from dictionary ... 0.273s
+    Building tree ... 57.331s
+    Linear scan to find expected terms ... 5.711s
+    Query tree ... 0.133s
+    2.1% of tree was queried
+
+This means that the BK-tree is about 40 times as fast as a linear search,
+although building the initial tree took 10 times as long as a linear search.
+
+As the threshold increases, the benefit is reduced. At threshold 3:
+
+    Query tree ... 3.368s
+    62.9% of tree was queried
+
+## Limitations
+
+* Memory usage: around 6 MB for a 20,000-word tree.
+* Maximum tree depth is limited by the stack.

data/lib/bk.rb

@@ -0,0 +1,75 @@
+require 'text/levenshtein'
+require 'yaml'
+
+module BK
+  # Paul Battley 2007
+  # See http://blog.notdot.net/archives/30-Damn-Cool-Algorithms,-Part-1-BK-Trees.html
+  # and http://www.dcc.uchile.cl/~gnavarro/ps/spire98.2.ps.gz
+
+  class LevenshteinDistancer
+    def call(a, b)
+      Text::Levenshtein.distance(a, b)
+    end
+  end
+
+  class Node
+    attr_reader :term, :children
+
+    def initialize(term, distancer)
+      @term = term
+      @children = {}
+      @distancer = distancer
+    end
+
+    def add(term)
+      score = distance(term)
+      if child = children[score]
+        child.add(term)
+      else
+        children[score] = Node.new(term, @distancer)
+      end
+    end
+
+    def query(term, threshold, collected)
+      distance_at_node = distance(term)
+      collected[self.term] = distance_at_node if distance_at_node <= threshold
+      ((distance_at_node-threshold)..(threshold+distance_at_node)).each do |score|
+        child = children[score]
+        child.query(term, threshold, collected) if child
+      end
+    end
+
+    def distance(term)
+      @distancer.call(term, self.term)
+    end
+  end
+
+  class Tree
+    def initialize(distancer = LevenshteinDistancer.new)
+      @root = nil
+      @distancer = distancer
+    end
+
+    def add(term)
+      if @root
+        @root.add(term)
+      else
+        @root = Node.new(term, @distancer)
+      end
+    end
+
+    def query(term, threshold)
+      collected = {}
+      @root.query(term, threshold, collected)
+      return collected
+    end
+
+    def export(stream)
+      stream.write(YAML.dump(self))
+    end
+
+    def self.import(stream)
+      YAML.load(stream.read)
+    end
+  end
+end

data/lib/bk/dot_graph.rb

@@ -0,0 +1,23 @@
+require 'bk'
+
+module BK
+  module DotGraphable
+    def graph
+      %{"#{term}" [label = "#{term}"]\n} +
+      children.sort_by{ |distance, child| distance }.map{ |distance, child|
+        child.graph +
+        %{edge [label = "#{distance}"]\n"#{term}" -> "#{child.term}"\n}
+      }.join
+    end
+  end
+
+  class Node
+    include DotGraphable
+  end
+
+  class Tree
+    def dot_graph
+      ["digraph G {", @root.graph, "}"].join("\n")
+    end
+  end
+end

data/lib/bk/dump.rb

@@ -0,0 +1,23 @@
+require 'bk'
+
+module BK
+  module Dumpable
+    def dump
+      if children.any?
+        [term, children.inject({}){ |h,(score,child)| h[score] = child.dump; h }]
+      else
+        [term]
+      end
+    end
+  end
+
+  class Node
+    include Dumpable
+  end
+
+  class Tree
+    def dump
+      @root ? @root.dump : []
+    end
+  end
+end

data/lib/bk/version.rb

@@ -0,0 +1,3 @@
+module BK
+  VERSION = '0.0.1'
+end

data/samples/graph.rb

@@ -0,0 +1,11 @@
+$:.unshift( File.join( File.dirname(__FILE__), '..', 'lib' ))
+require 'bk'
+require 'bk/dot_graph'
+
+tree = BK::Tree.new
+$stdin.each_with_index do |line, i|
+  tree.add(line.strip)
+  File.open('bk-%04d.dot' % i, 'w') do |io|
+    io << tree.dot_graph
+  end
+end

data/samples/performance.rb

@@ -0,0 +1,63 @@
+$:.unshift( File.join( File.dirname(__FILE__), '..', 'lib' ))
+require 'bk'
+
+class CountingLevenshteinDistancer < BK::LevenshteinDistancer
+  attr_reader :count
+
+  def initialize
+    @count = 0
+    @counting = false
+  end
+
+  def call(a, b)
+    @count += 1 if @counting
+    super
+  end
+
+  def start_counting
+    @counting = true
+  end
+end
+
+def time(message)
+  t0 = Time.now
+  print "#{message} ... "
+  $stdout.flush
+  retval = yield
+  puts "%0.3fs" % [Time.now - t0]
+  return retval
+end
+
+search_term = 'alien'
+threshold = 1
+distancer = CountingLevenshteinDistancer.new
+
+terms = time('Loading 10 K words from dictionary'){
+  File.read('/usr/share/dict/words').scan(/\w+/)[0, 10000]
+}
+
+tree = time('Building tree'){
+  tree = BK::Tree.new(distancer)
+  terms.each do |term|
+    tree.add(term)
+  end
+  tree
+}
+
+expected = time('Linear scan to find expected terms'){
+  terms.inject({}){ |acc, t|
+    d = Text::Levenshtein.distance(t, search_term)
+    acc[t] = d if d <= threshold
+    acc
+  }
+}
+
+distancer.start_counting
+
+actual = time('Query tree'){
+  tree.query(search_term, threshold)
+}
+
+raise 'Results of linear and tree scan differ' unless expected == actual
+
+puts '%0.1f%% of tree was queried' % [(distancer.count * 100.0) / terms.length]

data/test/test_all.rb

@@ -0,0 +1,3 @@
+Dir[ File.join( File.dirname(__FILE__), '*.rb' )].each do |f|
+  require f
+end

data/test/test_building_tree.rb

@@ -0,0 +1,66 @@
+$:.unshift( File.join( File.dirname(__FILE__), '..', 'lib' ))
+require 'test/unit'
+require 'bk'
+require 'bk/dump'
+
+class BKTreeBuildingWhiteBoxTest < Test::Unit::TestCase
+  attr_reader :tree
+
+  def setup
+    @tree = BK::Tree.new
+  end
+
+  def test_should_build_root
+    tree.add('book')
+    assert_equal ['book'], tree.dump
+  end
+
+  def test_should_add_one_term
+    tree.add('book')
+    tree.add('rook')
+    assert_equal(
+      [ 'book', {
+        1 => [ 'rook' ]}],
+      tree.dump
+    )
+  end
+
+  def test_should_add_second_term
+    %w[ book rook nooks ].each do |word|
+      tree.add(word)
+    end
+    assert_equal(
+      [ 'book', {
+        1 => [ 'rook' ],
+        2 => [ 'nooks' ]}],
+      tree.dump
+    )
+  end
+
+  def test_should_add_third_term
+    %w[ book rook nooks boon ].each do |word|
+      tree.add(word)
+    end
+    assert_equal(
+      [ 'book', {
+        1 => [ 'rook', {
+          2 => [ 'boon' ]}],
+        2 => [ 'nooks' ]}],
+      tree.dump
+    )
+  end
+
+  def test_should_add_fourth_term
+    %w[ book rook nooks boon boot ].each do |word|
+      tree.add(word)
+    end
+    assert_equal(
+      [ 'book', {
+        1 => [ 'rook', {
+          2 => [ 'boon', {
+            1 => [ 'boot' ]}]}],
+        2 => [ 'nooks' ]}],
+      tree.dump
+    )
+  end
+end

data/test/test_import_and_export.rb

@@ -0,0 +1,33 @@
+$:.unshift( File.join( File.dirname(__FILE__), '..', 'lib' ))
+require 'test/unit'
+require 'bk'
+require 'stringio'
+
+class BKTreeImportAndExportTest < Test::Unit::TestCase
+  def test_should_give_correct_results_after_exporting_and_reimporting
+    tree = BK::Tree.new
+    terms = %w[
+      lorem ipsum dolor sit amet consectetuer adipiscing elit donec eget lectus vivamus nec
+      odio non ipsum adipiscing ornare etiam sapien
+    ].uniq
+    terms.each do |term|
+      tree.add(term)
+    end
+
+    stream = StringIO.new
+    tree.export(stream)
+
+    stream.rewind
+    tree = BK::Tree.import(stream)
+
+    search_term = 'sapient'
+    threshold = 1
+    expected = terms.inject({}){ |acc, t|
+      d = Text::Levenshtein.distance(t, search_term)
+      acc[t] = d if d <= threshold
+      acc
+    }
+    assert expected.any?
+    assert_equal expected, tree.query(search_term, threshold)
+  end
+end

data/test/test_querying_tree.rb

@@ -0,0 +1,84 @@
+$:.unshift( File.join( File.dirname(__FILE__), '..', 'lib' ))
+require 'test/unit'
+require 'bk'
+
+class BKTreeQueryAccuracyTest < Test::Unit::TestCase
+  def test_should_match_the_results_of_a_linear_scan
+    tree = BK::Tree.new
+    terms = %w[
+      lorem ipsum dolor sit amet consectetuer adipiscing elit donec eget lectus vivamus nec
+      odio non ipsum adipiscing ornare etiam sapien
+    ].uniq
+    terms.each do |term|
+      tree.add(term)
+    end
+
+    search_term = 'sapient'
+    threshold = 1
+    expected = terms.inject({}){ |acc, t|
+      d = Text::Levenshtein.distance(t, search_term)
+      acc[t] = d if d <= threshold
+      acc
+    }
+    assert expected.any?
+    assert_equal expected, tree.query(search_term, threshold)
+  end
+end
+
+class BKTreeSearchSpaceTest < Test::Unit::TestCase
+
+  class RecordingLevenshteinDistancer < BK::LevenshteinDistancer
+    attr_reader :history
+
+    def initialize
+      @history = []
+      @counting = false
+    end
+
+    def call(a, b)
+      @history << [a, b] if @recording
+      super
+    end
+
+    def start_recording
+      @recording = true
+    end
+  end
+
+  def test_should_compare_only_necessary_nodes
+    tree = BK::Tree.new
+    terms = %w[
+      infighting
+      birded
+      inebriation
+      stargazers
+      troika
+      bostonians
+      contemplating
+      gamey
+      skydove
+      scandalously
+      archaeological
+      soundness
+      tightwads
+      wanderlust
+    ]
+    distancer = RecordingLevenshteinDistancer.new
+    tree = BK::Tree.new(distancer)
+    terms.each do |term|
+      tree.add(term)
+    end
+    distancer.start_recording
+    tree.query('game', 1)
+    expected = [
+      %w[ game infighting ],
+      %w[ game contemplating],
+      %w[ game birded],
+      %w[ game gamey ],
+      %w[ game troika ],
+      %w[ game skydove],
+      %w[ game soundness ]
+    ]
+    assert_equal expected, distancer.history
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: bk
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Paul Battley
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-13 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: text
+  requirement: &70195843345880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70195843345880
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: &70195843345460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70195843345460
+description: Burkhard Keller Tree implementation in Ruby
+email: pbattley@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Gemfile
+- lib/bk/dot_graph.rb
+- lib/bk/dump.rb
+- lib/bk/version.rb
+- lib/bk.rb
+- samples/graph.rb
+- samples/performance.rb
+- test/test_all.rb
+- test/test_building_tree.rb
+- test/test_import_and_export.rb
+- test/test_querying_tree.rb
+homepage: https://github.com/threedaymonk/bktree
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Burkhard Keller Tree implementation in Ruby
+test_files:
+- test/test_all.rb
+- test/test_building_tree.rb
+- test/test_import_and_export.rb
+- test/test_querying_tree.rb