heroku-tokyotyrant 0.4

data/README.rdoc

@@ -0,0 +1,231 @@
+= TokyoTyrant Ruby Client
+
+This is a c extension for Ruby to access TokyoTyrant databases.  It currently supports key/value, table databases and table queries.
+
+== Install
+
+  # install tokyocabinet (1.4.34) and tokyotyrant (requires 1.1.35)
+  # after installing tc and tt on linux I had to /sbin/ldconfig (as root)
+  sudo gem install gemcutter
+  sudo gem tumble
+  sudo gem install ruby-tokyotyrant
+
+== Performance
+
+This is not in production but the initial benchmarks are very interesting. Results look closer to the memcached gem than any other tyrant client I've seen for Ruby.
+
+* Key/Value Store: http://gist.github.com/75212
+* Table Store: http://gist.github.com/74116
+* Bulk Operations: http://gist.github.com/83194
+* Bulk Table Operations: http://gist.github.com/87215
+
+== Example 
+
+=== Hash DB
+
+  # start tyrant like so:
+  # ttserver example.tch
+
+  require 'tokyo_tyrant'
+  db = TokyoTyrant::DB.new('127.0.0.1', 1978)
+
+  db['foo'] = 'Bar' # => "Bar"
+  db['foo']         # => "Bar"
+
+  db.each{ |k,v| puts [k, v].inspect }
+  # ["foo", "Bar"]
+  # => nil
+
+  db.mput("1"=>"number_1", "2"=>"number_2", "3"=>"number_3", "4"=>"number_4", "5"=>"number_5")
+  db.mget(1..3) # => {"1"=>"number_1", "2"=>"number_2", "3"=>"number_3"}
+
+=== Table DB
+
+  # start tyrant like so:
+  # ttserver example.tct
+
+  require 'tokyo_tyrant'
+  t = TokyoTyrant::Table.new('127.0.0.1', 1978)
+
+  t['bar'] = { :baz => 'box' } # => { :baz => 'box' }
+  t['bar']                     # => { :baz => 'box' }
+
+  t.each{ |k,v| puts [k, v].inspect }
+  # ["bar", {:baz=>"box"}]
+  # => nil
+
+  # bulk operations
+  h = {}
+  100.times do |i|
+    h[i] = { :name => 'Pat', :sex => i % 2 > 0 ? 'male' : 'female' }
+  end
+  t.mput(h)
+  t.mget(0..3)
+  # => {"0"=>{:name=>"Pat", :sex=>"female"}, "1"=>{:name=>"Pat", :sex=>"male"}, "2"=>{:name=>"Pat", :sex=>"female"}, "3"=>{:name=>"Pat", :sex=>"male"}}
+
+=== Table Query
+
+  require 'tokyo_tyrant'
+  t = TokyoTyrant::Table.new('127.0.0.1', 1978)
+
+  100.times do |i|
+    t[i] = { 'name' => "Pat #{i}", 'sex' => i % 2 > 0 ? 'male' : 'female' }
+  end
+
+  q = t.query
+  q.condition('sex', :streq, 'male')
+  q.limit(5)
+  # Get a list of IDs
+  ids = q.search
+  # => ["1", "3", "5", "7", "9"]
+  q.order_by(:name, :strdesc)
+  ids = q.search
+  # => ["99", "97", "95", "93", "91"]
+  # Get the actual records
+  q.get
+  # => [{:__id=>"99", :sex=>"male", :name=>"Pat 99"}, {:__id=>"97", :sex=>"male", :name=>"Pat 97"}, {:__id=>"95", :sex=>"male", :name=>"Pat 95"}, {:__id=>"93", :sex=>"male", :name=>"Pat 93"}, {:__id=>"91", :sex=>"male", :name=>"Pat 91"}]
+
+  # Alternative Syntax (better)
+
+  # Query using a block
+  t.query{ |q|
+    q.condition('sex', :streq, 'male')
+    q.limit(5)
+  }
+  # => ["1", "3", "5", "7", "9"]
+
+  # Get records for a query
+  t.find{ |q|
+    q.condition('sex', :streq, 'male')
+    q.limit(5)
+  }
+  # => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]
+
+  # Prepare but don't run a search, return a prepared query object
+  q = t.prepare_query{ |q|
+    q.condition('sex', :streq, 'male')
+    q.limit(5)
+  }
+  # => #<TokyoTyrant::Query:0x247c14 @rdb=#<Object:0x2800a0>, @rdbquery=#<Object:0x247c00>>
+  q.search
+  # => ["1", "3", "5", "7", "9"]
+  q.get
+  # => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]
+
+=== Full Text Search
+
+  require 'tokyo_tyrant'
+  require 'nokogiri'
+  require 'open-uri'
+
+  t = TokyoTyrant::Table.new('127.0.0.1', 1978)
+
+  (1..13).each do |n|
+    doc = Nokogiri::HTML(open("http://www.sacred-texts.com/chr/herm/hermes#{n}.htm"))
+    chapter = doc.css('h2').last.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
+    doc.css('p').each_with_index do |paragraph, i|
+      paragraph = paragraph.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
+      key = "chapter:#{n}:paragraph:#{i+1}"
+      t[key] = { :chapter => chapter, :paragraph => paragraph }
+    end
+  end
+
+  # full-text search with the phrase of
+  t.query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
+  # => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "chapter:13:paragraph:69", "chapter:13:paragraph:125"]
+
+  # full-text search with all tokens in
+  t.query{ |q| q.condition(:paragraph, :ftsand, 'logos word') }
+  # => ["chapter:1:paragraph:12", "chapter:1:paragraph:14", "chapter:1:paragraph:17", "chapter:1:paragraph:19", "chapter:1:paragraph:24", "chapter:1:paragraph:27", "chapter:1:paragraph:43", "chapter:1:paragraph:53", "... lots more ..."]
+
+  # full-text search with at least one token in
+  t.query{ |q| q.condition(:paragraph, :ftsor, 'sermon key') }
+  # => ["chapter:5:paragraph:1", "chapter:9:paragraph:3", "chapter:10:paragraph:1", "chapter:10:paragraph:4", "chapter:10:paragraph:28", "chapter:11:paragraph:3", "chapter:11:paragraph:66", "chapter:11:paragraph:69", "... lots more ..."]
+
+  # negated full-text search with at least one token in
+  t.query{ |q| q.condition(:paragraph, '!ftsor', 'the god he and I that said') }
+  # => ["chapter:1:paragraph:95", "chapter:1:paragraph:96", "chapter:1:paragraph:97", "chapter:1:paragraph:98", "chapter:1:paragraph:99", "chapter:2:paragraph:3", "chapter:2:paragraph:5", "chapter:2:paragraph:6", "... lots more ..."]
+
+==== Meta Search (Multi Query)
+
+  query1 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
+  query2 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'logos') }
+
+  # Get the union of two query sets (OR)
+  t.search(:union, query1, query2)
+  # => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "... lots more ..."]
+
+  # Get the intersection of two query sets (AND)
+  t.search(:intersection, query1, query2)
+  # => ["chapter:13:paragraph:5", "chapter:13:paragraph:44", "chapter:13:paragraph:69"]
+
+  # Get the difference of two query sets (ANDNOT)
+  t.search(:diff, query1, query2)
+  # => ["chapter:13:paragraph:4", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:57", "chapter:13:paragraph:125"]
+
+=== Parallel Querying (Take that scalability!)
+
+  require 'tokyo_tyrant'
+  require 'faker'
+  tyrants = { 1 => TokyoTyrant::Table.new('127.0.0.1', 1978),
+              2 => TokyoTyrant::Table.new('127.0.0.1', 1979) }
+
+  # dummy data
+  tyrants.each{ |account_id, table|
+    table.clear
+    20.times do |i|
+      table["#{account_id}/#{i}"] = { # consistent hashing would be good here
+        :name => Faker::Company.name,
+        :plan => rand(3),
+      }
+    end
+  }
+
+  queries = tyrants.collect{ |account_id, table|
+    table.prepare_query{ |q| q.condition 'plan', :numlt, '1' }
+  }
+
+  TokyoTyrant::Query.parallel_search(*queries).collect{ |r| {r[""] => r["name"]} }
+  # => [{"1/3"=>"Zemlak-Jerde"}, {"1/6"=>"O'Conner-Batz"}, {"1/9"=>"Kutch, Erdman and Aufderhar"}, {"1/11"=>"Bartoletti, Armstrong and Barrows"}, {"1/12"=>"Ferry-Dicki"}, {"2/0"=>"Schultz-O'Hara"}, {"2/1"=>"Emmerich, Feest and Huels"}, {"2/2"=>"Borer and Sons"}, {"2/3"=>"D'Amore Inc"}, {"2/5"=>"Koch and Sons"}, {"2/8"=>"Schaefer Group"}, {"2/11"=>"Stroman, Toy and Abernathy"}, {"2/19"=>"Gaylord, Reinger and White"}]
+
+=== Lua Extension
+
+  # ttserver -ext spec/ext.lua 
+  require 'tokyo_tyrant'
+  t = TokyoTyrant::Table.new('127.0.0.1', 1978)
+
+  t.run(:echo, 'hello', 'world')
+  # => "hello\tworld"
+
+=== Balanced Nodes with Consistent Hashing
+
+  # usage is similar to single node
+  require 'tokyo_tyrant/balancer'
+
+  servers = ['127.0.0.1:1978',
+             '127.0.0.1:1979',
+             '127.0.0.1:1980',
+             '127.0.0.1:1981']
+
+  tb = TokyoTyrant::Balancer::Table.new(servers)
+
+  # store server is determined by key which is consistent
+  tb[:foo] = { 'foo' => 'bar' }
+  tb[:bar] = { 'bar' => 'baz' }
+
+  # retrieval server is determined by key which is consistent
+  tb[:foo]
+  # => { 'foo' => 'bar' }
+
+  # aggregate from all nodes
+  tb.size
+
+  # parallel_search based querying across all nodes
+  tb.find{ |q| q.condition(:foo, :streq, 'bar') }
+
+== Contributors
+
+* Flinn Mueller (actsasflinn) author/maintainer
+* Justin Reagor (cheapRoc) specs
+* Seth Yates (sethyates) run method (lua ext)
+* John Mettraux (jmettraux) inspiration (rufus-tokyo)

data/Rakefile

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+$root = Pathname(__FILE__).dirname
+
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+task :spec do
+  load $root.join('spec', 'spec_base.rb')
+end
+task :default => [ :spec ]
+
+CLEAN.include('pkg', 'tmp')
+
+gemspec = Gem::Specification.new do |s|
+  s.name = 'heroku-tokyotyrant'
+  s.version = '0.4'
+  s.authors = [ 'Matt Bauer' ]
+  s.email = 'bauer@pedalbrain.com'
+  s.homepage = 'http://github.com/mattbauer/heroku-tokyotyrant/'
+  s.platform = Gem::Platform::RUBY
+  s.summary = 'A C based TokyoTyrant Ruby binding'
+  s.require_path = 'ext'
+  s.test_file = 'spec/spec.rb'
+  s.has_rdoc = true
+  s.extra_rdoc_files = %w{ README.rdoc }
+
+  s.files = ['COPYING',
+             'Rakefile',
+             'README.rdoc'] +
+             Dir['ext/**/*'] +
+             Dir['lib/**/*.rb'] +
+             Dir['spec/**/*'] +
+             Dir['benchmarks/**/*']
+  s.extensions << "ext/extconf.rb"
+
+  s.add_runtime_dependency('fast_hash_ring', '>= 0.1.1')
+end
+
+task :gemspec do
+  File.open('heroku-tokyotyrant.gemspec', 'w') do |f|
+    f.write(gemspec.to_ruby)
+  end
+end
+
+Rake::GemPackageTask.new(gemspec) do |pkg|
+  pkg.need_tar = true
+end
+
+Rake::PackageTask.new('heroku-tokyotyrant', '0.4') do |pkg|
+  pkg.need_zip = true
+  pkg.package_files = FileList[
+    'COPYING',
+    'Rakefile',
+    'README.rdoc',
+    'ext/**/*',
+    'lib/**/*.[rb]',
+    'spec/**/*',
+    'benchmarks/**/*'
+  ].to_a
+  class << pkg
+    def package_name
+      "#{@name}-#{@version}-src"
+    end
+  end
+end

data/benchmarks/balancer.rb

@@ -0,0 +1,101 @@
+require 'benchmark'
+require 'rubygems'
+require 'faker'
+require 'date'
+
+#
+# the data
+#
+ 
+colnames = %w{ name sex birthday divisions }
+ 
+$year = (1909 .. 2009).to_a
+$month = (1..12).to_a
+$day = (1..28).to_a # not bothering with month diffs
+ 
+def rbdate
+  DateTime.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
+end
+ 
+def rdiv
+  case rand(3)
+  when 0
+    'dev'
+  when 1
+    'brd'
+  else
+    'brd,dev'
+  end
+end
+ 
+def rgen
+  (rand(2) == 1 ? 'male' : 'female')
+end
+ 
+data = [
+  [ 'Alphonse Armalite', 'male', DateTime.new(1972, 10, 14), 'brd,dev' ],
+  [ 'Brutus Beromunster', 'male', DateTime.new(1964, 07, 14), 'dev' ],
+  [ 'Crystel Chucknorris', 'female', DateTime.new(1980, 07, 12), 'brd' ],
+  [ 'Desree Dylan', 'female', DateTime.new(1954, 07, 13), 'brd,dev' ]
+]
+ 
+10_000.times do |i|
+  data << [ Faker::Name.name, rgen, rbdate, rdiv]  
+end
+ 
+$find_name_list = []
+100.times { $find_name_list << data[rand(data.size)][0] }
+ 
+data.collect! { |e|
+  (0..colnames.length - 1).inject({}) { |h, i| h[colnames[i]] = e[i]; h }
+}
+
+data_h = {}
+i = 0
+data1 = data.collect { |e|
+  i = i + 1
+  h = e.dup
+  h['birthday'] = h['birthday'].to_s
+  data_h[i.to_s] = h
+  h
+}
+
+require 'tokyo_tyrant'
+
+t = TokyoTyrant::Table.new('127.0.0.1', 45001)
+t.clear
+
+2.times { puts }
+puts 'TokyoTyrant Single'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| t[i.to_s] = e }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = t[i.to_s] }
+  end
+end
+
+require 'lib/tokyo_tyrant/balancer'
+
+servers = ['127.0.0.1:45001',
+           '127.0.0.1:45002',
+           '127.0.0.1:45003']
+
+tb = TokyoTyrant::Balancer::Table.new(servers)
+tb.clear
+
+2.times { puts }
+puts 'TokyoTyrant Balancer'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    data1.each_with_index { |e, i| tb[i.to_s] = e }
+  end
+
+  b.report('reading data') do
+    data1.each_with_index { |e, i| nothing = tb[i.to_s] }
+  end
+end

data/benchmarks/bulk_db.rb

@@ -0,0 +1,74 @@
+require 'benchmark'
+require 'rubygems'
+require 'faker'
+
+puts "Tokyo Tyrant Bulk Operations Benchmark"
+
+data   = {}
+data_e = {}
+data_a = []
+
+10_000.times do |i|
+  data[i.to_s]   = Faker::Name.name
+  data_e[i.to_s] = nil
+  data_a << i.to_s
+  data_a << data[i.to_s]
+end
+
+require 'tokyotyrant'
+
+rdb = TokyoTyrant::RDB::new
+rdb.open("127.0.0.1", 45000)
+rdb.clear
+nothing = nil
+
+2.times { puts }
+puts 'TokyoTyrant::RDB (Ruby) mget'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data') do
+    rdb.misc('putlist', data_a)
+  end
+
+  b.report('reading data') do
+    rdb.mget(data_e)
+  end
+end
+
+require 'tokyo_tyrant'
+t = TokyoTyrant::DB.new('127.0.0.1', 45000)
+t.clear
+nothing = nil
+
+2.times { puts }
+puts 'TokyoTyrant (c) mput/mget'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data ') do
+    t.mput(data)
+  end
+
+  b.report('reading data') do
+    nothing = t.mget(0..9999)
+  end
+end
+
+require 'memcached'
+m = Memcached.new('127.0.0.1:11211')
+m.flush
+nothing = nil
+
+2.times { puts }
+puts 'Memcached (C) set/get_multi'
+
+Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
+  b.report('inserting data*') do
+    data.each_pair { |k, v| m.set(k, v) }
+  end
+
+  b.report('reading data') do
+    nothing = m.get(data.keys)
+  end
+end
+
+puts "* bulk operation not supported"