ruby-tokyotyrant 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/COPYING +504 -0
- data/README.rdoc +204 -0
- data/Rakefile +68 -0
- data/benchmarks/bulk_db.rb +74 -0
- data/benchmarks/bulk_table.rb +87 -0
- data/benchmarks/db.rb +114 -0
- data/benchmarks/table.rb +161 -0
- data/ext/extconf.rb +39 -0
- data/ext/tokyo_tyrant.c +147 -0
- data/ext/tokyo_tyrant.h +48 -0
- data/ext/tokyo_tyrant_db.c +227 -0
- data/ext/tokyo_tyrant_db.h +8 -0
- data/ext/tokyo_tyrant_module.c +453 -0
- data/ext/tokyo_tyrant_module.h +10 -0
- data/ext/tokyo_tyrant_query.c +226 -0
- data/ext/tokyo_tyrant_query.h +9 -0
- data/ext/tokyo_tyrant_table.c +319 -0
- data/ext/tokyo_tyrant_table.h +8 -0
- data/spec/ext.lua +4 -0
- data/spec/plu_db.rb +538 -0
- data/spec/spec.rb +1 -0
- data/spec/spec_base.rb +16 -0
- data/spec/start_tyrants.sh +36 -0
- data/spec/stop_tyrants.sh +9 -0
- data/spec/tokyo_tyrant_query_spec.rb +159 -0
- data/spec/tokyo_tyrant_spec.rb +254 -0
- data/spec/tokyo_tyrant_table_spec.rb +301 -0
- metadata +80 -0
data/README.rdoc
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
= TokyoTyrant Ruby Client
|
|
2
|
+
|
|
3
|
+
This is a c extension for Ruby to access TokyoTyrant databases. It currently supports key/value, table databases and table queries.
|
|
4
|
+
|
|
5
|
+
== Install
|
|
6
|
+
|
|
7
|
+
# install tokyocabinet (1.4.30) and tokyotyrant (requires 1.1.33)
|
|
8
|
+
# after installing tc and tt on linux I had to /sbin/ldconfig (as root)
|
|
9
|
+
gem sources -a http://gems.github.com
|
|
10
|
+
sudo gem install actsasflinn-ruby-tokyotyrant
|
|
11
|
+
|
|
12
|
+
== Performance
|
|
13
|
+
|
|
14
|
+
This is not in production but the initial benchmarks are very interesting. Results look closer to the memcached gem than any other tyrant client I've seen for Ruby.
|
|
15
|
+
|
|
16
|
+
* Key/Value Store: http://gist.github.com/75212
|
|
17
|
+
* Table Store: http://gist.github.com/74116
|
|
18
|
+
* Bulk Operations: http://gist.github.com/83194
|
|
19
|
+
* Bulk Table Operations: http://gist.github.com/87215
|
|
20
|
+
|
|
21
|
+
== Example
|
|
22
|
+
|
|
23
|
+
=== Hash DB
|
|
24
|
+
|
|
25
|
+
# start tyrant like so:
|
|
26
|
+
# ttserver example.tch
|
|
27
|
+
|
|
28
|
+
require 'tokyo_tyrant'
|
|
29
|
+
db = TokyoTyrant::DB.new('127.0.0.1', 1978)
|
|
30
|
+
|
|
31
|
+
db['foo'] = 'Bar' # => "Bar"
|
|
32
|
+
db['foo'] # => "Bar"
|
|
33
|
+
|
|
34
|
+
db.each{ |k,v| puts [k, v].inspect }
|
|
35
|
+
# ["foo", "Bar"]
|
|
36
|
+
# => nil
|
|
37
|
+
|
|
38
|
+
db.mput("1"=>"number_1", "2"=>"number_2", "3"=>"number_3", "4"=>"number_4", "5"=>"number_5")
|
|
39
|
+
db.mget(1..3) # => {"1"=>"number_1", "2"=>"number_2", "3"=>"number_3"}
|
|
40
|
+
|
|
41
|
+
=== Table DB
|
|
42
|
+
|
|
43
|
+
# start tyrant like so:
|
|
44
|
+
# ttserver example.tct
|
|
45
|
+
|
|
46
|
+
require 'tokyo_tyrant'
|
|
47
|
+
t = TokyoTyrant::Table.new('127.0.0.1', 1978)
|
|
48
|
+
|
|
49
|
+
t['bar'] = { :baz => 'box' } # => { :baz => 'box' }
|
|
50
|
+
t['bar'] # => { :baz => 'box' }
|
|
51
|
+
|
|
52
|
+
t.each{ |k,v| puts [k, v].inspect }
|
|
53
|
+
# ["bar", {:baz=>"box"}]
|
|
54
|
+
# => nil
|
|
55
|
+
|
|
56
|
+
# bulk operations
|
|
57
|
+
h = {}
|
|
58
|
+
100.times do |i|
|
|
59
|
+
h[i] = { :name => 'Pat', :sex => i % 2 > 0 ? 'male' : 'female' }
|
|
60
|
+
end
|
|
61
|
+
t.mput(h)
|
|
62
|
+
t.mget(0..3)
|
|
63
|
+
# => {"0"=>{:name=>"Pat", :sex=>"female"}, "1"=>{:name=>"Pat", :sex=>"male"}, "2"=>{:name=>"Pat", :sex=>"female"}, "3"=>{:name=>"Pat", :sex=>"male"}}
|
|
64
|
+
|
|
65
|
+
=== Table Query
|
|
66
|
+
|
|
67
|
+
require 'tokyo_tyrant'
|
|
68
|
+
t = TokyoTyrant::Table.new('127.0.0.1', 1978)
|
|
69
|
+
|
|
70
|
+
100.times do |i|
|
|
71
|
+
t[i] = { 'name' => "Pat #{i}", 'sex' => i % 2 > 0 ? 'male' : 'female' }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
q = t.query
|
|
75
|
+
q.condition('sex', :streq, 'male')
|
|
76
|
+
q.limit(5)
|
|
77
|
+
# Get a list of IDs
|
|
78
|
+
ids = q.search
|
|
79
|
+
# => ["1", "3", "5", "7", "9"]
|
|
80
|
+
q.order_by(:name, :strdesc)
|
|
81
|
+
ids = q.search
|
|
82
|
+
# => ["99", "97", "95", "93", "91"]
|
|
83
|
+
# Get the actual records
|
|
84
|
+
q.get
|
|
85
|
+
# => [{:__id=>"99", :sex=>"male", :name=>"Pat 99"}, {:__id=>"97", :sex=>"male", :name=>"Pat 97"}, {:__id=>"95", :sex=>"male", :name=>"Pat 95"}, {:__id=>"93", :sex=>"male", :name=>"Pat 93"}, {:__id=>"91", :sex=>"male", :name=>"Pat 91"}]
|
|
86
|
+
|
|
87
|
+
# Alternative Syntax (better)
|
|
88
|
+
|
|
89
|
+
# Query using a block
|
|
90
|
+
t.query{ |q|
|
|
91
|
+
q.condition('sex', :streq, 'male')
|
|
92
|
+
q.limit(5)
|
|
93
|
+
}
|
|
94
|
+
# => ["1", "3", "5", "7", "9"]
|
|
95
|
+
|
|
96
|
+
# Get records for a query
|
|
97
|
+
t.find{ |q|
|
|
98
|
+
q.condition('sex', :streq, 'male')
|
|
99
|
+
q.limit(5)
|
|
100
|
+
}
|
|
101
|
+
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]
|
|
102
|
+
|
|
103
|
+
# Prepare but don't run a search, return a prepared query object
|
|
104
|
+
q = t.prepare_query{ |q|
|
|
105
|
+
q.condition('sex', :streq, 'male')
|
|
106
|
+
q.limit(5)
|
|
107
|
+
}
|
|
108
|
+
# => #<TokyoTyrant::Query:0x247c14 @rdb=#<Object:0x2800a0>, @rdbquery=#<Object:0x247c00>>
|
|
109
|
+
q.search
|
|
110
|
+
# => ["1", "3", "5", "7", "9"]
|
|
111
|
+
q.get
|
|
112
|
+
# => [{:sex=>"male", :name=>"Pat 1", :__id=>"1"}, {:sex=>"male", :name=>"Pat 3", :__id=>"3"}, {:sex=>"male", :name=>"Pat 5", :__id=>"5"}, {:sex=>"male", :name=>"Pat 7", :__id=>"7"}, {:sex=>"male", :name=>"Pat 9", :__id=>"9"}]
|
|
113
|
+
|
|
114
|
+
=== Full Text Search
|
|
115
|
+
|
|
116
|
+
require 'tokyo_tyrant'
|
|
117
|
+
require 'nokogiri'
|
|
118
|
+
require 'open-uri'
|
|
119
|
+
|
|
120
|
+
t = TokyoTyrant::Table.new('127.0.0.1', 1978)
|
|
121
|
+
|
|
122
|
+
(1..13).each do |n|
|
|
123
|
+
doc = Nokogiri::HTML(open("http://www.sacred-texts.com/chr/herm/hermes#{n}.htm"))
|
|
124
|
+
chapter = doc.css('h2').last.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
|
|
125
|
+
doc.css('p').each_with_index do |paragraph, i|
|
|
126
|
+
paragraph = paragraph.inner_text.gsub(/\n/, '').gsub(/ +/, ' ').strip
|
|
127
|
+
key = "chapter:#{n}:paragraph:#{i+1}"
|
|
128
|
+
t[key] = { :chapter => chapter, :paragraph => paragraph }
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# full-text search with the phrase of
|
|
133
|
+
t.query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
|
|
134
|
+
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "chapter:13:paragraph:69", "chapter:13:paragraph:125"]
|
|
135
|
+
|
|
136
|
+
# full-text search with all tokens in
|
|
137
|
+
t.query{ |q| q.condition(:paragraph, :ftsand, 'logos word') }
|
|
138
|
+
# => ["chapter:1:paragraph:12", "chapter:1:paragraph:14", "chapter:1:paragraph:17", "chapter:1:paragraph:19", "chapter:1:paragraph:24", "chapter:1:paragraph:27", "chapter:1:paragraph:43", "chapter:1:paragraph:53", "... lots more ..."]
|
|
139
|
+
|
|
140
|
+
# full-text search with at least one token in
|
|
141
|
+
t.query{ |q| q.condition(:paragraph, :ftsor, 'sermon key') }
|
|
142
|
+
# => ["chapter:5:paragraph:1", "chapter:9:paragraph:3", "chapter:10:paragraph:1", "chapter:10:paragraph:4", "chapter:10:paragraph:28", "chapter:11:paragraph:3", "chapter:11:paragraph:66", "chapter:11:paragraph:69", "... lots more ..."]
|
|
143
|
+
|
|
144
|
+
# negated full-text search with at least one token in
|
|
145
|
+
t.query{ |q| q.condition(:paragraph, '!ftsor', 'the god he and I that said') }
|
|
146
|
+
# => ["chapter:1:paragraph:95", "chapter:1:paragraph:96", "chapter:1:paragraph:97", "chapter:1:paragraph:98", "chapter:1:paragraph:99", "chapter:2:paragraph:3", "chapter:2:paragraph:5", "chapter:2:paragraph:6", "... lots more ..."]
|
|
147
|
+
|
|
148
|
+
==== Meta Search (Multi Query)
|
|
149
|
+
|
|
150
|
+
query1 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'rebirth') }
|
|
151
|
+
query2 = t.prepare_query{ |q| q.condition(:paragraph, :fts, 'logos') }
|
|
152
|
+
|
|
153
|
+
# Get the union of two query sets (OR)
|
|
154
|
+
t.search(:union, query1, query2)
|
|
155
|
+
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:5", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:44", "chapter:13:paragraph:57", "... lots more ..."]
|
|
156
|
+
|
|
157
|
+
# Get the intersection of two query sets (AND)
|
|
158
|
+
t.search(:intersection, query1, query2)
|
|
159
|
+
# => ["chapter:13:paragraph:5", "chapter:13:paragraph:44", "chapter:13:paragraph:69"]
|
|
160
|
+
|
|
161
|
+
# Get the difference of two query sets (ANDNOT)
|
|
162
|
+
t.search(:diff, query1, query2)
|
|
163
|
+
# => ["chapter:13:paragraph:4", "chapter:13:paragraph:7", "chapter:13:paragraph:19", "chapter:13:paragraph:27", "chapter:13:paragraph:57", "chapter:13:paragraph:125"]
|
|
164
|
+
|
|
165
|
+
=== Parallel Querying (Take that scalability!)
|
|
166
|
+
|
|
167
|
+
require 'tokyo_tyrant'
|
|
168
|
+
require 'faker'
|
|
169
|
+
tyrants = { 1 => TokyoTyrant::Table.new('127.0.0.1', 1978),
|
|
170
|
+
2 => TokyoTyrant::Table.new('127.0.0.1', 1979) }
|
|
171
|
+
|
|
172
|
+
# dummy data
|
|
173
|
+
tyrants.each{ |account_id, table|
|
|
174
|
+
table.clear
|
|
175
|
+
20.times do |i|
|
|
176
|
+
table["#{account_id}/#{i}"] = { # consistent hashing would be good here
|
|
177
|
+
:name => Faker::Company.name,
|
|
178
|
+
:plan => rand(3),
|
|
179
|
+
}
|
|
180
|
+
end
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
queries = tyrants.collect{ |account_id, table|
|
|
184
|
+
table.prepare_query{ |q| q.condition 'plan', :numlt, '1' }
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
TokyoTyrant::Query.parallel_search(*queries).collect{ |r| {r[""] => r["name"]} }
|
|
188
|
+
# => [{"1/3"=>"Zemlak-Jerde"}, {"1/6"=>"O'Conner-Batz"}, {"1/9"=>"Kutch, Erdman and Aufderhar"}, {"1/11"=>"Bartoletti, Armstrong and Barrows"}, {"1/12"=>"Ferry-Dicki"}, {"2/0"=>"Schultz-O'Hara"}, {"2/1"=>"Emmerich, Feest and Huels"}, {"2/2"=>"Borer and Sons"}, {"2/3"=>"D'Amore Inc"}, {"2/5"=>"Koch and Sons"}, {"2/8"=>"Schaefer Group"}, {"2/11"=>"Stroman, Toy and Abernathy"}, {"2/19"=>"Gaylord, Reinger and White"}]
|
|
189
|
+
|
|
190
|
+
=== Lua Extension
|
|
191
|
+
|
|
192
|
+
# ttserver -ext spec/ext.lua
|
|
193
|
+
require 'tokyo_tyrant'
|
|
194
|
+
t = TokyoTyrant::Table.new('127.0.0.1', 1978)
|
|
195
|
+
|
|
196
|
+
t.run(:echo, 'hello', 'world')
|
|
197
|
+
# => "hello\tworld"
|
|
198
|
+
|
|
199
|
+
== Contributors
|
|
200
|
+
|
|
201
|
+
* Flinn Mueller (actsasflinn) author/maintainer
|
|
202
|
+
* Justin Reagor (cheapRoc) specs
|
|
203
|
+
* Seth Yates (sethyates) run method (lua ext)
|
|
204
|
+
* John Mettraux (jmettraux) inspiration (rufus-tokyo)
|
data/Rakefile
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'pathname'
|
|
4
|
+
$root = Pathname(__FILE__).dirname
|
|
5
|
+
|
|
6
|
+
require 'rubygems'
|
|
7
|
+
require 'rake'
|
|
8
|
+
require 'rake/clean'
|
|
9
|
+
require 'rake/packagetask'
|
|
10
|
+
require 'rake/gempackagetask'
|
|
11
|
+
require 'rake/testtask'
|
|
12
|
+
require 'rake/rdoctask'
|
|
13
|
+
|
|
14
|
+
task :spec do
|
|
15
|
+
load $root.join('spec', 'spec_base.rb')
|
|
16
|
+
end
|
|
17
|
+
task :default => [ :spec ]
|
|
18
|
+
|
|
19
|
+
CLEAN.include('pkg', 'tmp')
|
|
20
|
+
|
|
21
|
+
gemspec = Gem::Specification.new do |s|
|
|
22
|
+
s.name = 'ruby-tokyotyrant'
|
|
23
|
+
s.version = '0.3.0'
|
|
24
|
+
s.authors = [ 'Flinn' ]
|
|
25
|
+
s.email = 'flinn@actsasflinn.com'
|
|
26
|
+
s.homepage = 'http://github.com/actsasflinn/ruby-tokyotyrant/'
|
|
27
|
+
s.platform = Gem::Platform::RUBY
|
|
28
|
+
s.summary = 'A C based TokyoTyrant Ruby binding'
|
|
29
|
+
s.require_path = 'ext'
|
|
30
|
+
s.test_file = 'spec/spec.rb'
|
|
31
|
+
s.has_rdoc = true
|
|
32
|
+
s.extra_rdoc_files = %w{ README.rdoc }
|
|
33
|
+
|
|
34
|
+
s.files = ['COPYING',
|
|
35
|
+
'Rakefile',
|
|
36
|
+
'README.rdoc'] +
|
|
37
|
+
Dir['ext/**/*.[rb|c|h]'] +
|
|
38
|
+
Dir['spec/**/*'] +
|
|
39
|
+
Dir['benchmarks/**/*']
|
|
40
|
+
s.extensions << "ext/extconf.rb"
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
task :gemspec do
|
|
44
|
+
File.open('ruby-tokyotyrant.gemspec', 'w') do |f|
|
|
45
|
+
f.write(gemspec.to_ruby)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
Rake::GemPackageTask.new(gemspec) do |pkg|
|
|
50
|
+
pkg.need_tar = true
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
Rake::PackageTask.new('ruby-tokyotyrant', '0.1') do |pkg|
|
|
54
|
+
pkg.need_zip = true
|
|
55
|
+
pkg.package_files = FileList[
|
|
56
|
+
'COPYING',
|
|
57
|
+
'Rakefile',
|
|
58
|
+
'README.rdoc',
|
|
59
|
+
'ext/**/*',
|
|
60
|
+
'spec/**/*',
|
|
61
|
+
'benchmarks/**/*'
|
|
62
|
+
].to_a
|
|
63
|
+
class << pkg
|
|
64
|
+
def package_name
|
|
65
|
+
"#{@name}-#{@version}-src"
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
require 'benchmark'
|
|
2
|
+
require 'rubygems'
|
|
3
|
+
require 'faker'
|
|
4
|
+
|
|
5
|
+
puts "Tokyo Tyrant Bulk Operations Benchmark"
|
|
6
|
+
|
|
7
|
+
data = {}
|
|
8
|
+
data_e = {}
|
|
9
|
+
data_a = []
|
|
10
|
+
|
|
11
|
+
10_000.times do |i|
|
|
12
|
+
data[i.to_s] = Faker::Name.name
|
|
13
|
+
data_e[i.to_s] = nil
|
|
14
|
+
data_a << i.to_s
|
|
15
|
+
data_a << data[i.to_s]
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
require 'tokyotyrant'
|
|
19
|
+
|
|
20
|
+
rdb = TokyoTyrant::RDB::new
|
|
21
|
+
rdb.open("127.0.0.1", 45000)
|
|
22
|
+
rdb.clear
|
|
23
|
+
nothing = nil
|
|
24
|
+
|
|
25
|
+
2.times { puts }
|
|
26
|
+
puts 'TokyoTyrant::RDB (Ruby) mget'
|
|
27
|
+
|
|
28
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
29
|
+
b.report('inserting data') do
|
|
30
|
+
rdb.misc('putlist', data_a)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
b.report('reading data') do
|
|
34
|
+
rdb.mget(data_e)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
require 'tokyo_tyrant'
|
|
39
|
+
t = TokyoTyrant::DB.new('127.0.0.1', 45000)
|
|
40
|
+
t.clear
|
|
41
|
+
nothing = nil
|
|
42
|
+
|
|
43
|
+
2.times { puts }
|
|
44
|
+
puts 'TokyoTyrant (c) mput/mget'
|
|
45
|
+
|
|
46
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
47
|
+
b.report('inserting data ') do
|
|
48
|
+
t.mput(data)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
b.report('reading data') do
|
|
52
|
+
nothing = t.mget(0..9999)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
require 'memcached'
|
|
57
|
+
m = Memcached.new('127.0.0.1:11211')
|
|
58
|
+
m.flush
|
|
59
|
+
nothing = nil
|
|
60
|
+
|
|
61
|
+
2.times { puts }
|
|
62
|
+
puts 'Memcached (C) set/get_multi'
|
|
63
|
+
|
|
64
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
65
|
+
b.report('inserting data*') do
|
|
66
|
+
data.each_pair { |k, v| m.set(k, v) }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
b.report('reading data') do
|
|
70
|
+
nothing = m.get(data.keys)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
puts "* bulk operation not supported"
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
require 'benchmark'
|
|
2
|
+
require 'rubygems'
|
|
3
|
+
require 'faker'
|
|
4
|
+
require 'date'
|
|
5
|
+
|
|
6
|
+
puts "Tokyo Tyrant Bulk Table Operations Benchmark"
|
|
7
|
+
|
|
8
|
+
$year = (1909 .. 2009).to_a
|
|
9
|
+
$month = (1..12).to_a
|
|
10
|
+
$day = (1..28).to_a # not bothering with month diffs
|
|
11
|
+
|
|
12
|
+
def rbdate
|
|
13
|
+
DateTime.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def rdiv
|
|
17
|
+
case rand(3)
|
|
18
|
+
when 0
|
|
19
|
+
'dev'
|
|
20
|
+
when 1
|
|
21
|
+
'brd'
|
|
22
|
+
else
|
|
23
|
+
'brd,dev'
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def rgen
|
|
28
|
+
(rand(2) == 1 ? 'male' : 'female')
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
data = {}
|
|
32
|
+
10_000.times do |i|
|
|
33
|
+
data[i] = { :name => Faker::Name.name, :sex => rgen, :birthday => rbdate.to_s, :divisions => rdiv }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
require 'tokyo_tyrant'
|
|
37
|
+
t = TokyoTyrant::Table.new('127.0.0.1', 45001)
|
|
38
|
+
t.clear
|
|
39
|
+
|
|
40
|
+
2.times { puts }
|
|
41
|
+
puts 'TokyoTyrant (C)'
|
|
42
|
+
|
|
43
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
44
|
+
b.report('bulk writing data') do
|
|
45
|
+
nothing = t.mput(data)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
b.report('bulk reading data') do
|
|
49
|
+
nothing = t.mget(0..9999)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
require 'tokyotyrant'
|
|
54
|
+
|
|
55
|
+
rdb = TokyoTyrant::RDB::new
|
|
56
|
+
rdb.open("127.0.0.1", 45001)
|
|
57
|
+
rdb.clear
|
|
58
|
+
|
|
59
|
+
2.times { puts }
|
|
60
|
+
puts 'TokyoTyrant::RDB (Ruby)'
|
|
61
|
+
|
|
62
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
63
|
+
b.report('bulk inserting data*') do
|
|
64
|
+
# is this fair to put in the benchmark? yes because it happens within the c-ext
|
|
65
|
+
data_a = []
|
|
66
|
+
data.each_pair{ |i,v|
|
|
67
|
+
data_a << i.to_s
|
|
68
|
+
data_a << v.collect{ |k,v| [k.to_s,v] }.join("\0")
|
|
69
|
+
}
|
|
70
|
+
rdb.misc('putlist', data_a)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
b.report('bulk reading data ') do
|
|
74
|
+
l = rdb.misc('getlist', data.keys)
|
|
75
|
+
h = Hash[*l]
|
|
76
|
+
h.each_pair do |k,v|
|
|
77
|
+
a = v.split("\0")
|
|
78
|
+
h[k] = Hash[*a]
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
puts "\nNotes:"
|
|
84
|
+
puts "* To supply a hash for bulk operations TokyoTyrant::RDB creates an array and join hash columns."
|
|
85
|
+
puts "* This operation is included in the benchmark because the same code exists within the c-ext."
|
|
86
|
+
puts "Rufus::Tokyo::TyrantTable does not support the misc method, therefor cannot handle a putlist/getlist"
|
|
87
|
+
puts "Memcached does not support bulk writing"
|
data/benchmarks/db.rb
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
require 'benchmark'
|
|
2
|
+
require 'rubygems'
|
|
3
|
+
require 'faker'
|
|
4
|
+
|
|
5
|
+
data = []
|
|
6
|
+
|
|
7
|
+
10_000.times do |i|
|
|
8
|
+
data << Faker::Name.name
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
require 'rufus/tokyo/tyrant'
|
|
12
|
+
|
|
13
|
+
r = Rufus::Tokyo::Tyrant.new('127.0.0.1', 45000)
|
|
14
|
+
r.clear
|
|
15
|
+
|
|
16
|
+
2.times { puts }
|
|
17
|
+
puts 'Tokyo::Tyrant (Ruby FFI)'
|
|
18
|
+
|
|
19
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
20
|
+
b.report('inserting data') do
|
|
21
|
+
data.each_with_index { |e, i| r[i.to_s] = e }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
b.report('reading data') do
|
|
25
|
+
data.each_with_index { |e, i| nothing = r[i.to_s] }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
require 'tokyotyrant'
|
|
30
|
+
|
|
31
|
+
rdb = TokyoTyrant::RDB::new
|
|
32
|
+
rdb.open("127.0.0.1", 45000)
|
|
33
|
+
rdb.clear
|
|
34
|
+
|
|
35
|
+
2.times { puts }
|
|
36
|
+
puts 'TokyoTyrant::RDB (Ruby)'
|
|
37
|
+
|
|
38
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
39
|
+
b.report('inserting data') do
|
|
40
|
+
data.each_with_index { |e, i| rdb.put(i.to_s, e) }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
b.report('reading data') do
|
|
44
|
+
data.each_with_index { |e, i| nothing = rdb.get(i.to_s) }
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
require 'tokyo_tyrant'
|
|
49
|
+
t = TokyoTyrant::DB.new('127.0.0.1', 45000)
|
|
50
|
+
t.clear
|
|
51
|
+
|
|
52
|
+
2.times { puts }
|
|
53
|
+
puts 'TokyoTyrant (C)'
|
|
54
|
+
|
|
55
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
56
|
+
b.report('inserting data') do
|
|
57
|
+
data.each_with_index { |e, i| t[i] = e }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
b.report('reading data') do
|
|
61
|
+
data.each_with_index { |e, i| nothing = t[i] }
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
require 'memcached'
|
|
66
|
+
m = Memcached.new('127.0.0.1:45000')
|
|
67
|
+
m.flush
|
|
68
|
+
|
|
69
|
+
2.times { puts }
|
|
70
|
+
puts 'Memcached (C) to Tyrant'
|
|
71
|
+
|
|
72
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
73
|
+
b.report('inserting data') do
|
|
74
|
+
data.each_with_index { |e, i| m.set(i.to_s, e) }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
b.report('reading data') do
|
|
78
|
+
data.each_with_index { |e, i| nothing = m.get(i.to_s) }
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
require 'memcached'
|
|
83
|
+
m = Memcached.new('127.0.0.1:11211')
|
|
84
|
+
m.flush
|
|
85
|
+
|
|
86
|
+
2.times { puts }
|
|
87
|
+
puts 'Memcached (C) to Memcached'
|
|
88
|
+
|
|
89
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
90
|
+
b.report('inserting data') do
|
|
91
|
+
data.each_with_index { |e, i| m.set(i.to_s, e) }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
b.report('reading data') do
|
|
95
|
+
data.each_with_index { |e, i| nothing = m.get(i.to_s) }
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
require 'memcache'
|
|
100
|
+
mc = MemCache.new('127.0.0.1:11211')
|
|
101
|
+
mc.flush_all
|
|
102
|
+
|
|
103
|
+
2.times { puts }
|
|
104
|
+
puts 'MemCache (Ruby)'
|
|
105
|
+
|
|
106
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
107
|
+
b.report('inserting data') do
|
|
108
|
+
data.each_with_index { |e, i| mc.set(i.to_s, e) }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
b.report('reading data') do
|
|
112
|
+
data.each_with_index { |e, i| nothing = mc.get(i.to_s) }
|
|
113
|
+
end
|
|
114
|
+
end
|
data/benchmarks/table.rb
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
require 'benchmark'
|
|
2
|
+
require 'rubygems'
|
|
3
|
+
require 'faker'
|
|
4
|
+
require 'date'
|
|
5
|
+
|
|
6
|
+
#
|
|
7
|
+
# the data
|
|
8
|
+
#
|
|
9
|
+
|
|
10
|
+
colnames = %w{ name sex birthday divisions }
|
|
11
|
+
|
|
12
|
+
$year = (1909 .. 2009).to_a
|
|
13
|
+
$month = (1..12).to_a
|
|
14
|
+
$day = (1..28).to_a # not bothering with month diffs
|
|
15
|
+
|
|
16
|
+
def rbdate
|
|
17
|
+
DateTime.new($year[rand($year.size)], $month[rand($month.size)], $day[rand($day.size)])
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def rdiv
|
|
21
|
+
case rand(3)
|
|
22
|
+
when 0
|
|
23
|
+
'dev'
|
|
24
|
+
when 1
|
|
25
|
+
'brd'
|
|
26
|
+
else
|
|
27
|
+
'brd,dev'
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def rgen
|
|
32
|
+
(rand(2) == 1 ? 'male' : 'female')
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
data = [
|
|
36
|
+
[ 'Alphonse Armalite', 'male', DateTime.new(1972, 10, 14), 'brd,dev' ],
|
|
37
|
+
[ 'Brutus Beromunster', 'male', DateTime.new(1964, 07, 14), 'dev' ],
|
|
38
|
+
[ 'Crystel Chucknorris', 'female', DateTime.new(1980, 07, 12), 'brd' ],
|
|
39
|
+
[ 'Desree Dylan', 'female', DateTime.new(1954, 07, 13), 'brd,dev' ]
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
10_000.times do |i|
|
|
43
|
+
data << [ Faker::Name.name, rgen, rbdate, rdiv]
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
$find_name_list = []
|
|
47
|
+
100.times { $find_name_list << data[rand(data.size)][0] }
|
|
48
|
+
|
|
49
|
+
data.collect! { |e|
|
|
50
|
+
(0..colnames.length - 1).inject({}) { |h, i| h[colnames[i]] = e[i]; h }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
data_h = {}
|
|
54
|
+
i = 0
|
|
55
|
+
data1 = data.collect { |e|
|
|
56
|
+
i = i + 1
|
|
57
|
+
h = e.dup
|
|
58
|
+
h['birthday'] = h['birthday'].to_s
|
|
59
|
+
data_h[i.to_s] = h
|
|
60
|
+
h
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
require 'rufus/tokyo/tyrant'
|
|
64
|
+
|
|
65
|
+
r = Rufus::Tokyo::TyrantTable.new('127.0.0.1', 45001)
|
|
66
|
+
r.clear
|
|
67
|
+
|
|
68
|
+
2.times { puts }
|
|
69
|
+
puts 'Tokyo::TyrantTable (Ruby FFI)'
|
|
70
|
+
|
|
71
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
72
|
+
b.report('inserting data') do
|
|
73
|
+
data1.each_with_index { |e, i| r[i.to_s] = e }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
b.report('reading data') do
|
|
77
|
+
data1.each_with_index { |e, i| nothing = r[i.to_s] }
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
require 'tokyotyrant'
|
|
82
|
+
|
|
83
|
+
rdb = TokyoTyrant::RDBTBL::new
|
|
84
|
+
rdb.open("127.0.0.1", 45001)
|
|
85
|
+
rdb.clear
|
|
86
|
+
|
|
87
|
+
2.times { puts }
|
|
88
|
+
puts 'TokyoTyrant::RDB (Ruby)'
|
|
89
|
+
|
|
90
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
91
|
+
b.report('inserting data') do
|
|
92
|
+
data1.each_with_index { |e, i| rdb.put(i.to_s, e) }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
b.report('reading data') do
|
|
96
|
+
data1.each_with_index { |e, i| nothing = rdb.get(i.to_s) }
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
require 'tokyo_tyrant'
|
|
101
|
+
t = TokyoTyrant::Table.new('127.0.0.1', 45001)
|
|
102
|
+
t.clear
|
|
103
|
+
|
|
104
|
+
2.times { puts }
|
|
105
|
+
puts 'TokyoTyrant (C)'
|
|
106
|
+
|
|
107
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
108
|
+
b.report('inserting data') do
|
|
109
|
+
data1.each_with_index { |e, i| t[i] = e }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
b.report('reading data') do
|
|
113
|
+
data1.each_with_index { |e, i| nothing = t[i] }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
b.report('bulk writing data') do
|
|
117
|
+
nothing = t.mput(data_h)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
b.report('bulk reading data') do
|
|
121
|
+
nothing = t.mget(0..9999)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
require 'memcached'
|
|
126
|
+
m = Memcached.new('127.0.0.1:11211')
|
|
127
|
+
m.flush
|
|
128
|
+
|
|
129
|
+
2.times { puts }
|
|
130
|
+
puts 'Memcached (C)'
|
|
131
|
+
|
|
132
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
133
|
+
b.report('inserting data') do
|
|
134
|
+
data1.each_with_index { |e, i| m.set(i.to_s, e) }
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
b.report('reading data') do
|
|
138
|
+
data1.each_with_index { |e, i| nothing = m.get(i.to_s) }
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
b.report('bulk reading data') do
|
|
142
|
+
nothing = m.get((0..9999).to_a.collect{|i| i.to_s})
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
require 'memcache'
|
|
147
|
+
mc = MemCache.new('127.0.0.1:11211')
|
|
148
|
+
mc.flush_all
|
|
149
|
+
|
|
150
|
+
2.times { puts }
|
|
151
|
+
puts 'MemCache (Ruby)'
|
|
152
|
+
|
|
153
|
+
Benchmark.benchmark(' ' * 20 + Benchmark::Tms::CAPTION, 20) do |b|
|
|
154
|
+
b.report('inserting data') do
|
|
155
|
+
data1.each_with_index { |e, i| mc.set(i.to_s, e) }
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
b.report('reading data') do
|
|
159
|
+
data1.each_with_index { |e, i| nothing = mc.get(i.to_s) }
|
|
160
|
+
end
|
|
161
|
+
end
|