oklahoma_mixer 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.rdoc +10 -0
- data/TODO.rdoc +3 -3
- data/lib/oklahoma_mixer.rb +14 -5
- data/lib/oklahoma_mixer/b_tree_database.rb +185 -0
- data/lib/oklahoma_mixer/b_tree_database/c.rb +45 -0
- data/lib/oklahoma_mixer/cursor.rb +43 -0
- data/lib/oklahoma_mixer/cursor/c.rb +45 -0
- data/lib/oklahoma_mixer/fixed_length_database.rb +95 -0
- data/lib/oklahoma_mixer/fixed_length_database/c.rb +62 -0
- data/lib/oklahoma_mixer/hash_database.rb +84 -54
- data/lib/oklahoma_mixer/hash_database/c.rb +3 -96
- data/lib/oklahoma_mixer/utilities.rb +110 -5
- data/test/b_tree_binary_data_test.rb +45 -0
- data/test/b_tree_tuning_test.rb +132 -0
- data/test/binary_data_test.rb +3 -13
- data/test/cursor_based_iteration_test.rb +151 -0
- data/test/duplicate_storage_test.rb +107 -0
- data/test/fixed_length_tuning_test.rb +77 -0
- data/test/getting_and_setting_by_id_test.rb +288 -0
- data/test/getting_and_setting_keys_test.rb +37 -3
- data/test/iteration_test.rb +2 -76
- data/test/key_range_test.rb +161 -0
- data/test/order_test.rb +122 -0
- data/test/shared_binary_data.rb +14 -0
- data/test/shared_iteration.rb +99 -0
- data/test/shared_tuning.rb +91 -0
- data/test/test_helper.rb +10 -11
- data/test/top_level_interface_test.rb +12 -0
- data/test/tuning_test.rb +11 -87
- metadata +27 -2
data/CHANGELOG.rdoc
CHANGED
@@ -2,6 +2,16 @@
|
|
2
2
|
|
3
3
|
Below is a complete listing of changes for each revision of Oklahoma Mixer.
|
4
4
|
|
5
|
+
== 0.2.0
|
6
|
+
|
7
|
+
* Added a to_hash() iterator that can preserve defaults
|
8
|
+
* Added B+Tree Database support (order functions, key ranges,
|
9
|
+
and duplicate storages)
|
10
|
+
* Added Fixed-length Database (indexing and iterating by ID's, support for
|
11
|
+
special ID constants, and size limits)
|
12
|
+
* Fixed a bug that could cause Ruby to crash during GC after storing a value
|
13
|
+
with the duplication handler block
|
14
|
+
|
5
15
|
== 0.1.0
|
6
16
|
|
7
17
|
* Initial public release with Hash Database support
|
data/TODO.rdoc
CHANGED
@@ -3,9 +3,9 @@
|
|
3
3
|
The following is a list of planned expansions for Oklahoma Mixer in the order I
|
4
4
|
intend to address them.
|
5
5
|
|
6
|
-
1. Add support for
|
7
|
-
2.
|
8
|
-
3.
|
6
|
+
1. Add support for Table Databases
|
7
|
+
2. Ensure Ruby 1.9 compatibility
|
8
|
+
3. Write API documentation
|
9
9
|
4. Add support for Tokyo Tyrant
|
10
10
|
5. Add support for Tokyo Dystopia
|
11
11
|
6. Include some higher level abstractions like mixed tables, queues, and shards
|
data/lib/oklahoma_mixer.rb
CHANGED
@@ -7,18 +7,27 @@ require "oklahoma_mixer/extensible_string/c"
|
|
7
7
|
require "oklahoma_mixer/extensible_string"
|
8
8
|
require "oklahoma_mixer/array_list/c"
|
9
9
|
require "oklahoma_mixer/array_list"
|
10
|
+
require "oklahoma_mixer/cursor/c"
|
11
|
+
require "oklahoma_mixer/cursor"
|
10
12
|
|
11
13
|
require "oklahoma_mixer/hash_database/c"
|
12
14
|
require "oklahoma_mixer/hash_database"
|
15
|
+
require "oklahoma_mixer/b_tree_database/c"
|
16
|
+
require "oklahoma_mixer/b_tree_database"
|
17
|
+
require "oklahoma_mixer/fixed_length_database/c"
|
18
|
+
require "oklahoma_mixer/fixed_length_database"
|
13
19
|
|
14
20
|
module OklahomaMixer
|
15
|
-
VERSION = "0.
|
21
|
+
VERSION = "0.2.0"
|
16
22
|
|
17
23
|
def self.open(path, *args)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
24
|
+
db_class = case File.extname(path).downcase
|
25
|
+
when ".tch" then HashDatabase
|
26
|
+
when ".tcb" then BTreeDatabase
|
27
|
+
when ".tcf" then FixedLengthDatabase
|
28
|
+
else fail ArgumentError, "unsupported database type"
|
29
|
+
end
|
30
|
+
db = db_class.new(path, *args)
|
22
31
|
if block_given?
|
23
32
|
begin
|
24
33
|
yield db
|
@@ -0,0 +1,185 @@
|
|
1
|
+
module OklahomaMixer
|
2
|
+
class BTreeDatabase < HashDatabase
|
3
|
+
###################
|
4
|
+
### File System ###
|
5
|
+
###################
|
6
|
+
|
7
|
+
def optimize(options)
|
8
|
+
try( options[:tune] ? :tune : :optimize,
|
9
|
+
options.fetch(:lmemb, 0).to_i,
|
10
|
+
options.fetch(:nmemb, 0).to_i,
|
11
|
+
options.fetch(:bnum, 0).to_i,
|
12
|
+
options.fetch(:apow, -1).to_i,
|
13
|
+
options.fetch(:fpow, -1).to_i,
|
14
|
+
to_enum_int(options.fetch(:opts, 0xFF), :opt) )
|
15
|
+
end
|
16
|
+
|
17
|
+
################################
|
18
|
+
### Getting and Setting Keys ###
|
19
|
+
################################
|
20
|
+
|
21
|
+
def store(key, value, mode = nil)
|
22
|
+
if mode == :dup
|
23
|
+
try(:putdup, cast_key_in(key), cast_value_in(value))
|
24
|
+
value
|
25
|
+
else
|
26
|
+
super
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def keys(options = { })
|
31
|
+
if options.include? :range
|
32
|
+
warn "range supersedes prefix" if options[:prefix]
|
33
|
+
range = options[:range]
|
34
|
+
fail ArgumentError, "Range expected" unless range.is_a? Range
|
35
|
+
start = cast_key_in(range.first)
|
36
|
+
include_start = !options.fetch(:exclude_start, false)
|
37
|
+
finish = cast_key_in(range.last)
|
38
|
+
include_finish = !range.exclude_end?
|
39
|
+
limit = options.fetch(:limit, -1)
|
40
|
+
begin
|
41
|
+
list = ArrayList.new( lib.range( @db,
|
42
|
+
*[ start, include_start,
|
43
|
+
finish, include_finish,
|
44
|
+
limit ].flatten ) )
|
45
|
+
list.to_a
|
46
|
+
ensure
|
47
|
+
list.free if list
|
48
|
+
end
|
49
|
+
else
|
50
|
+
super
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def values(key = nil)
|
55
|
+
if key.nil?
|
56
|
+
super()
|
57
|
+
else
|
58
|
+
begin
|
59
|
+
pointer = try( :get4, cast_key_in(key),
|
60
|
+
:failure => lambda { |ptr| ptr.address.zero? },
|
61
|
+
:no_error => {22 => nil} )
|
62
|
+
if pointer.nil?
|
63
|
+
[ ]
|
64
|
+
else
|
65
|
+
list = ArrayList.new(pointer)
|
66
|
+
list.to_a
|
67
|
+
end
|
68
|
+
ensure
|
69
|
+
list.free if list
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def delete(key, mode = nil, &missing_handler)
|
75
|
+
if mode == :dup
|
76
|
+
values = values(key)
|
77
|
+
if try(:out3, cast_key_in(key), :no_error => {22 => false})
|
78
|
+
values
|
79
|
+
else
|
80
|
+
missing_handler ? missing_handler[key] : values
|
81
|
+
end
|
82
|
+
else
|
83
|
+
super(key, &missing_handler)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def size(key = nil)
|
88
|
+
if key.nil?
|
89
|
+
super()
|
90
|
+
else
|
91
|
+
try(:vnum, cast_key_in(key), :failure => 0, :no_error => {22 => 0})
|
92
|
+
end
|
93
|
+
end
|
94
|
+
alias_method :length, :size
|
95
|
+
|
96
|
+
#################
|
97
|
+
### Iteration ###
|
98
|
+
#################
|
99
|
+
|
100
|
+
def each_key(start = nil)
|
101
|
+
cursor_in_loop(start) do |iterator|
|
102
|
+
throw(:finish_iteration) unless key = iterator.key
|
103
|
+
yield key
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def each(start = nil)
|
108
|
+
cursor_in_loop(start) do |iterator|
|
109
|
+
throw(:finish_iteration) unless key_and_value = iterator.key_and_value
|
110
|
+
yield key_and_value
|
111
|
+
end
|
112
|
+
end
|
113
|
+
alias_method :each_pair, :each
|
114
|
+
|
115
|
+
def reverse_each(start = nil)
|
116
|
+
cursor_in_loop(start, :reverse) do |iterator|
|
117
|
+
throw(:finish_iteration) unless key_and_value = iterator.key_and_value
|
118
|
+
yield key_and_value
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def each_value(start = nil)
|
123
|
+
cursor_in_loop(start) do |iterator|
|
124
|
+
throw(:finish_iteration) unless value = iterator.value
|
125
|
+
yield value
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def delete_if(start = nil)
|
130
|
+
cursor(start) do |iterator|
|
131
|
+
loop do
|
132
|
+
break unless key_and_value = iterator.key_and_value
|
133
|
+
break unless iterator.send(yield(*key_and_value) ? :delete : :next)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
#######
|
139
|
+
private
|
140
|
+
#######
|
141
|
+
|
142
|
+
def tune(options)
|
143
|
+
super
|
144
|
+
if cmpfunc = options[:cmpfunc]
|
145
|
+
callback = lambda { |a_pointer, a_size, b_pointer, b_size, _|
|
146
|
+
a = a_pointer.get_bytes(0, a_size)
|
147
|
+
b = b_pointer.get_bytes(0, b_size)
|
148
|
+
cmpfunc[a, b]
|
149
|
+
}
|
150
|
+
try(:setcmpfunc, callback, nil)
|
151
|
+
end
|
152
|
+
if options.values_at(:lmemb, :nmemb, :bnum, :apow, :fpow, :opts).any?
|
153
|
+
optimize(options.merge(:tune => true))
|
154
|
+
end
|
155
|
+
if options.values_at(:lcnum, :ncnum).any?
|
156
|
+
setcache(options)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def setcache(options)
|
161
|
+
try( :setcache,
|
162
|
+
options.fetch(:lcnum, 0).to_i,
|
163
|
+
options.fetch(:ncnum, 0).to_i )
|
164
|
+
end
|
165
|
+
|
166
|
+
def cursor(start = nil, reverse = false)
|
167
|
+
cursor = Cursor.new(@db, start.nil? ? start : cast_key_in(start), reverse)
|
168
|
+
yield cursor
|
169
|
+
self
|
170
|
+
ensure
|
171
|
+
cursor.free if cursor
|
172
|
+
end
|
173
|
+
|
174
|
+
def cursor_in_loop(start = nil, reverse = false)
|
175
|
+
cursor(start, reverse) do |iterator|
|
176
|
+
catch(:finish_iteration) do
|
177
|
+
loop do
|
178
|
+
yield iterator
|
179
|
+
break unless iterator.next
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module OklahomaMixer
|
2
|
+
class BTreeDatabase < HashDatabase
|
3
|
+
module C # :nodoc:
|
4
|
+
extend OklahomaMixer::Utilities::FFIDSL
|
5
|
+
|
6
|
+
prefix :tcbdb
|
7
|
+
|
8
|
+
def_hash_database_consts_and_funcs
|
9
|
+
|
10
|
+
call :name => :TCCMP,
|
11
|
+
:args => [:pointer, :int, :pointer, :int, :pointer],
|
12
|
+
:returns => :int
|
13
|
+
func :name => :setcmpfunc,
|
14
|
+
:args => [:pointer, :TCCMP, :pointer],
|
15
|
+
:returns => :bool
|
16
|
+
func :name => :tune,
|
17
|
+
:args => [:pointer, :int32, :int32, :int64, :int8, :int8, OPTS],
|
18
|
+
:returns => :bool
|
19
|
+
func :name => :setcache,
|
20
|
+
:args => [:pointer, :int32, :int32],
|
21
|
+
:returns => :bool
|
22
|
+
func :name => :optimize,
|
23
|
+
:args => [:pointer, :int32, :int32, :int64, :int8, :int8, OPTS],
|
24
|
+
:returns => :bool
|
25
|
+
|
26
|
+
func :name => :putdup,
|
27
|
+
:args => [:pointer, :pointer, :int, :pointer, :int],
|
28
|
+
:returns => :bool
|
29
|
+
func :name => :out3,
|
30
|
+
:args => [:pointer, :pointer, :int],
|
31
|
+
:returns => :bool
|
32
|
+
func :name => :get4,
|
33
|
+
:args => [:pointer, :pointer, :int],
|
34
|
+
:returns => :pointer
|
35
|
+
func :name => :vnum,
|
36
|
+
:args => [:pointer, :pointer, :int],
|
37
|
+
:returns => :int
|
38
|
+
|
39
|
+
func :name => :range,
|
40
|
+
:args => [:pointer, :pointer, :int, :bool, :pointer, :int, :bool,
|
41
|
+
:int],
|
42
|
+
:returns => :pointer
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module OklahomaMixer
|
2
|
+
class Cursor # :nodoc:
|
3
|
+
def initialize(b_tree_pointer, start = nil, reverse = false)
|
4
|
+
@pointer = C.new(b_tree_pointer)
|
5
|
+
@reverse = reverse
|
6
|
+
if start
|
7
|
+
C.jump(@pointer, *start)
|
8
|
+
else
|
9
|
+
C.send(@reverse ? :last : :first, @pointer)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def key
|
14
|
+
C.read_from_func(:key, @pointer)
|
15
|
+
end
|
16
|
+
|
17
|
+
def value
|
18
|
+
C.read_from_func(:val, @pointer)
|
19
|
+
end
|
20
|
+
|
21
|
+
def key_and_value
|
22
|
+
Utilities.temp_xstr do |key|
|
23
|
+
Utilities.temp_xstr do |value|
|
24
|
+
if C.rec(@pointer, key.pointer, value.pointer)
|
25
|
+
[key.to_s, value.to_s]
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def next
|
32
|
+
C.send(@reverse ? :prev : :next, @pointer)
|
33
|
+
end
|
34
|
+
|
35
|
+
def delete
|
36
|
+
C.out(@pointer)
|
37
|
+
end
|
38
|
+
|
39
|
+
def free
|
40
|
+
C.del(@pointer)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module OklahomaMixer
|
2
|
+
class Cursor
|
3
|
+
module C # :nodoc:
|
4
|
+
extend OklahomaMixer::Utilities::FFIDSL
|
5
|
+
|
6
|
+
prefix :tcbdbcur
|
7
|
+
|
8
|
+
func :name => :new,
|
9
|
+
:args => :pointer,
|
10
|
+
:returns => :pointer
|
11
|
+
func :name => :del,
|
12
|
+
:args => :pointer
|
13
|
+
|
14
|
+
func :name => :first,
|
15
|
+
:args => :pointer,
|
16
|
+
:returns => :bool
|
17
|
+
func :name => :last,
|
18
|
+
:args => :pointer,
|
19
|
+
:returns => :bool
|
20
|
+
func :name => :jump,
|
21
|
+
:args => [:pointer, :pointer, :int],
|
22
|
+
:returns => :bool
|
23
|
+
func :name => :next,
|
24
|
+
:args => :pointer,
|
25
|
+
:returns => :bool
|
26
|
+
func :name => :prev,
|
27
|
+
:args => :pointer,
|
28
|
+
:returns => :bool
|
29
|
+
|
30
|
+
func :name => :key,
|
31
|
+
:args => [:pointer, :pointer],
|
32
|
+
:returns => :pointer
|
33
|
+
func :name => :val,
|
34
|
+
:args => [:pointer, :pointer],
|
35
|
+
:returns => :pointer
|
36
|
+
func :name => :rec,
|
37
|
+
:args => [:pointer, :pointer, :pointer],
|
38
|
+
:returns => :bool
|
39
|
+
|
40
|
+
func :name => :out,
|
41
|
+
:args => :pointer,
|
42
|
+
:returns => :bool
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
module OklahomaMixer
|
2
|
+
class FixedLengthDatabase < HashDatabase
|
3
|
+
###################
|
4
|
+
### File System ###
|
5
|
+
###################
|
6
|
+
|
7
|
+
def optimize(options)
|
8
|
+
try( options[:tune] ? :tune : :optimize,
|
9
|
+
options.fetch(:width, 0).to_i,
|
10
|
+
options.fetch(:limsiz, 0).to_i )
|
11
|
+
end
|
12
|
+
|
13
|
+
def defrag(steps = 0)
|
14
|
+
# do nothing: not needed, but provided for a consistent interface
|
15
|
+
end
|
16
|
+
|
17
|
+
################################
|
18
|
+
### Getting and Setting Keys ###
|
19
|
+
################################
|
20
|
+
|
21
|
+
def keys(options = { })
|
22
|
+
if options.include? :range
|
23
|
+
warn "range supersedes prefix" if options[:prefix]
|
24
|
+
range = options[:range]
|
25
|
+
unless range.respond_to?(:first) and range.respond_to?(:last)
|
26
|
+
fail ArgumentError, "Range or two element Array expected"
|
27
|
+
end
|
28
|
+
start = cast_key_in(range.first)
|
29
|
+
include_start = !options.fetch(:exclude_start, false)
|
30
|
+
finish = cast_key_in(range.last)
|
31
|
+
include_finish = !( range.respond_to?(:exclude_end?) ?
|
32
|
+
range.exclude_end? :
|
33
|
+
options.fetch(:exclude_end, false) )
|
34
|
+
else
|
35
|
+
fail ArgumentError, "prefix not supported" if options[:prefix]
|
36
|
+
start = cast_key_in(:min)
|
37
|
+
include_start = true
|
38
|
+
finish = cast_key_in(:max)
|
39
|
+
include_finish = true
|
40
|
+
end
|
41
|
+
limit = options.fetch(:limit, -1)
|
42
|
+
Utilities.temp_int do |count|
|
43
|
+
begin
|
44
|
+
list = lib.range(@db, start, finish, limit, count)
|
45
|
+
array = list.get_array_of_uint64(0, count.get_int(0))
|
46
|
+
array.shift if array.first == start and not include_start
|
47
|
+
array.pop if array.last == finish and not include_finish
|
48
|
+
array
|
49
|
+
ensure
|
50
|
+
Utilities.free(list) if list
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
#################
|
56
|
+
### Iteration ###
|
57
|
+
#################
|
58
|
+
|
59
|
+
def each_key
|
60
|
+
try(:iterinit)
|
61
|
+
loop do
|
62
|
+
return self unless key = try( :iternext,
|
63
|
+
:failure => 0,
|
64
|
+
:no_error => {22 => nil} )
|
65
|
+
yield key
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def each
|
70
|
+
each_key do |key|
|
71
|
+
yield [key, self[key]]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
alias_method :each_pair, :each
|
75
|
+
|
76
|
+
#######
|
77
|
+
private
|
78
|
+
#######
|
79
|
+
|
80
|
+
def tune(options)
|
81
|
+
if options.values_at(:width, :limsiz).any?
|
82
|
+
optimize(options.merge(:tune => true))
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def cast_key_in(key)
|
87
|
+
case key
|
88
|
+
when :min, :max, :prev, :next, "min", "max", "prev", "next"
|
89
|
+
C::IDS["FDBID#{key.to_s.upcase}".to_sym]
|
90
|
+
else
|
91
|
+
key.to_i
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|