oklahoma_mixer 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,335 @@
1
+ module OklahomaMixer
2
+ class HashDatabase
3
+ #################
4
+ ### Constants ###
5
+ #################
6
+
7
+ MODES = { "r" => :HDBOREADER,
8
+ "w" => :HDBOWRITER,
9
+ "c" => :HDBOCREAT,
10
+ "t" => :HDBOTRUNC,
11
+ "e" => :HDBONOLCK,
12
+ "f" => :HDBOLCKNB,
13
+ "s" => :HDBOTSYNC }
14
+ OPTS = { "l" => :HDBTLARGE,
15
+ "d" => :HDBTDEFLATE,
16
+ "b" => :HDBTBZIP,
17
+ "t" => :HDBTTCBS }
18
+
19
+ ###################
20
+ ### File System ###
21
+ ###################
22
+
23
+ def initialize(path, *args)
24
+ options = args.last.is_a?(Hash) ? args.last : { }
25
+ mode = !args.first.is_a?(Hash) ? args.first : nil
26
+ @path = path
27
+ @db = C.new
28
+ self.default = options[:default]
29
+ @in_transaction = false
30
+ @abort = false
31
+ @nested_transactions = options[:nested_transactions]
32
+
33
+ try(:setmutex) if options[:mutex]
34
+ if options.values_at(:bnum, :apow, :fpow, :opts).any?
35
+ optimize(options.merge(:tune => true))
36
+ end
37
+ {:rcnum => :cache, :xmsiz => nil, :dfunit => nil}.each do |option, func|
38
+ if i = options[option]
39
+ try("set#{func || option}", i.to_i)
40
+ end
41
+ end
42
+
43
+ warn "mode option supersedes mode argument" if mode and options[:mode]
44
+ try(:open, path, to_enum_int(options.fetch(:mode, mode || "wc"), :mode))
45
+ end
46
+
47
+ def optimize(options)
48
+ try( options[:tune] ? :tune : :optimize,
49
+ options.fetch(:bnum, 0).to_i,
50
+ options.fetch(:apow, -1).to_i,
51
+ options.fetch(:fpow, -1).to_i,
52
+ to_enum_int(options.fetch(:opts, 0xFF), :opt) )
53
+ end
54
+
55
+ attr_reader :path
56
+
57
+ def file_size
58
+ C.fsiz(@db)
59
+ end
60
+
61
+ def flush
62
+ try(:sync)
63
+ end
64
+ alias_method :sync, :flush
65
+ alias_method :fsync, :flush
66
+
67
+ def copy(path)
68
+ try(:copy, path)
69
+ end
70
+ alias_method :backup, :copy
71
+
72
+ def defrag(steps = 0)
73
+ try(:defrag, steps.to_i)
74
+ end
75
+
76
+ def close
77
+ try(:del) # closes before it deletes the object
78
+ end
79
+
80
+ ################################
81
+ ### Getting and Setting Keys ###
82
+ ################################
83
+
84
+ def default(key = nil)
85
+ @default[key] if @default
86
+ end
87
+
88
+ def default=(value_or_proc)
89
+ @default = case value_or_proc
90
+ when Proc then value_or_proc
91
+ when nil then nil
92
+ else lambda { |key| value_or_proc }
93
+ end
94
+ end
95
+
96
+ def store(key, value, mode = nil)
97
+ k, v = key.to_s, value.to_s
98
+ result = value
99
+ if block_given?
100
+ warn "block supersedes mode argument" unless mode.nil?
101
+ callback = lambda { |old_value_pointer, old_size, returned_size, _|
102
+ old_value = old_value_pointer.get_bytes(0, old_size)
103
+ replacement = yield(key, old_value, value).to_s
104
+ returned_size.put_int(0, replacement.size)
105
+ FFI::MemoryPointer.from_string(replacement)
106
+ }
107
+ try(:putproc, k, k.size, v, v.size, callback, nil)
108
+ else
109
+ case mode
110
+ when :keep
111
+ result = try( :putkeep, k, k.size, v, v.size,
112
+ :no_error => {21 => false} )
113
+ when :cat
114
+ try(:putcat, k, k.size, v, v.size)
115
+ when :async
116
+ try(:putasync, k, k.size, v, v.size)
117
+ when :add
118
+ result = case value
119
+ when Float then try( :adddouble, k, k.size, value,
120
+ :failure => lambda { |n| n.nan? } )
121
+ else try( :addint, k, k.size, value.to_i,
122
+ :failure => Utilities::INT_MIN )
123
+ end
124
+ else
125
+ try(:put, k, k.size, v, v.size)
126
+ end
127
+ end
128
+ result
129
+ end
130
+ alias_method :[]=, :store
131
+
132
+ def fetch(key, *default)
133
+ k = key.to_s
134
+ if value = try( :read_from_func, :get, k, k.size,
135
+ :failure => nil, :no_error => {22 => nil} )
136
+ value
137
+ else
138
+ if block_given?
139
+ warn "block supersedes default value argument" unless default.empty?
140
+ yield key
141
+ elsif not default.empty?
142
+ default.first
143
+ else
144
+ fail IndexError, "key not found"
145
+ end
146
+ end
147
+ end
148
+
149
+ def [](key)
150
+ fetch(key, &@default)
151
+ rescue IndexError
152
+ nil
153
+ end
154
+
155
+ def update(hash, &dup_handler)
156
+ hash.each do |key, value|
157
+ store(key, value, &dup_handler)
158
+ end
159
+ self
160
+ end
161
+
162
+ def values_at(*keys)
163
+ keys.map { |key| self[key] }
164
+ end
165
+
166
+ def keys(options = { })
167
+ prefix = options.fetch(:prefix, "").to_s
168
+ limit = options.fetch(:limit, -1)
169
+ list = ArrayList.new(C.fwmkeys(@db, prefix, prefix.size, limit))
170
+ list.to_a
171
+ ensure
172
+ list.free if list
173
+ end
174
+
175
+ def values
176
+ values = [ ]
177
+ each_value do |value|
178
+ values << value
179
+ end
180
+ values
181
+ end
182
+
183
+ def delete(key, &missing_handler)
184
+ value = fetch(key, &missing_handler)
185
+ k = key.to_s
186
+ try(:out, k, k.size, :no_error => {22 => nil})
187
+ value
188
+ rescue IndexError
189
+ nil
190
+ end
191
+
192
+ def clear
193
+ try(:vanish)
194
+ self
195
+ end
196
+
197
+ def include?(key)
198
+ fetch(key)
199
+ true
200
+ rescue IndexError
201
+ false
202
+ end
203
+ alias_method :has_key?, :include?
204
+ alias_method :key?, :include?
205
+ alias_method :member?, :include?
206
+
207
+ def size
208
+ C.rnum(@db)
209
+ end
210
+ alias_method :length, :size
211
+
212
+ #################
213
+ ### Iteration ###
214
+ #################
215
+
216
+ include Enumerable
217
+
218
+ def each_key
219
+ try(:iterinit)
220
+ loop do
221
+ return self unless key = try( :read_from_func, :iternext,
222
+ :failure => nil,
223
+ :no_error => {22 => nil} )
224
+ yield key
225
+ end
226
+ end
227
+
228
+ def each
229
+ try(:iterinit)
230
+ loop do
231
+ Utilities.temp_xstr do |key|
232
+ Utilities.temp_xstr do |value|
233
+ return self unless try( :iternext3, key.pointer, value.pointer,
234
+ :no_error => {22 => false} )
235
+ yield [key.to_s, value.to_s]
236
+ end
237
+ end
238
+ end
239
+ end
240
+ alias_method :each_pair, :each
241
+
242
+ def each_value
243
+ each do |key, value|
244
+ yield value
245
+ end
246
+ end
247
+
248
+ def delete_if
249
+ each do |key, value|
250
+ delete(key) if yield key, value
251
+ end
252
+ end
253
+
254
+ ####################
255
+ ### Transactions ###
256
+ ####################
257
+
258
+ def transaction
259
+ if @in_transaction
260
+ case @nested_transactions
261
+ when :ignore
262
+ return yield
263
+ when :fail, :raise
264
+ fail Error::TransactionError, "nested transaction"
265
+ end
266
+ end
267
+
268
+ @in_transaction = true
269
+ @abort = false
270
+
271
+ begin
272
+ catch(:finish_transaction) do
273
+ try(:tranbegin)
274
+ yield
275
+ end
276
+ rescue Exception
277
+ @abort = true
278
+ raise
279
+ ensure
280
+ try("tran#{@abort ? :abort : :commit}")
281
+ @in_transaction = false
282
+ end
283
+ end
284
+
285
+ def commit
286
+ fail Error::TransactionError, "not in transaction" unless @in_transaction
287
+ throw :finish_transaction
288
+ end
289
+
290
+ def abort
291
+ fail Error::TransactionError, "not in transaction" unless @in_transaction
292
+ @abort = true
293
+ throw :finish_transaction
294
+ end
295
+
296
+ #######
297
+ private
298
+ #######
299
+
300
+ def try(func, *args)
301
+ options = args.last.is_a?(Hash) ? args.pop : { }
302
+ failure = options.fetch(:failure, false)
303
+ no_error = options.fetch(:no_error, { })
304
+ result = func == :read_from_func ?
305
+ C.read_from_func(args[0], @db, *args[1..-1]) :
306
+ C.send(func, @db, *args)
307
+ if (failure.is_a?(Proc) and failure[result]) or result == failure
308
+ error_code = C.ecode(@db)
309
+ if no_error.include? error_code
310
+ no_error[error_code]
311
+ else
312
+ error_message = C.errmsg(error_code)
313
+ fail Error::CabinetError, "#{error_message} (#{error_code})"
314
+ end
315
+ else
316
+ result
317
+ end
318
+ end
319
+
320
+ def to_enum_int(str_or_int, name)
321
+ return str_or_int if str_or_int.is_a? Integer
322
+ const = "#{name.to_s.upcase}S"
323
+ names = self.class.const_get(const)
324
+ enum = C.const_get(const)
325
+ str_or_int.to_s.downcase.scan(/./m).inject(0) do |int, c|
326
+ if n = names[c]
327
+ int | enum[n]
328
+ else
329
+ warn "skipping unrecognized #{name}"
330
+ int
331
+ end
332
+ end
333
+ end
334
+ end
335
+ end
@@ -0,0 +1,88 @@
1
+ module OklahomaMixer
2
+ module Utilities # :nodoc:
3
+ module FFIDSL # :nodoc:
4
+ def self.extended(ffi_interface)
5
+ ffi_interface.extend(FFI::Library)
6
+ ffi_interface.ffi_lib(
7
+ *Array(
8
+ ENV.fetch(
9
+ "TOKYO_CABINET_LIB",
10
+ Dir["/{opt,usr}/{,local/}lib{,64}/libtokyocabinet.{dylib,so*}"]
11
+ )
12
+ )
13
+ )
14
+ rescue LoadError
15
+ fail "Tokyo Cabinet could not be loaded " +
16
+ "(you can install it from http://1978th.net/ " +
17
+ "and set the TOKYO_CABINET_LIB environment variable to its path)"
18
+ end
19
+
20
+ def prefix(new_prefix = nil)
21
+ @prefix = new_prefix unless new_prefix.nil?
22
+ defined?(@prefix) and @prefix
23
+ end
24
+
25
+ def func(details)
26
+ args = [ ]
27
+ args << details.fetch(:alias, details[:name])
28
+ args << "#{prefix}#{details[:name]}".to_sym
29
+ args << Array(details[:args])
30
+ args << details.fetch(:returns, :void)
31
+ attach_function(*args)
32
+ end
33
+
34
+ def read_from_func(func, *args)
35
+ Utilities.temp_int do |size|
36
+ begin
37
+ args << size
38
+ pointer = send(func, *args)
39
+ pointer.address.zero? ? nil : pointer.get_bytes(0, size.get_int(0))
40
+ ensure
41
+ Utilities.free(pointer) if pointer
42
+ end
43
+ end
44
+ end
45
+
46
+ def call(details)
47
+ args = [ ]
48
+ args << details[:name]
49
+ args << Array(details[:args])
50
+ args << details.fetch(:returns, :void)
51
+ callback(*args)
52
+ end
53
+
54
+ def def_new_and_del_funcs
55
+ func :name => :new,
56
+ :returns => :pointer
57
+ func :name => :del,
58
+ :args => :pointer
59
+ end
60
+ end
61
+
62
+ int_min = `getconf INT_MIN 2>&1`[/-\d+/]
63
+ unless INT_MIN = int_min && int_min.to_i
64
+ warn "set OKMixer::Utilities::INT_MIN before using counters"
65
+ end
66
+
67
+ def self.temp_int
68
+ int = FFI::MemoryPointer.new(:int)
69
+ yield int
70
+ ensure
71
+ int.free if int
72
+ end
73
+
74
+ def self.temp_xstr
75
+ xstr = ExtensibleString.new
76
+ yield xstr
77
+ ensure
78
+ xstr.free if xstr
79
+ end
80
+
81
+ extend FFIDSL
82
+
83
+ prefix :tc
84
+
85
+ func :name => :free,
86
+ :args => :pointer
87
+ end
88
+ end
@@ -0,0 +1,33 @@
1
+ require "ffi"
2
+
3
+ require "oklahoma_mixer/error"
4
+ require "oklahoma_mixer/utilities"
5
+
6
+ require "oklahoma_mixer/extensible_string/c"
7
+ require "oklahoma_mixer/extensible_string"
8
+ require "oklahoma_mixer/array_list/c"
9
+ require "oklahoma_mixer/array_list"
10
+
11
+ require "oklahoma_mixer/hash_database/c"
12
+ require "oklahoma_mixer/hash_database"
13
+
14
+ module OklahomaMixer
15
+ VERSION = "0.1.0"
16
+
17
+ def self.open(path, *args)
18
+ db = case File.extname(path).downcase
19
+ when ".tch" then HashDatabase.new(path, *args)
20
+ else fail ArgumentError, "unsupported database type"
21
+ end
22
+ if block_given?
23
+ begin
24
+ yield db
25
+ ensure
26
+ db.close
27
+ end
28
+ else
29
+ db
30
+ end
31
+ end
32
+ end
33
+ OKMixer = OklahomaMixer
@@ -0,0 +1,41 @@
1
+ require "test_helper"
2
+
3
+ class TestBinaryData < Test::Unit::TestCase
4
+ def setup
5
+ @db = hdb
6
+ @key = "Binary\0Name"
7
+ @value = "James\0Edward\0Gray\0II"
8
+ @db[@key] = @value
9
+ end
10
+
11
+ def teardown
12
+ @db.close
13
+ remove_db_files
14
+ end
15
+
16
+ def test_keys_and_values_can_be_read_with_null_bytes
17
+ assert_equal(@value, @db[@key])
18
+ end
19
+
20
+ def test_null_bytes_are_preservered_by_update_callback
21
+ @db.update(@key => "conflict") { |key, old_value, new_value| "new\0value" }
22
+ assert_equal("new\0value", @db[@key])
23
+ end
24
+
25
+ def test_null_bytes_are_preserved_during_key_iteration
26
+ @db.each_key do |key|
27
+ assert_equal(@key, key)
28
+ end
29
+ end
30
+
31
+ def test_null_bytes_are_preserved_during_iteration
32
+ @db.each do |key, value|
33
+ assert_equal(@key, key)
34
+ assert_equal(@value, value)
35
+ end
36
+ end
37
+
38
+ def test_null_bytes_are_preserved_by_key_listing
39
+ assert_equal([@key], @db.keys)
40
+ end
41
+ end
@@ -0,0 +1,74 @@
1
+ require "test_helper"
2
+
3
+ class TestFileSystem < Test::Unit::TestCase
4
+ def setup
5
+ @db = hdb
6
+ end
7
+
8
+ def teardown
9
+ @db.close
10
+ remove_db_files
11
+ end
12
+
13
+ def test_creating_a_hash_database_creates_the_corresponding_file
14
+ assert(File.exist?(db_path("tch")), "The HashDatabase file was not created")
15
+ end
16
+
17
+ def test_path_returns_the_path_the_database_was_created_with
18
+ assert_equal(db_path("tch"), @db.path)
19
+ end
20
+
21
+ def test_file_size_returns_the_size_of_the_contents_on_disk
22
+ @db[:data] = "X" * 1024
23
+ assert_operator(@db.file_size, :>, 1024)
24
+ end
25
+
26
+ def test_flush_and_aliases_force_contents_to_be_written_to_disk
27
+ @db[:data] = "X" * 1024
28
+ @db.flush
29
+ assert_operator(File.size(@db.path), :>, 1024)
30
+ @db[:more_data] = "X" * 1024
31
+ @db.sync
32
+ assert_operator(File.size(@db.path), :>, 2048)
33
+ @db[:even_more_data] = "X" * 1024
34
+ @db.fsync
35
+ assert_operator(File.size(@db.path), :>, 3072)
36
+ end
37
+
38
+ def test_copy_and_backup_create_a_backup_copy_of_the_database_file
39
+ @db[:data] = "X" * 1024
40
+ old_path = @db.path
41
+ old_ext = File.extname(old_path)
42
+ new_path = "#{File.basename(old_path, old_ext)}_2#{old_ext}"
43
+ new_path_2 = "#{File.basename(old_path, old_ext)}_3#{old_ext}"
44
+ assert(@db.copy(new_path), "Database could not be copied")
45
+ new_db = OKMixer::HashDatabase.new(new_path)
46
+ assert_equal(@db.to_a.sort, new_db.to_a.sort)
47
+ assert(@db.backup(new_path_2), "Database could not be copied")
48
+ new_db_2 = OKMixer::HashDatabase.new(new_path_2)
49
+ assert_equal(@db.to_a.sort, new_db_2.to_a.sort)
50
+ ensure
51
+ File.unlink(new_path) if new_path and File.exist? new_path
52
+ File.unlink(new_path_2) if new_path_2 and File.exist? new_path_2
53
+ new_db.close if new_db
54
+ new_db_2.close if new_db_2
55
+ end
56
+
57
+ def test_defrag_removes_wholes_in_the_database_file
58
+ # load some data
59
+ data = "X" * 1024
60
+ 100.times do |i|
61
+ @db[i] = data
62
+ end
63
+ # delete some data
64
+ (0...100).sort_by { rand }.first(10).each do |i|
65
+ @db.delete(i)
66
+ end
67
+ # push changes to disk
68
+ @db.flush
69
+ # defragment the database file
70
+ old_size = File.size(@db.path)
71
+ @db.defrag
72
+ assert_operator(old_size, :>, File.size(@db.path))
73
+ end
74
+ end