daybreak 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +3 -0
- data/lib/daybreak/db.rb +119 -100
- data/lib/daybreak/format.rb +2 -2
- data/lib/daybreak/queue.rb +4 -106
- data/lib/daybreak/queue/mri.rb +69 -0
- data/lib/daybreak/queue/threaded.rb +39 -0
- data/lib/daybreak/version.rb +1 -1
- data/script/bench +14 -1
- data/test/prof.rb +3 -3
- data/test/test.rb +58 -17
- metadata +63 -50
data/.travis.yml
CHANGED
data/lib/daybreak/db.rb
CHANGED
@@ -5,19 +5,24 @@ module Daybreak
|
|
5
5
|
class DB
|
6
6
|
include Enumerable
|
7
7
|
|
8
|
-
#
|
9
|
-
|
10
|
-
|
8
|
+
# Database file name
|
9
|
+
attr_reader :file
|
10
|
+
|
11
|
+
# Counter of how many records are in
|
12
|
+
attr_reader :logsize
|
13
|
+
|
14
|
+
# Set default value, can be a callable
|
11
15
|
attr_writer :default
|
12
16
|
|
13
|
-
|
14
|
-
|
17
|
+
@@databases = []
|
18
|
+
@@databases_mutex = Mutex.new
|
15
19
|
|
16
20
|
# A handler that will ensure that databases are closed and synced when the
|
17
21
|
# current process exits.
|
18
|
-
|
22
|
+
# @api private
|
23
|
+
def self.exit_handler
|
19
24
|
loop do
|
20
|
-
db =
|
25
|
+
db = @@databases_mutex.synchronize { @@databases.first }
|
21
26
|
break unless db
|
22
27
|
warn "Daybreak database #{db.file} was not closed, state might be inconsistent"
|
23
28
|
begin
|
@@ -28,17 +33,7 @@ module Daybreak
|
|
28
33
|
end
|
29
34
|
end
|
30
35
|
|
31
|
-
|
32
|
-
# @api private
|
33
|
-
def register(db)
|
34
|
-
@databases_mutex.synchronize { @databases << db }
|
35
|
-
end
|
36
|
-
|
37
|
-
# @api private
|
38
|
-
def unregister(db)
|
39
|
-
@databases_mutex.synchronize { @databases.delete(db) }
|
40
|
-
end
|
41
|
-
end
|
36
|
+
at_exit { Daybreak::DB.exit_handler }
|
42
37
|
|
43
38
|
# Create a new Daybreak::DB. The second argument is the default value
|
44
39
|
# to store when accessing a previously unset key, this follows the
|
@@ -52,21 +47,21 @@ module Daybreak
|
|
52
47
|
@file = file
|
53
48
|
@serializer = (options[:serializer] || Serializer::Default).new
|
54
49
|
@format = (options[:format] || Format).new
|
55
|
-
@default = block ? block : options[:default]
|
56
50
|
@queue = Queue.new
|
57
|
-
@table =
|
51
|
+
@table = Hash.new(&method(:hash_default))
|
52
|
+
@default = block ? block : options[:default]
|
58
53
|
open
|
59
54
|
@mutex = Mutex.new # Mutex to make #lock thread safe
|
60
55
|
@worker = Thread.new(&method(:worker))
|
61
56
|
@worker.priority = -1
|
62
|
-
|
63
|
-
|
57
|
+
load
|
58
|
+
@@databases_mutex.synchronize { @@databases << self }
|
64
59
|
end
|
65
60
|
|
66
61
|
# Return default value belonging to key
|
67
62
|
# @param key the default value to retrieve.
|
68
63
|
def default(key = nil)
|
69
|
-
@
|
64
|
+
@table.default(key)
|
70
65
|
end
|
71
66
|
|
72
67
|
# Retrieve a value at key from the database. If the default value was specified
|
@@ -74,15 +69,7 @@ module Daybreak
|
|
74
69
|
# as <tt>get</tt>.
|
75
70
|
# @param key the value to retrieve from the database.
|
76
71
|
def [](key)
|
77
|
-
|
78
|
-
value = @table[skey]
|
79
|
-
if value != nil || @table.has_key?(skey)
|
80
|
-
value
|
81
|
-
elsif @default
|
82
|
-
value = default(key)
|
83
|
-
@queue << [skey, value]
|
84
|
-
@table[skey] = value
|
85
|
-
end
|
72
|
+
@table[@serializer.key_for(key)]
|
86
73
|
end
|
87
74
|
alias_method :get, :'[]'
|
88
75
|
|
@@ -122,6 +109,23 @@ module Daybreak
|
|
122
109
|
value
|
123
110
|
end
|
124
111
|
|
112
|
+
# Update database with hash (Fast batch update)
|
113
|
+
def update(hash)
|
114
|
+
shash = {}
|
115
|
+
hash.each do |key, value|
|
116
|
+
shash[@serializer.key_for(key)] = value
|
117
|
+
end
|
118
|
+
@queue << shash
|
119
|
+
@table.update(shash)
|
120
|
+
self
|
121
|
+
end
|
122
|
+
|
123
|
+
# Updata database and flush data to disk.
|
124
|
+
def update!(hash)
|
125
|
+
update(hash)
|
126
|
+
flush
|
127
|
+
end
|
128
|
+
|
125
129
|
# Does this db have a value for this key?
|
126
130
|
# @param key the key to check if the DB has a key.
|
127
131
|
def has_key?(key)
|
@@ -143,6 +147,12 @@ module Daybreak
|
|
143
147
|
end
|
144
148
|
alias_method :length, :size
|
145
149
|
|
150
|
+
# Utility method that will return the size of the database in bytes,
|
151
|
+
# useful for determining when to compact
|
152
|
+
def bytesize
|
153
|
+
@fd.stat.size unless closed?
|
154
|
+
end
|
155
|
+
|
146
156
|
# Return true if database is empty.
|
147
157
|
# @return [Boolean]
|
148
158
|
def empty?
|
@@ -166,24 +176,26 @@ module Daybreak
|
|
166
176
|
# Flush all changes to disk.
|
167
177
|
def flush
|
168
178
|
@queue.flush
|
179
|
+
self
|
169
180
|
end
|
170
181
|
|
171
182
|
# Sync the database with what is on disk, by first flushing changes, and
|
172
183
|
# then reading the file if necessary.
|
173
184
|
def sync
|
174
185
|
flush
|
175
|
-
|
186
|
+
load
|
176
187
|
end
|
177
188
|
|
178
189
|
# Lock the database for an exclusive commit accross processes and threads
|
179
190
|
# @yield a block where every change to the database is synced
|
180
191
|
def lock
|
181
192
|
@mutex.synchronize do
|
182
|
-
#
|
183
|
-
#
|
193
|
+
# Flush everything to start with a clean state
|
194
|
+
# and to protect the @locked variable
|
184
195
|
flush
|
185
|
-
|
186
|
-
|
196
|
+
|
197
|
+
with_flock(File::LOCK_EX) do
|
198
|
+
load
|
187
199
|
result = yield
|
188
200
|
flush
|
189
201
|
result
|
@@ -209,13 +221,11 @@ module Daybreak
|
|
209
221
|
def compact
|
210
222
|
sync
|
211
223
|
with_tmpfile do |path, file|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
#
|
216
|
-
if
|
217
|
-
# Compactified database has the same size -> return
|
218
|
-
return self if stat.size == compactsize
|
224
|
+
# Compactified database has the same size -> return
|
225
|
+
return self if @pos == file.write(dump)
|
226
|
+
with_flock(File::LOCK_EX) do
|
227
|
+
# Database was compactified in the meantime
|
228
|
+
if @pos != nil
|
219
229
|
# Append changed journal records if the database changed during compactification
|
220
230
|
file.write(read)
|
221
231
|
file.close
|
@@ -224,8 +234,7 @@ module Daybreak
|
|
224
234
|
end
|
225
235
|
end
|
226
236
|
open
|
227
|
-
|
228
|
-
self
|
237
|
+
load
|
229
238
|
end
|
230
239
|
|
231
240
|
# Close the database for reading and writing.
|
@@ -234,7 +243,7 @@ module Daybreak
|
|
234
243
|
@worker.join
|
235
244
|
@fd.close
|
236
245
|
@queue.stop if @queue.respond_to?(:stop)
|
237
|
-
|
246
|
+
@@databases_mutex.synchronize { @@databases.delete(self) }
|
238
247
|
nil
|
239
248
|
end
|
240
249
|
|
@@ -245,9 +254,18 @@ module Daybreak
|
|
245
254
|
|
246
255
|
private
|
247
256
|
|
248
|
-
#
|
249
|
-
def
|
250
|
-
|
257
|
+
# The block used in @table for new entries
|
258
|
+
def hash_default(_, key)
|
259
|
+
if @default != nil
|
260
|
+
value = @default.respond_to?(:call) ? @default.call(key) : @default
|
261
|
+
@queue << [key, value]
|
262
|
+
@table[key] = value
|
263
|
+
end
|
264
|
+
end
|
265
|
+
|
266
|
+
# Update the @table with records
|
267
|
+
def load
|
268
|
+
buf = read
|
251
269
|
until buf.empty?
|
252
270
|
record = @format.parse(buf)
|
253
271
|
if record.size == 1
|
@@ -257,29 +275,10 @@ module Daybreak
|
|
257
275
|
end
|
258
276
|
@logsize += 1
|
259
277
|
end
|
278
|
+
self
|
260
279
|
end
|
261
280
|
|
262
|
-
#
|
263
|
-
def new_records
|
264
|
-
loop do
|
265
|
-
unless @exclusive
|
266
|
-
# HACK: JRuby returns false if the process is already hold by the same process
|
267
|
-
# see https://github.com/jruby/jruby/issues/496
|
268
|
-
Thread.pass until @fd.flock(File::LOCK_SH)
|
269
|
-
end
|
270
|
-
# Check if database was compactified in the meantime
|
271
|
-
# break if not
|
272
|
-
stat = @fd.stat
|
273
|
-
break if stat.nlink > 0 && stat.ino == @inode
|
274
|
-
open
|
275
|
-
end
|
276
|
-
|
277
|
-
# Read new journal records
|
278
|
-
read
|
279
|
-
ensure
|
280
|
-
@fd.flock(File::LOCK_UN) unless @exclusive
|
281
|
-
end
|
282
|
-
|
281
|
+
# Open or reopen file
|
283
282
|
def open
|
284
283
|
@fd.close if @fd
|
285
284
|
@fd = File.open(@file, 'ab+')
|
@@ -287,24 +286,24 @@ module Daybreak
|
|
287
286
|
stat = @fd.stat
|
288
287
|
@inode = stat.ino
|
289
288
|
@logsize = 0
|
290
|
-
if stat.size == 0
|
291
|
-
@fd.write(@format.header)
|
292
|
-
@fd.flush
|
293
|
-
end
|
289
|
+
write(@format.header) if stat.size == 0
|
294
290
|
@pos = nil
|
295
291
|
end
|
296
292
|
|
293
|
+
# Read new file content
|
297
294
|
def read
|
298
|
-
|
299
|
-
|
300
|
-
@
|
301
|
-
|
302
|
-
|
303
|
-
|
295
|
+
with_flock(File::LOCK_SH) do
|
296
|
+
# File was opened
|
297
|
+
unless @pos
|
298
|
+
@fd.pos = 0
|
299
|
+
@format.read_header(@fd)
|
300
|
+
else
|
301
|
+
@fd.pos = @pos
|
302
|
+
end
|
303
|
+
buf = @fd.read
|
304
|
+
@pos = @fd.pos
|
305
|
+
buf
|
304
306
|
end
|
305
|
-
buf = @fd.read
|
306
|
-
@pos = @fd.pos
|
307
|
-
buf
|
308
307
|
end
|
309
308
|
|
310
309
|
# Return database dump as string
|
@@ -321,49 +320,69 @@ module Daybreak
|
|
321
320
|
# Worker thread
|
322
321
|
def worker
|
323
322
|
loop do
|
324
|
-
record = @queue.next
|
325
|
-
|
323
|
+
case record = @queue.next
|
324
|
+
when Hash
|
325
|
+
write_batch(record)
|
326
|
+
when nil
|
327
|
+
@queue.pop
|
328
|
+
break
|
329
|
+
else
|
330
|
+
write_record(record)
|
331
|
+
end
|
326
332
|
@queue.pop
|
327
|
-
break unless record
|
328
333
|
end
|
329
334
|
rescue Exception => ex
|
330
335
|
warn "Daybreak worker: #{ex.message}"
|
331
336
|
retry
|
332
337
|
end
|
333
338
|
|
334
|
-
# Write
|
335
|
-
|
339
|
+
# Write batch update
|
340
|
+
def write_batch(records)
|
341
|
+
dump = ''
|
342
|
+
records.each do |record|
|
343
|
+
record[1] = @serializer.dump(record.last)
|
344
|
+
dump << @format.dump(record)
|
345
|
+
end
|
346
|
+
write(dump)
|
347
|
+
@logsize += records.size
|
348
|
+
end
|
349
|
+
|
350
|
+
# Write single record
|
336
351
|
def write_record(record)
|
337
352
|
record[1] = @serializer.dump(record.last) if record.size > 1
|
338
|
-
|
339
|
-
|
340
|
-
|
353
|
+
write(@format.dump(record))
|
354
|
+
@logsize += 1
|
355
|
+
end
|
356
|
+
|
357
|
+
# Write data to output stream and advance @pos
|
358
|
+
def write(dump)
|
359
|
+
with_flock(File::LOCK_EX) do
|
360
|
+
@fd.write(dump)
|
341
361
|
# Flush to make sure the file is really updated
|
342
362
|
@fd.flush
|
343
363
|
end
|
344
|
-
@pos = @fd.pos if @pos && @fd.pos == @pos +
|
345
|
-
@logsize += 1
|
364
|
+
@pos = @fd.pos if @pos && @fd.pos == @pos + dump.bytesize
|
346
365
|
end
|
347
366
|
|
348
|
-
#
|
349
|
-
def
|
350
|
-
return yield if @
|
367
|
+
# Block with file lock
|
368
|
+
def with_flock(mode)
|
369
|
+
return yield if @locked
|
351
370
|
begin
|
352
371
|
loop do
|
353
372
|
# HACK: JRuby returns false if the process is already hold by the same process
|
354
373
|
# see https://github.com/jruby/jruby/issues/496
|
355
|
-
Thread.pass until @fd.flock(
|
374
|
+
Thread.pass until @fd.flock(mode)
|
356
375
|
# Check if database was compactified in the meantime
|
357
376
|
# break if not
|
358
377
|
stat = @fd.stat
|
359
378
|
break if stat.nlink > 0 && stat.ino == @inode
|
360
379
|
open
|
361
380
|
end
|
362
|
-
@
|
381
|
+
@locked = true
|
363
382
|
yield
|
364
383
|
ensure
|
365
384
|
@fd.flock(File::LOCK_UN)
|
366
|
-
@
|
385
|
+
@locked = false
|
367
386
|
end
|
368
387
|
end
|
369
388
|
|
data/lib/daybreak/format.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Daybreak
|
2
2
|
# Database format serializer and deserializer. You can create
|
3
|
-
# your own
|
3
|
+
# your own implementation of this class and define
|
4
4
|
# your own database format!
|
5
5
|
# @api public
|
6
6
|
class Format
|
@@ -35,7 +35,7 @@ module Daybreak
|
|
35
35
|
def parse(buf)
|
36
36
|
key_size, value_size = buf[0, 8].unpack('NN')
|
37
37
|
data = buf.slice!(0, 8 + key_size + (value_size == DELETE ? 0 : value_size))
|
38
|
-
raise 'CRC mismatch' unless buf.slice!(0, 4) == crc32(data)
|
38
|
+
raise 'CRC mismatch: your data might be corrupted!' unless buf.slice!(0, 4) == crc32(data)
|
39
39
|
value_size == DELETE ? [data[8, key_size]] : [data[8, key_size], data[8 + key_size, value_size]]
|
40
40
|
end
|
41
41
|
|
data/lib/daybreak/queue.rb
CHANGED
@@ -1,107 +1,5 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
# HACK: Dangerous optimization on MRI which has a
|
6
|
-
# global interpreter lock and makes the @queue array
|
7
|
-
# thread safe.
|
8
|
-
if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby'
|
9
|
-
def initialize
|
10
|
-
@queue, @full, @empty = [], [], []
|
11
|
-
@stop = false
|
12
|
-
@heartbeat = Thread.new(&method(:heartbeat))
|
13
|
-
@heartbeat.priority = -9
|
14
|
-
end
|
15
|
-
|
16
|
-
def <<(x)
|
17
|
-
@queue << x
|
18
|
-
thread = @full.first
|
19
|
-
thread.wakeup if thread
|
20
|
-
end
|
21
|
-
|
22
|
-
def pop
|
23
|
-
@queue.shift
|
24
|
-
if @queue.empty?
|
25
|
-
thread = @empty.first
|
26
|
-
thread.wakeup if thread
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
def next
|
31
|
-
while @queue.empty?
|
32
|
-
begin
|
33
|
-
@full << Thread.current
|
34
|
-
# If a push happens before Thread.stop, the thread won't be woken up
|
35
|
-
Thread.stop while @queue.empty?
|
36
|
-
ensure
|
37
|
-
@full.delete(Thread.current)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
@queue.first
|
41
|
-
end
|
42
|
-
|
43
|
-
def flush
|
44
|
-
until @queue.empty?
|
45
|
-
begin
|
46
|
-
@empty << Thread.current
|
47
|
-
# If a pop happens before Thread.stop, the thread won't be woken up
|
48
|
-
Thread.stop until @queue.empty?
|
49
|
-
ensure
|
50
|
-
@empty.delete(Thread.current)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
def stop
|
56
|
-
@stop = true
|
57
|
-
@heartbeat.join
|
58
|
-
end
|
59
|
-
|
60
|
-
private
|
61
|
-
|
62
|
-
# Check threads 10 times per second to avoid deadlocks
|
63
|
-
# since there is a race condition below
|
64
|
-
def heartbeat
|
65
|
-
until @stop
|
66
|
-
@empty.each(&:wakeup)
|
67
|
-
@full.each(&:wakeup)
|
68
|
-
sleep 0.1
|
69
|
-
end
|
70
|
-
end
|
71
|
-
else
|
72
|
-
def initialize
|
73
|
-
@mutex = Mutex.new
|
74
|
-
@full = ConditionVariable.new
|
75
|
-
@empty = ConditionVariable.new
|
76
|
-
@queue = []
|
77
|
-
end
|
78
|
-
|
79
|
-
def <<(x)
|
80
|
-
@mutex.synchronize do
|
81
|
-
@queue << x
|
82
|
-
@full.signal
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
def pop
|
87
|
-
@mutex.synchronize do
|
88
|
-
@queue.shift
|
89
|
-
@empty.signal if @queue.empty?
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
def next
|
94
|
-
@mutex.synchronize do
|
95
|
-
@full.wait(@mutex) while @queue.empty?
|
96
|
-
@queue.first
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
def flush
|
101
|
-
@mutex.synchronize do
|
102
|
-
@empty.wait(@mutex) until @queue.empty?
|
103
|
-
end
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
1
|
+
if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby'
|
2
|
+
require 'daybreak/queue/mri'
|
3
|
+
else
|
4
|
+
require 'daybreak/queue/threaded'
|
107
5
|
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
module Daybreak
|
2
|
+
class Queue
|
3
|
+
# HACK: Dangerous optimization on MRI which has a
|
4
|
+
# global interpreter lock and makes the @queue array
|
5
|
+
# thread safe.
|
6
|
+
def initialize
|
7
|
+
@queue, @full, @empty = [], [], []
|
8
|
+
@stop = false
|
9
|
+
@heartbeat = Thread.new(&method(:heartbeat))
|
10
|
+
@heartbeat.priority = -9
|
11
|
+
end
|
12
|
+
|
13
|
+
def <<(x)
|
14
|
+
@queue << x
|
15
|
+
thread = @full.first
|
16
|
+
thread.wakeup if thread
|
17
|
+
end
|
18
|
+
|
19
|
+
def pop
|
20
|
+
@queue.shift
|
21
|
+
if @queue.empty?
|
22
|
+
thread = @empty.first
|
23
|
+
thread.wakeup if thread
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def next
|
28
|
+
while @queue.empty?
|
29
|
+
begin
|
30
|
+
@full << Thread.current
|
31
|
+
# If a push happens before Thread.stop, the thread won't be woken up
|
32
|
+
Thread.stop while @queue.empty?
|
33
|
+
ensure
|
34
|
+
@full.delete(Thread.current)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
@queue.first
|
38
|
+
end
|
39
|
+
|
40
|
+
def flush
|
41
|
+
until @queue.empty?
|
42
|
+
begin
|
43
|
+
@empty << Thread.current
|
44
|
+
# If a pop happens before Thread.stop, the thread won't be woken up
|
45
|
+
Thread.stop until @queue.empty?
|
46
|
+
ensure
|
47
|
+
@empty.delete(Thread.current)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def stop
|
53
|
+
@stop = true
|
54
|
+
@heartbeat.join
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
# Check threads 10 times per second to avoid deadlocks
|
60
|
+
# since there is a race condition above
|
61
|
+
def heartbeat
|
62
|
+
until @stop
|
63
|
+
@empty.each(&:wakeup)
|
64
|
+
@full.each(&:wakeup)
|
65
|
+
sleep 0.1
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Daybreak
|
2
|
+
# A queue for threaded implementations of ruby without a GIL
|
3
|
+
# @api private
|
4
|
+
class Queue
|
5
|
+
def initialize
|
6
|
+
@mutex = Mutex.new
|
7
|
+
@full = ConditionVariable.new
|
8
|
+
@empty = ConditionVariable.new
|
9
|
+
@queue = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def <<(x)
|
13
|
+
@mutex.synchronize do
|
14
|
+
@queue << x
|
15
|
+
@full.signal
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def pop
|
20
|
+
@mutex.synchronize do
|
21
|
+
@queue.shift
|
22
|
+
@empty.signal if @queue.empty?
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def next
|
27
|
+
@mutex.synchronize do
|
28
|
+
@full.wait(@mutex) while @queue.empty?
|
29
|
+
@queue.first
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def flush
|
34
|
+
@mutex.synchronize do
|
35
|
+
@empty.wait(@mutex) until @queue.empty?
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/daybreak/version.rb
CHANGED
data/script/bench
CHANGED
@@ -43,6 +43,7 @@ def run(instance, message = '', &blk)
|
|
43
43
|
report measure(instance, &blk)
|
44
44
|
puts '=' * 64
|
45
45
|
ensure
|
46
|
+
instance.clear if instance.respond_to? :clear
|
46
47
|
instance.close if instance.respond_to? :close
|
47
48
|
end
|
48
49
|
|
@@ -70,7 +71,8 @@ rescue Exception => ex
|
|
70
71
|
end
|
71
72
|
|
72
73
|
run Hash.new
|
73
|
-
|
74
|
+
db = Daybreak::DB.new DB_PATH
|
75
|
+
run db
|
74
76
|
|
75
77
|
db = Daybreak::DB.new DB_PATH
|
76
78
|
run db, 'with lock' do
|
@@ -93,3 +95,14 @@ run db, 'with sync' do
|
|
93
95
|
db.sync
|
94
96
|
end
|
95
97
|
end
|
98
|
+
|
99
|
+
hash = DATA.reduce({}) {|m, v| m.merge! v => v }
|
100
|
+
db = Daybreak::DB.new DB_PATH
|
101
|
+
run db, 'bulk updates' do
|
102
|
+
db.update hash
|
103
|
+
end
|
104
|
+
|
105
|
+
db = Daybreak::DB.new DB_PATH
|
106
|
+
run db, 'bulk updates, synced' do
|
107
|
+
db.update! hash
|
108
|
+
end
|
data/test/prof.rb
CHANGED
@@ -4,10 +4,10 @@ require 'ruby-prof'
|
|
4
4
|
|
5
5
|
result = RubyProf.profile do
|
6
6
|
db = Daybreak::DB.new './t.db'
|
7
|
-
|
8
|
-
db.flush
|
7
|
+
10.times {|n| db[n] = n}
|
8
|
+
db.flush
|
9
|
+
db.close
|
9
10
|
end
|
10
|
-
|
11
11
|
File.unlink './t.db'
|
12
12
|
printer = RubyProf::MultiPrinter.new(result)
|
13
13
|
FileUtils.mkdir('./profile') unless File.exists? './profile'
|
data/test/test.rb
CHANGED
@@ -11,6 +11,8 @@ describe Daybreak::DB do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
it 'should insert' do
|
14
|
+
assert_equal @db[1], nil
|
15
|
+
assert_equal @db.include?(1), false
|
14
16
|
@db[1] = 1
|
15
17
|
assert_equal @db[1], 1
|
16
18
|
assert @db.has_key?(1)
|
@@ -19,24 +21,40 @@ describe Daybreak::DB do
|
|
19
21
|
assert_equal @db.length, 1
|
20
22
|
end
|
21
23
|
|
24
|
+
it 'should support batch inserts' do
|
25
|
+
@db.update(1 => :a, 2 => :b)
|
26
|
+
assert_equal @db[1], :a
|
27
|
+
assert_equal @db[2], :b
|
28
|
+
assert_equal @db.length, 2
|
29
|
+
end
|
30
|
+
|
22
31
|
it 'should persist values' do
|
23
32
|
@db['1'] = '4'
|
24
33
|
@db['4'] = '1'
|
25
|
-
@db.sync
|
34
|
+
assert_equal @db.sync, @db
|
26
35
|
|
27
36
|
assert_equal @db['1'], '4'
|
28
37
|
db2 = Daybreak::DB.new DB_PATH
|
29
38
|
assert_equal db2['1'], '4'
|
30
39
|
assert_equal db2['4'], '1'
|
31
|
-
db2.close
|
40
|
+
assert_equal db2.close, nil
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should persist after batch update' do
|
44
|
+
@db.update!(1 => :a, 2 => :b)
|
45
|
+
|
46
|
+
db2 = Daybreak::DB.new DB_PATH
|
47
|
+
assert_equal db2[1], :a
|
48
|
+
assert_equal db2[2], :b
|
49
|
+
assert_equal db2.close, nil
|
32
50
|
end
|
33
51
|
|
34
52
|
it 'should persist after clear' do
|
35
53
|
@db['1'] = 'xy'
|
36
|
-
@db.clear
|
54
|
+
assert_equal @db.clear, @db
|
37
55
|
@db['1'] = '4'
|
38
56
|
@db['4'] = '1'
|
39
|
-
@db.close
|
57
|
+
assert_equal @db.close, nil
|
40
58
|
|
41
59
|
@db = Daybreak::DB.new DB_PATH
|
42
60
|
assert_equal @db['1'], '4'
|
@@ -46,10 +64,10 @@ describe Daybreak::DB do
|
|
46
64
|
it 'should persist after compact' do
|
47
65
|
@db['1'] = 'xy'
|
48
66
|
@db['1'] = 'z'
|
49
|
-
@db.compact
|
67
|
+
assert_equal @db.compact, @db
|
50
68
|
@db['1'] = '4'
|
51
69
|
@db['4'] = '1'
|
52
|
-
@db.close
|
70
|
+
assert_equal @db.close, nil
|
53
71
|
|
54
72
|
@db = Daybreak::DB.new DB_PATH
|
55
73
|
assert_equal @db['1'], '4'
|
@@ -61,10 +79,10 @@ describe Daybreak::DB do
|
|
61
79
|
|
62
80
|
@db['1'] = 'xy'
|
63
81
|
@db['1'] = 'z'
|
64
|
-
@db.compact
|
82
|
+
assert_equal @db.compact, @db
|
65
83
|
@db['1'] = '4'
|
66
84
|
@db['4'] = '1'
|
67
|
-
@db.flush
|
85
|
+
assert_equal @db.flush, @db
|
68
86
|
|
69
87
|
db.sync
|
70
88
|
assert_equal db['1'], '4'
|
@@ -100,16 +118,24 @@ describe Daybreak::DB do
|
|
100
118
|
end
|
101
119
|
|
102
120
|
it 'should allow for default values' do
|
103
|
-
|
104
|
-
assert_equal
|
105
|
-
|
106
|
-
|
107
|
-
|
121
|
+
db = Daybreak::DB.new(DB_PATH, :default => 0)
|
122
|
+
assert_equal db[1], 0
|
123
|
+
assert db.include? '1'
|
124
|
+
db[1] = 1
|
125
|
+
assert_equal db[1], 1
|
126
|
+
db.default = 42
|
127
|
+
assert_equal db['x'], 42
|
128
|
+
db.close
|
108
129
|
end
|
109
130
|
|
110
131
|
it 'should handle default values that are procs' do
|
111
|
-
db = Daybreak::DB.new(DB_PATH) {|key| Set.new }
|
132
|
+
db = Daybreak::DB.new(DB_PATH) {|key| set = Set.new; set << key }
|
112
133
|
assert db['foo'].is_a? Set
|
134
|
+
assert db.include? 'foo'
|
135
|
+
assert db['bar'].include? 'bar'
|
136
|
+
db.default = proc {|key| [key] }
|
137
|
+
assert db[1].is_a? Array
|
138
|
+
assert db[2] == ['2']
|
113
139
|
db.close
|
114
140
|
end
|
115
141
|
|
@@ -192,8 +218,8 @@ describe Daybreak::DB do
|
|
192
218
|
db["b#{i}"] = i
|
193
219
|
sleep 0.01 if i % 100 == 0
|
194
220
|
end
|
195
|
-
|
196
|
-
|
221
|
+
db.close
|
222
|
+
end
|
197
223
|
Process.wait a
|
198
224
|
Process.wait b
|
199
225
|
@db = Daybreak::DB.new DB_PATH
|
@@ -310,13 +336,28 @@ describe Daybreak::DB do
|
|
310
336
|
end
|
311
337
|
end
|
312
338
|
|
313
|
-
it 'should support set! in lock' do
|
339
|
+
it 'should support set! and delete! in lock' do
|
314
340
|
@db[1] = 2
|
315
341
|
@db.lock do
|
316
342
|
@db.set!(1, 2)
|
343
|
+
@db.delete!(1)
|
317
344
|
end
|
318
345
|
end
|
319
346
|
|
347
|
+
it 'should allow for inheritance' do
|
348
|
+
class Subclassed < Daybreak::DB
|
349
|
+
def increment(key, amount = 1)
|
350
|
+
lock { self[key] += amount }
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
354
|
+
db = Subclassed.new DB_PATH
|
355
|
+
db[1] = 1
|
356
|
+
assert_equal db.increment(1), 2
|
357
|
+
db.clear
|
358
|
+
db.close
|
359
|
+
end
|
360
|
+
|
320
361
|
after do
|
321
362
|
@db.clear
|
322
363
|
@db.close
|
metadata
CHANGED
@@ -1,57 +1,62 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: daybreak
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 21
|
5
5
|
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
- 1
|
10
|
+
version: 0.2.1
|
6
11
|
platform: ruby
|
7
|
-
authors:
|
12
|
+
authors:
|
8
13
|
- Jeff Larson
|
9
14
|
- Daniel Mendler
|
10
15
|
autorequire:
|
11
16
|
bindir: bin
|
12
17
|
cert_chain: []
|
13
|
-
|
14
|
-
|
15
|
-
|
18
|
+
|
19
|
+
date: 2013-01-16 00:00:00 Z
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
16
22
|
name: rake
|
17
|
-
requirement: !ruby/object:Gem::Requirement
|
18
|
-
none: false
|
19
|
-
requirements:
|
20
|
-
- - ! '>='
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: '0'
|
23
|
-
type: :development
|
24
23
|
prerelease: false
|
25
|
-
|
26
|
-
none: false
|
27
|
-
requirements:
|
28
|
-
- - ! '>='
|
29
|
-
- !ruby/object:Gem::Version
|
30
|
-
version: '0'
|
31
|
-
- !ruby/object:Gem::Dependency
|
32
|
-
name: minitest
|
33
|
-
requirement: !ruby/object:Gem::Requirement
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
34
25
|
none: false
|
35
|
-
requirements:
|
36
|
-
- -
|
37
|
-
- !ruby/object:Gem::Version
|
38
|
-
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 3
|
30
|
+
segments:
|
31
|
+
- 0
|
32
|
+
version: "0"
|
39
33
|
type: :development
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: minitest
|
40
37
|
prerelease: false
|
41
|
-
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
42
39
|
none: false
|
43
|
-
requirements:
|
44
|
-
- -
|
45
|
-
- !ruby/object:Gem::Version
|
46
|
-
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
hash: 3
|
44
|
+
segments:
|
45
|
+
- 0
|
46
|
+
version: "0"
|
47
|
+
type: :development
|
48
|
+
version_requirements: *id002
|
47
49
|
description: Incredibly fast pure-ruby key-value store
|
48
|
-
email:
|
50
|
+
email:
|
49
51
|
- thejefflarson@gmail.com
|
50
52
|
- mail@daniel-mendler.de
|
51
53
|
executables: []
|
54
|
+
|
52
55
|
extensions: []
|
56
|
+
|
53
57
|
extra_rdoc_files: []
|
54
|
-
|
58
|
+
|
59
|
+
files:
|
55
60
|
- .gitignore
|
56
61
|
- .travis.yml
|
57
62
|
- .yardopts
|
@@ -65,6 +70,8 @@ files:
|
|
65
70
|
- lib/daybreak/db.rb
|
66
71
|
- lib/daybreak/format.rb
|
67
72
|
- lib/daybreak/queue.rb
|
73
|
+
- lib/daybreak/queue/mri.rb
|
74
|
+
- lib/daybreak/queue/threaded.rb
|
68
75
|
- lib/daybreak/serializer.rb
|
69
76
|
- lib/daybreak/version.rb
|
70
77
|
- script/bench
|
@@ -73,33 +80,39 @@ files:
|
|
73
80
|
- test/test.rb
|
74
81
|
- test/test_helper.rb
|
75
82
|
homepage: http://propublica.github.com/daybreak/
|
76
|
-
licenses:
|
83
|
+
licenses:
|
77
84
|
- MIT
|
78
85
|
post_install_message:
|
79
86
|
rdoc_options: []
|
80
|
-
|
87
|
+
|
88
|
+
require_paths:
|
81
89
|
- lib
|
82
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
90
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
83
91
|
none: false
|
84
|
-
requirements:
|
85
|
-
- -
|
86
|
-
- !ruby/object:Gem::Version
|
87
|
-
|
88
|
-
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
hash: 3
|
96
|
+
segments:
|
97
|
+
- 0
|
98
|
+
version: "0"
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
89
100
|
none: false
|
90
|
-
requirements:
|
91
|
-
- -
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
hash: 3
|
105
|
+
segments:
|
106
|
+
- 0
|
107
|
+
version: "0"
|
94
108
|
requirements: []
|
109
|
+
|
95
110
|
rubyforge_project:
|
96
111
|
rubygems_version: 1.8.24
|
97
112
|
signing_key:
|
98
113
|
specification_version: 3
|
99
|
-
summary: Daybreak provides an incredibly fast pure-ruby in memory key-value store,
|
100
|
-
|
101
|
-
test_files:
|
114
|
+
summary: Daybreak provides an incredibly fast pure-ruby in memory key-value store, which is multi-process safe and uses a journal log to store the data.
|
115
|
+
test_files:
|
102
116
|
- test/prof.rb
|
103
117
|
- test/test.rb
|
104
118
|
- test/test_helper.rb
|
105
|
-
has_rdoc:
|