daybreak 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +3 -0
- data/lib/daybreak/db.rb +119 -100
- data/lib/daybreak/format.rb +2 -2
- data/lib/daybreak/queue.rb +4 -106
- data/lib/daybreak/queue/mri.rb +69 -0
- data/lib/daybreak/queue/threaded.rb +39 -0
- data/lib/daybreak/version.rb +1 -1
- data/script/bench +14 -1
- data/test/prof.rb +3 -3
- data/test/test.rb +58 -17
- metadata +63 -50
data/.travis.yml
CHANGED
data/lib/daybreak/db.rb
CHANGED
@@ -5,19 +5,24 @@ module Daybreak
|
|
5
5
|
class DB
|
6
6
|
include Enumerable
|
7
7
|
|
8
|
-
#
|
9
|
-
|
10
|
-
|
8
|
+
# Database file name
|
9
|
+
attr_reader :file
|
10
|
+
|
11
|
+
# Counter of how many records are in
|
12
|
+
attr_reader :logsize
|
13
|
+
|
14
|
+
# Set default value, can be a callable
|
11
15
|
attr_writer :default
|
12
16
|
|
13
|
-
|
14
|
-
|
17
|
+
@@databases = []
|
18
|
+
@@databases_mutex = Mutex.new
|
15
19
|
|
16
20
|
# A handler that will ensure that databases are closed and synced when the
|
17
21
|
# current process exits.
|
18
|
-
|
22
|
+
# @api private
|
23
|
+
def self.exit_handler
|
19
24
|
loop do
|
20
|
-
db =
|
25
|
+
db = @@databases_mutex.synchronize { @@databases.first }
|
21
26
|
break unless db
|
22
27
|
warn "Daybreak database #{db.file} was not closed, state might be inconsistent"
|
23
28
|
begin
|
@@ -28,17 +33,7 @@ module Daybreak
|
|
28
33
|
end
|
29
34
|
end
|
30
35
|
|
31
|
-
|
32
|
-
# @api private
|
33
|
-
def register(db)
|
34
|
-
@databases_mutex.synchronize { @databases << db }
|
35
|
-
end
|
36
|
-
|
37
|
-
# @api private
|
38
|
-
def unregister(db)
|
39
|
-
@databases_mutex.synchronize { @databases.delete(db) }
|
40
|
-
end
|
41
|
-
end
|
36
|
+
at_exit { Daybreak::DB.exit_handler }
|
42
37
|
|
43
38
|
# Create a new Daybreak::DB. The second argument is the default value
|
44
39
|
# to store when accessing a previously unset key, this follows the
|
@@ -52,21 +47,21 @@ module Daybreak
|
|
52
47
|
@file = file
|
53
48
|
@serializer = (options[:serializer] || Serializer::Default).new
|
54
49
|
@format = (options[:format] || Format).new
|
55
|
-
@default = block ? block : options[:default]
|
56
50
|
@queue = Queue.new
|
57
|
-
@table =
|
51
|
+
@table = Hash.new(&method(:hash_default))
|
52
|
+
@default = block ? block : options[:default]
|
58
53
|
open
|
59
54
|
@mutex = Mutex.new # Mutex to make #lock thread safe
|
60
55
|
@worker = Thread.new(&method(:worker))
|
61
56
|
@worker.priority = -1
|
62
|
-
|
63
|
-
|
57
|
+
load
|
58
|
+
@@databases_mutex.synchronize { @@databases << self }
|
64
59
|
end
|
65
60
|
|
66
61
|
# Return default value belonging to key
|
67
62
|
# @param key the default value to retrieve.
|
68
63
|
def default(key = nil)
|
69
|
-
@
|
64
|
+
@table.default(key)
|
70
65
|
end
|
71
66
|
|
72
67
|
# Retrieve a value at key from the database. If the default value was specified
|
@@ -74,15 +69,7 @@ module Daybreak
|
|
74
69
|
# as <tt>get</tt>.
|
75
70
|
# @param key the value to retrieve from the database.
|
76
71
|
def [](key)
|
77
|
-
|
78
|
-
value = @table[skey]
|
79
|
-
if value != nil || @table.has_key?(skey)
|
80
|
-
value
|
81
|
-
elsif @default
|
82
|
-
value = default(key)
|
83
|
-
@queue << [skey, value]
|
84
|
-
@table[skey] = value
|
85
|
-
end
|
72
|
+
@table[@serializer.key_for(key)]
|
86
73
|
end
|
87
74
|
alias_method :get, :'[]'
|
88
75
|
|
@@ -122,6 +109,23 @@ module Daybreak
|
|
122
109
|
value
|
123
110
|
end
|
124
111
|
|
112
|
+
# Update database with hash (Fast batch update)
|
113
|
+
def update(hash)
|
114
|
+
shash = {}
|
115
|
+
hash.each do |key, value|
|
116
|
+
shash[@serializer.key_for(key)] = value
|
117
|
+
end
|
118
|
+
@queue << shash
|
119
|
+
@table.update(shash)
|
120
|
+
self
|
121
|
+
end
|
122
|
+
|
123
|
+
# Updata database and flush data to disk.
|
124
|
+
def update!(hash)
|
125
|
+
update(hash)
|
126
|
+
flush
|
127
|
+
end
|
128
|
+
|
125
129
|
# Does this db have a value for this key?
|
126
130
|
# @param key the key to check if the DB has a key.
|
127
131
|
def has_key?(key)
|
@@ -143,6 +147,12 @@ module Daybreak
|
|
143
147
|
end
|
144
148
|
alias_method :length, :size
|
145
149
|
|
150
|
+
# Utility method that will return the size of the database in bytes,
|
151
|
+
# useful for determining when to compact
|
152
|
+
def bytesize
|
153
|
+
@fd.stat.size unless closed?
|
154
|
+
end
|
155
|
+
|
146
156
|
# Return true if database is empty.
|
147
157
|
# @return [Boolean]
|
148
158
|
def empty?
|
@@ -166,24 +176,26 @@ module Daybreak
|
|
166
176
|
# Flush all changes to disk.
|
167
177
|
def flush
|
168
178
|
@queue.flush
|
179
|
+
self
|
169
180
|
end
|
170
181
|
|
171
182
|
# Sync the database with what is on disk, by first flushing changes, and
|
172
183
|
# then reading the file if necessary.
|
173
184
|
def sync
|
174
185
|
flush
|
175
|
-
|
186
|
+
load
|
176
187
|
end
|
177
188
|
|
178
189
|
# Lock the database for an exclusive commit accross processes and threads
|
179
190
|
# @yield a block where every change to the database is synced
|
180
191
|
def lock
|
181
192
|
@mutex.synchronize do
|
182
|
-
#
|
183
|
-
#
|
193
|
+
# Flush everything to start with a clean state
|
194
|
+
# and to protect the @locked variable
|
184
195
|
flush
|
185
|
-
|
186
|
-
|
196
|
+
|
197
|
+
with_flock(File::LOCK_EX) do
|
198
|
+
load
|
187
199
|
result = yield
|
188
200
|
flush
|
189
201
|
result
|
@@ -209,13 +221,11 @@ module Daybreak
|
|
209
221
|
def compact
|
210
222
|
sync
|
211
223
|
with_tmpfile do |path, file|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
#
|
216
|
-
if
|
217
|
-
# Compactified database has the same size -> return
|
218
|
-
return self if stat.size == compactsize
|
224
|
+
# Compactified database has the same size -> return
|
225
|
+
return self if @pos == file.write(dump)
|
226
|
+
with_flock(File::LOCK_EX) do
|
227
|
+
# Database was compactified in the meantime
|
228
|
+
if @pos != nil
|
219
229
|
# Append changed journal records if the database changed during compactification
|
220
230
|
file.write(read)
|
221
231
|
file.close
|
@@ -224,8 +234,7 @@ module Daybreak
|
|
224
234
|
end
|
225
235
|
end
|
226
236
|
open
|
227
|
-
|
228
|
-
self
|
237
|
+
load
|
229
238
|
end
|
230
239
|
|
231
240
|
# Close the database for reading and writing.
|
@@ -234,7 +243,7 @@ module Daybreak
|
|
234
243
|
@worker.join
|
235
244
|
@fd.close
|
236
245
|
@queue.stop if @queue.respond_to?(:stop)
|
237
|
-
|
246
|
+
@@databases_mutex.synchronize { @@databases.delete(self) }
|
238
247
|
nil
|
239
248
|
end
|
240
249
|
|
@@ -245,9 +254,18 @@ module Daybreak
|
|
245
254
|
|
246
255
|
private
|
247
256
|
|
248
|
-
#
|
249
|
-
def
|
250
|
-
|
257
|
+
# The block used in @table for new entries
|
258
|
+
def hash_default(_, key)
|
259
|
+
if @default != nil
|
260
|
+
value = @default.respond_to?(:call) ? @default.call(key) : @default
|
261
|
+
@queue << [key, value]
|
262
|
+
@table[key] = value
|
263
|
+
end
|
264
|
+
end
|
265
|
+
|
266
|
+
# Update the @table with records
|
267
|
+
def load
|
268
|
+
buf = read
|
251
269
|
until buf.empty?
|
252
270
|
record = @format.parse(buf)
|
253
271
|
if record.size == 1
|
@@ -257,29 +275,10 @@ module Daybreak
|
|
257
275
|
end
|
258
276
|
@logsize += 1
|
259
277
|
end
|
278
|
+
self
|
260
279
|
end
|
261
280
|
|
262
|
-
#
|
263
|
-
def new_records
|
264
|
-
loop do
|
265
|
-
unless @exclusive
|
266
|
-
# HACK: JRuby returns false if the process is already hold by the same process
|
267
|
-
# see https://github.com/jruby/jruby/issues/496
|
268
|
-
Thread.pass until @fd.flock(File::LOCK_SH)
|
269
|
-
end
|
270
|
-
# Check if database was compactified in the meantime
|
271
|
-
# break if not
|
272
|
-
stat = @fd.stat
|
273
|
-
break if stat.nlink > 0 && stat.ino == @inode
|
274
|
-
open
|
275
|
-
end
|
276
|
-
|
277
|
-
# Read new journal records
|
278
|
-
read
|
279
|
-
ensure
|
280
|
-
@fd.flock(File::LOCK_UN) unless @exclusive
|
281
|
-
end
|
282
|
-
|
281
|
+
# Open or reopen file
|
283
282
|
def open
|
284
283
|
@fd.close if @fd
|
285
284
|
@fd = File.open(@file, 'ab+')
|
@@ -287,24 +286,24 @@ module Daybreak
|
|
287
286
|
stat = @fd.stat
|
288
287
|
@inode = stat.ino
|
289
288
|
@logsize = 0
|
290
|
-
if stat.size == 0
|
291
|
-
@fd.write(@format.header)
|
292
|
-
@fd.flush
|
293
|
-
end
|
289
|
+
write(@format.header) if stat.size == 0
|
294
290
|
@pos = nil
|
295
291
|
end
|
296
292
|
|
293
|
+
# Read new file content
|
297
294
|
def read
|
298
|
-
|
299
|
-
|
300
|
-
@
|
301
|
-
|
302
|
-
|
303
|
-
|
295
|
+
with_flock(File::LOCK_SH) do
|
296
|
+
# File was opened
|
297
|
+
unless @pos
|
298
|
+
@fd.pos = 0
|
299
|
+
@format.read_header(@fd)
|
300
|
+
else
|
301
|
+
@fd.pos = @pos
|
302
|
+
end
|
303
|
+
buf = @fd.read
|
304
|
+
@pos = @fd.pos
|
305
|
+
buf
|
304
306
|
end
|
305
|
-
buf = @fd.read
|
306
|
-
@pos = @fd.pos
|
307
|
-
buf
|
308
307
|
end
|
309
308
|
|
310
309
|
# Return database dump as string
|
@@ -321,49 +320,69 @@ module Daybreak
|
|
321
320
|
# Worker thread
|
322
321
|
def worker
|
323
322
|
loop do
|
324
|
-
record = @queue.next
|
325
|
-
|
323
|
+
case record = @queue.next
|
324
|
+
when Hash
|
325
|
+
write_batch(record)
|
326
|
+
when nil
|
327
|
+
@queue.pop
|
328
|
+
break
|
329
|
+
else
|
330
|
+
write_record(record)
|
331
|
+
end
|
326
332
|
@queue.pop
|
327
|
-
break unless record
|
328
333
|
end
|
329
334
|
rescue Exception => ex
|
330
335
|
warn "Daybreak worker: #{ex.message}"
|
331
336
|
retry
|
332
337
|
end
|
333
338
|
|
334
|
-
# Write
|
335
|
-
|
339
|
+
# Write batch update
|
340
|
+
def write_batch(records)
|
341
|
+
dump = ''
|
342
|
+
records.each do |record|
|
343
|
+
record[1] = @serializer.dump(record.last)
|
344
|
+
dump << @format.dump(record)
|
345
|
+
end
|
346
|
+
write(dump)
|
347
|
+
@logsize += records.size
|
348
|
+
end
|
349
|
+
|
350
|
+
# Write single record
|
336
351
|
def write_record(record)
|
337
352
|
record[1] = @serializer.dump(record.last) if record.size > 1
|
338
|
-
|
339
|
-
|
340
|
-
|
353
|
+
write(@format.dump(record))
|
354
|
+
@logsize += 1
|
355
|
+
end
|
356
|
+
|
357
|
+
# Write data to output stream and advance @pos
|
358
|
+
def write(dump)
|
359
|
+
with_flock(File::LOCK_EX) do
|
360
|
+
@fd.write(dump)
|
341
361
|
# Flush to make sure the file is really updated
|
342
362
|
@fd.flush
|
343
363
|
end
|
344
|
-
@pos = @fd.pos if @pos && @fd.pos == @pos +
|
345
|
-
@logsize += 1
|
364
|
+
@pos = @fd.pos if @pos && @fd.pos == @pos + dump.bytesize
|
346
365
|
end
|
347
366
|
|
348
|
-
#
|
349
|
-
def
|
350
|
-
return yield if @
|
367
|
+
# Block with file lock
|
368
|
+
def with_flock(mode)
|
369
|
+
return yield if @locked
|
351
370
|
begin
|
352
371
|
loop do
|
353
372
|
# HACK: JRuby returns false if the process is already hold by the same process
|
354
373
|
# see https://github.com/jruby/jruby/issues/496
|
355
|
-
Thread.pass until @fd.flock(
|
374
|
+
Thread.pass until @fd.flock(mode)
|
356
375
|
# Check if database was compactified in the meantime
|
357
376
|
# break if not
|
358
377
|
stat = @fd.stat
|
359
378
|
break if stat.nlink > 0 && stat.ino == @inode
|
360
379
|
open
|
361
380
|
end
|
362
|
-
@
|
381
|
+
@locked = true
|
363
382
|
yield
|
364
383
|
ensure
|
365
384
|
@fd.flock(File::LOCK_UN)
|
366
|
-
@
|
385
|
+
@locked = false
|
367
386
|
end
|
368
387
|
end
|
369
388
|
|
data/lib/daybreak/format.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Daybreak
|
2
2
|
# Database format serializer and deserializer. You can create
|
3
|
-
# your own
|
3
|
+
# your own implementation of this class and define
|
4
4
|
# your own database format!
|
5
5
|
# @api public
|
6
6
|
class Format
|
@@ -35,7 +35,7 @@ module Daybreak
|
|
35
35
|
def parse(buf)
|
36
36
|
key_size, value_size = buf[0, 8].unpack('NN')
|
37
37
|
data = buf.slice!(0, 8 + key_size + (value_size == DELETE ? 0 : value_size))
|
38
|
-
raise 'CRC mismatch' unless buf.slice!(0, 4) == crc32(data)
|
38
|
+
raise 'CRC mismatch: your data might be corrupted!' unless buf.slice!(0, 4) == crc32(data)
|
39
39
|
value_size == DELETE ? [data[8, key_size]] : [data[8, key_size], data[8 + key_size, value_size]]
|
40
40
|
end
|
41
41
|
|
data/lib/daybreak/queue.rb
CHANGED
@@ -1,107 +1,5 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
# HACK: Dangerous optimization on MRI which has a
|
6
|
-
# global interpreter lock and makes the @queue array
|
7
|
-
# thread safe.
|
8
|
-
if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby'
|
9
|
-
def initialize
|
10
|
-
@queue, @full, @empty = [], [], []
|
11
|
-
@stop = false
|
12
|
-
@heartbeat = Thread.new(&method(:heartbeat))
|
13
|
-
@heartbeat.priority = -9
|
14
|
-
end
|
15
|
-
|
16
|
-
def <<(x)
|
17
|
-
@queue << x
|
18
|
-
thread = @full.first
|
19
|
-
thread.wakeup if thread
|
20
|
-
end
|
21
|
-
|
22
|
-
def pop
|
23
|
-
@queue.shift
|
24
|
-
if @queue.empty?
|
25
|
-
thread = @empty.first
|
26
|
-
thread.wakeup if thread
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
def next
|
31
|
-
while @queue.empty?
|
32
|
-
begin
|
33
|
-
@full << Thread.current
|
34
|
-
# If a push happens before Thread.stop, the thread won't be woken up
|
35
|
-
Thread.stop while @queue.empty?
|
36
|
-
ensure
|
37
|
-
@full.delete(Thread.current)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
@queue.first
|
41
|
-
end
|
42
|
-
|
43
|
-
def flush
|
44
|
-
until @queue.empty?
|
45
|
-
begin
|
46
|
-
@empty << Thread.current
|
47
|
-
# If a pop happens before Thread.stop, the thread won't be woken up
|
48
|
-
Thread.stop until @queue.empty?
|
49
|
-
ensure
|
50
|
-
@empty.delete(Thread.current)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
def stop
|
56
|
-
@stop = true
|
57
|
-
@heartbeat.join
|
58
|
-
end
|
59
|
-
|
60
|
-
private
|
61
|
-
|
62
|
-
# Check threads 10 times per second to avoid deadlocks
|
63
|
-
# since there is a race condition below
|
64
|
-
def heartbeat
|
65
|
-
until @stop
|
66
|
-
@empty.each(&:wakeup)
|
67
|
-
@full.each(&:wakeup)
|
68
|
-
sleep 0.1
|
69
|
-
end
|
70
|
-
end
|
71
|
-
else
|
72
|
-
def initialize
|
73
|
-
@mutex = Mutex.new
|
74
|
-
@full = ConditionVariable.new
|
75
|
-
@empty = ConditionVariable.new
|
76
|
-
@queue = []
|
77
|
-
end
|
78
|
-
|
79
|
-
def <<(x)
|
80
|
-
@mutex.synchronize do
|
81
|
-
@queue << x
|
82
|
-
@full.signal
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
def pop
|
87
|
-
@mutex.synchronize do
|
88
|
-
@queue.shift
|
89
|
-
@empty.signal if @queue.empty?
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
def next
|
94
|
-
@mutex.synchronize do
|
95
|
-
@full.wait(@mutex) while @queue.empty?
|
96
|
-
@queue.first
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
def flush
|
101
|
-
@mutex.synchronize do
|
102
|
-
@empty.wait(@mutex) until @queue.empty?
|
103
|
-
end
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
1
|
+
if !defined?(RUBY_ENGINE) || RUBY_ENGINE == 'ruby'
|
2
|
+
require 'daybreak/queue/mri'
|
3
|
+
else
|
4
|
+
require 'daybreak/queue/threaded'
|
107
5
|
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
module Daybreak
|
2
|
+
class Queue
|
3
|
+
# HACK: Dangerous optimization on MRI which has a
|
4
|
+
# global interpreter lock and makes the @queue array
|
5
|
+
# thread safe.
|
6
|
+
def initialize
|
7
|
+
@queue, @full, @empty = [], [], []
|
8
|
+
@stop = false
|
9
|
+
@heartbeat = Thread.new(&method(:heartbeat))
|
10
|
+
@heartbeat.priority = -9
|
11
|
+
end
|
12
|
+
|
13
|
+
def <<(x)
|
14
|
+
@queue << x
|
15
|
+
thread = @full.first
|
16
|
+
thread.wakeup if thread
|
17
|
+
end
|
18
|
+
|
19
|
+
def pop
|
20
|
+
@queue.shift
|
21
|
+
if @queue.empty?
|
22
|
+
thread = @empty.first
|
23
|
+
thread.wakeup if thread
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def next
|
28
|
+
while @queue.empty?
|
29
|
+
begin
|
30
|
+
@full << Thread.current
|
31
|
+
# If a push happens before Thread.stop, the thread won't be woken up
|
32
|
+
Thread.stop while @queue.empty?
|
33
|
+
ensure
|
34
|
+
@full.delete(Thread.current)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
@queue.first
|
38
|
+
end
|
39
|
+
|
40
|
+
def flush
|
41
|
+
until @queue.empty?
|
42
|
+
begin
|
43
|
+
@empty << Thread.current
|
44
|
+
# If a pop happens before Thread.stop, the thread won't be woken up
|
45
|
+
Thread.stop until @queue.empty?
|
46
|
+
ensure
|
47
|
+
@empty.delete(Thread.current)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def stop
|
53
|
+
@stop = true
|
54
|
+
@heartbeat.join
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
# Check threads 10 times per second to avoid deadlocks
|
60
|
+
# since there is a race condition above
|
61
|
+
def heartbeat
|
62
|
+
until @stop
|
63
|
+
@empty.each(&:wakeup)
|
64
|
+
@full.each(&:wakeup)
|
65
|
+
sleep 0.1
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Daybreak
|
2
|
+
# A queue for threaded implementations of ruby without a GIL
|
3
|
+
# @api private
|
4
|
+
class Queue
|
5
|
+
def initialize
|
6
|
+
@mutex = Mutex.new
|
7
|
+
@full = ConditionVariable.new
|
8
|
+
@empty = ConditionVariable.new
|
9
|
+
@queue = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def <<(x)
|
13
|
+
@mutex.synchronize do
|
14
|
+
@queue << x
|
15
|
+
@full.signal
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def pop
|
20
|
+
@mutex.synchronize do
|
21
|
+
@queue.shift
|
22
|
+
@empty.signal if @queue.empty?
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def next
|
27
|
+
@mutex.synchronize do
|
28
|
+
@full.wait(@mutex) while @queue.empty?
|
29
|
+
@queue.first
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def flush
|
34
|
+
@mutex.synchronize do
|
35
|
+
@empty.wait(@mutex) until @queue.empty?
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/daybreak/version.rb
CHANGED
data/script/bench
CHANGED
@@ -43,6 +43,7 @@ def run(instance, message = '', &blk)
|
|
43
43
|
report measure(instance, &blk)
|
44
44
|
puts '=' * 64
|
45
45
|
ensure
|
46
|
+
instance.clear if instance.respond_to? :clear
|
46
47
|
instance.close if instance.respond_to? :close
|
47
48
|
end
|
48
49
|
|
@@ -70,7 +71,8 @@ rescue Exception => ex
|
|
70
71
|
end
|
71
72
|
|
72
73
|
run Hash.new
|
73
|
-
|
74
|
+
db = Daybreak::DB.new DB_PATH
|
75
|
+
run db
|
74
76
|
|
75
77
|
db = Daybreak::DB.new DB_PATH
|
76
78
|
run db, 'with lock' do
|
@@ -93,3 +95,14 @@ run db, 'with sync' do
|
|
93
95
|
db.sync
|
94
96
|
end
|
95
97
|
end
|
98
|
+
|
99
|
+
hash = DATA.reduce({}) {|m, v| m.merge! v => v }
|
100
|
+
db = Daybreak::DB.new DB_PATH
|
101
|
+
run db, 'bulk updates' do
|
102
|
+
db.update hash
|
103
|
+
end
|
104
|
+
|
105
|
+
db = Daybreak::DB.new DB_PATH
|
106
|
+
run db, 'bulk updates, synced' do
|
107
|
+
db.update! hash
|
108
|
+
end
|
data/test/prof.rb
CHANGED
@@ -4,10 +4,10 @@ require 'ruby-prof'
|
|
4
4
|
|
5
5
|
result = RubyProf.profile do
|
6
6
|
db = Daybreak::DB.new './t.db'
|
7
|
-
|
8
|
-
db.flush
|
7
|
+
10.times {|n| db[n] = n}
|
8
|
+
db.flush
|
9
|
+
db.close
|
9
10
|
end
|
10
|
-
|
11
11
|
File.unlink './t.db'
|
12
12
|
printer = RubyProf::MultiPrinter.new(result)
|
13
13
|
FileUtils.mkdir('./profile') unless File.exists? './profile'
|
data/test/test.rb
CHANGED
@@ -11,6 +11,8 @@ describe Daybreak::DB do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
it 'should insert' do
|
14
|
+
assert_equal @db[1], nil
|
15
|
+
assert_equal @db.include?(1), false
|
14
16
|
@db[1] = 1
|
15
17
|
assert_equal @db[1], 1
|
16
18
|
assert @db.has_key?(1)
|
@@ -19,24 +21,40 @@ describe Daybreak::DB do
|
|
19
21
|
assert_equal @db.length, 1
|
20
22
|
end
|
21
23
|
|
24
|
+
it 'should support batch inserts' do
|
25
|
+
@db.update(1 => :a, 2 => :b)
|
26
|
+
assert_equal @db[1], :a
|
27
|
+
assert_equal @db[2], :b
|
28
|
+
assert_equal @db.length, 2
|
29
|
+
end
|
30
|
+
|
22
31
|
it 'should persist values' do
|
23
32
|
@db['1'] = '4'
|
24
33
|
@db['4'] = '1'
|
25
|
-
@db.sync
|
34
|
+
assert_equal @db.sync, @db
|
26
35
|
|
27
36
|
assert_equal @db['1'], '4'
|
28
37
|
db2 = Daybreak::DB.new DB_PATH
|
29
38
|
assert_equal db2['1'], '4'
|
30
39
|
assert_equal db2['4'], '1'
|
31
|
-
db2.close
|
40
|
+
assert_equal db2.close, nil
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should persist after batch update' do
|
44
|
+
@db.update!(1 => :a, 2 => :b)
|
45
|
+
|
46
|
+
db2 = Daybreak::DB.new DB_PATH
|
47
|
+
assert_equal db2[1], :a
|
48
|
+
assert_equal db2[2], :b
|
49
|
+
assert_equal db2.close, nil
|
32
50
|
end
|
33
51
|
|
34
52
|
it 'should persist after clear' do
|
35
53
|
@db['1'] = 'xy'
|
36
|
-
@db.clear
|
54
|
+
assert_equal @db.clear, @db
|
37
55
|
@db['1'] = '4'
|
38
56
|
@db['4'] = '1'
|
39
|
-
@db.close
|
57
|
+
assert_equal @db.close, nil
|
40
58
|
|
41
59
|
@db = Daybreak::DB.new DB_PATH
|
42
60
|
assert_equal @db['1'], '4'
|
@@ -46,10 +64,10 @@ describe Daybreak::DB do
|
|
46
64
|
it 'should persist after compact' do
|
47
65
|
@db['1'] = 'xy'
|
48
66
|
@db['1'] = 'z'
|
49
|
-
@db.compact
|
67
|
+
assert_equal @db.compact, @db
|
50
68
|
@db['1'] = '4'
|
51
69
|
@db['4'] = '1'
|
52
|
-
@db.close
|
70
|
+
assert_equal @db.close, nil
|
53
71
|
|
54
72
|
@db = Daybreak::DB.new DB_PATH
|
55
73
|
assert_equal @db['1'], '4'
|
@@ -61,10 +79,10 @@ describe Daybreak::DB do
|
|
61
79
|
|
62
80
|
@db['1'] = 'xy'
|
63
81
|
@db['1'] = 'z'
|
64
|
-
@db.compact
|
82
|
+
assert_equal @db.compact, @db
|
65
83
|
@db['1'] = '4'
|
66
84
|
@db['4'] = '1'
|
67
|
-
@db.flush
|
85
|
+
assert_equal @db.flush, @db
|
68
86
|
|
69
87
|
db.sync
|
70
88
|
assert_equal db['1'], '4'
|
@@ -100,16 +118,24 @@ describe Daybreak::DB do
|
|
100
118
|
end
|
101
119
|
|
102
120
|
it 'should allow for default values' do
|
103
|
-
|
104
|
-
assert_equal
|
105
|
-
|
106
|
-
|
107
|
-
|
121
|
+
db = Daybreak::DB.new(DB_PATH, :default => 0)
|
122
|
+
assert_equal db[1], 0
|
123
|
+
assert db.include? '1'
|
124
|
+
db[1] = 1
|
125
|
+
assert_equal db[1], 1
|
126
|
+
db.default = 42
|
127
|
+
assert_equal db['x'], 42
|
128
|
+
db.close
|
108
129
|
end
|
109
130
|
|
110
131
|
it 'should handle default values that are procs' do
|
111
|
-
db = Daybreak::DB.new(DB_PATH) {|key| Set.new }
|
132
|
+
db = Daybreak::DB.new(DB_PATH) {|key| set = Set.new; set << key }
|
112
133
|
assert db['foo'].is_a? Set
|
134
|
+
assert db.include? 'foo'
|
135
|
+
assert db['bar'].include? 'bar'
|
136
|
+
db.default = proc {|key| [key] }
|
137
|
+
assert db[1].is_a? Array
|
138
|
+
assert db[2] == ['2']
|
113
139
|
db.close
|
114
140
|
end
|
115
141
|
|
@@ -192,8 +218,8 @@ describe Daybreak::DB do
|
|
192
218
|
db["b#{i}"] = i
|
193
219
|
sleep 0.01 if i % 100 == 0
|
194
220
|
end
|
195
|
-
|
196
|
-
|
221
|
+
db.close
|
222
|
+
end
|
197
223
|
Process.wait a
|
198
224
|
Process.wait b
|
199
225
|
@db = Daybreak::DB.new DB_PATH
|
@@ -310,13 +336,28 @@ describe Daybreak::DB do
|
|
310
336
|
end
|
311
337
|
end
|
312
338
|
|
313
|
-
it 'should support set! in lock' do
|
339
|
+
it 'should support set! and delete! in lock' do
|
314
340
|
@db[1] = 2
|
315
341
|
@db.lock do
|
316
342
|
@db.set!(1, 2)
|
343
|
+
@db.delete!(1)
|
317
344
|
end
|
318
345
|
end
|
319
346
|
|
347
|
+
it 'should allow for inheritance' do
|
348
|
+
class Subclassed < Daybreak::DB
|
349
|
+
def increment(key, amount = 1)
|
350
|
+
lock { self[key] += amount }
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
354
|
+
db = Subclassed.new DB_PATH
|
355
|
+
db[1] = 1
|
356
|
+
assert_equal db.increment(1), 2
|
357
|
+
db.clear
|
358
|
+
db.close
|
359
|
+
end
|
360
|
+
|
320
361
|
after do
|
321
362
|
@db.clear
|
322
363
|
@db.close
|
metadata
CHANGED
@@ -1,57 +1,62 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: daybreak
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 21
|
5
5
|
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
- 1
|
10
|
+
version: 0.2.1
|
6
11
|
platform: ruby
|
7
|
-
authors:
|
12
|
+
authors:
|
8
13
|
- Jeff Larson
|
9
14
|
- Daniel Mendler
|
10
15
|
autorequire:
|
11
16
|
bindir: bin
|
12
17
|
cert_chain: []
|
13
|
-
|
14
|
-
|
15
|
-
|
18
|
+
|
19
|
+
date: 2013-01-16 00:00:00 Z
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
16
22
|
name: rake
|
17
|
-
requirement: !ruby/object:Gem::Requirement
|
18
|
-
none: false
|
19
|
-
requirements:
|
20
|
-
- - ! '>='
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: '0'
|
23
|
-
type: :development
|
24
23
|
prerelease: false
|
25
|
-
|
26
|
-
none: false
|
27
|
-
requirements:
|
28
|
-
- - ! '>='
|
29
|
-
- !ruby/object:Gem::Version
|
30
|
-
version: '0'
|
31
|
-
- !ruby/object:Gem::Dependency
|
32
|
-
name: minitest
|
33
|
-
requirement: !ruby/object:Gem::Requirement
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
34
25
|
none: false
|
35
|
-
requirements:
|
36
|
-
- -
|
37
|
-
- !ruby/object:Gem::Version
|
38
|
-
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 3
|
30
|
+
segments:
|
31
|
+
- 0
|
32
|
+
version: "0"
|
39
33
|
type: :development
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: minitest
|
40
37
|
prerelease: false
|
41
|
-
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
42
39
|
none: false
|
43
|
-
requirements:
|
44
|
-
- -
|
45
|
-
- !ruby/object:Gem::Version
|
46
|
-
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
hash: 3
|
44
|
+
segments:
|
45
|
+
- 0
|
46
|
+
version: "0"
|
47
|
+
type: :development
|
48
|
+
version_requirements: *id002
|
47
49
|
description: Incredibly fast pure-ruby key-value store
|
48
|
-
email:
|
50
|
+
email:
|
49
51
|
- thejefflarson@gmail.com
|
50
52
|
- mail@daniel-mendler.de
|
51
53
|
executables: []
|
54
|
+
|
52
55
|
extensions: []
|
56
|
+
|
53
57
|
extra_rdoc_files: []
|
54
|
-
|
58
|
+
|
59
|
+
files:
|
55
60
|
- .gitignore
|
56
61
|
- .travis.yml
|
57
62
|
- .yardopts
|
@@ -65,6 +70,8 @@ files:
|
|
65
70
|
- lib/daybreak/db.rb
|
66
71
|
- lib/daybreak/format.rb
|
67
72
|
- lib/daybreak/queue.rb
|
73
|
+
- lib/daybreak/queue/mri.rb
|
74
|
+
- lib/daybreak/queue/threaded.rb
|
68
75
|
- lib/daybreak/serializer.rb
|
69
76
|
- lib/daybreak/version.rb
|
70
77
|
- script/bench
|
@@ -73,33 +80,39 @@ files:
|
|
73
80
|
- test/test.rb
|
74
81
|
- test/test_helper.rb
|
75
82
|
homepage: http://propublica.github.com/daybreak/
|
76
|
-
licenses:
|
83
|
+
licenses:
|
77
84
|
- MIT
|
78
85
|
post_install_message:
|
79
86
|
rdoc_options: []
|
80
|
-
|
87
|
+
|
88
|
+
require_paths:
|
81
89
|
- lib
|
82
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
90
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
83
91
|
none: false
|
84
|
-
requirements:
|
85
|
-
- -
|
86
|
-
- !ruby/object:Gem::Version
|
87
|
-
|
88
|
-
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
hash: 3
|
96
|
+
segments:
|
97
|
+
- 0
|
98
|
+
version: "0"
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
89
100
|
none: false
|
90
|
-
requirements:
|
91
|
-
- -
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
hash: 3
|
105
|
+
segments:
|
106
|
+
- 0
|
107
|
+
version: "0"
|
94
108
|
requirements: []
|
109
|
+
|
95
110
|
rubyforge_project:
|
96
111
|
rubygems_version: 1.8.24
|
97
112
|
signing_key:
|
98
113
|
specification_version: 3
|
99
|
-
summary: Daybreak provides an incredibly fast pure-ruby in memory key-value store,
|
100
|
-
|
101
|
-
test_files:
|
114
|
+
summary: Daybreak provides an incredibly fast pure-ruby in memory key-value store, which is multi-process safe and uses a journal log to store the data.
|
115
|
+
test_files:
|
102
116
|
- test/prof.rb
|
103
117
|
- test/test.rb
|
104
118
|
- test/test_helper.rb
|
105
|
-
has_rdoc:
|