daybreak 0.1.3 → 0.2.0

data/test/test.rb

@@ -1,21 +1,16 @@
-require 'set'
+require 'minitest/autorun'
+require 'minitest/benchmark'
 
-begin
-  require 'simplecov'
-  SimpleCov.start
-  SimpleCov.command_name "Unit tests"
-rescue Exception => ex
-  puts "No coverage report generated: #{ex.message}"
-end
+require 'set'
 
 require File.expand_path(File.dirname(__FILE__)) + '/test_helper.rb'
 
-describe "database functions" do
+describe Daybreak::DB do
   before do
     @db = Daybreak::DB.new DB_PATH
   end
 
-  it "should insert" do
+  it 'should insert' do
     @db[1] = 1
     assert_equal @db[1], 1
     assert @db.has_key?(1)
@@ -24,98 +19,141 @@ describe "database functions" do
     assert_equal @db.length, 1
   end
 
-  it "should persist values" do
-    @db.set('1', '4', true)
-    @db.set('4', '1', true)
+  it 'should persist values' do
+    @db['1'] = '4'
+    @db['4'] = '1'
+    @db.sync
 
     assert_equal @db['1'], '4'
     db2 = Daybreak::DB.new DB_PATH
     assert_equal db2['1'], '4'
     assert_equal db2['4'], '1'
-    db2.close!
+    db2.close
+  end
+
+  it 'should persist after clear' do
+    @db['1'] = 'xy'
+    @db.clear
+    @db['1'] = '4'
+    @db['4'] = '1'
+    @db.close
+
+    @db = Daybreak::DB.new DB_PATH
+    assert_equal @db['1'], '4'
+    assert_equal @db['4'], '1'
+  end
+
+  it 'should persist after compact' do
+    @db['1'] = 'xy'
+    @db['1'] = 'z'
+    @db.compact
+    @db['1'] = '4'
+    @db['4'] = '1'
+    @db.close
+
+    @db = Daybreak::DB.new DB_PATH
+    assert_equal @db['1'], '4'
+    assert_equal @db['4'], '1'
+  end
+
+  it 'should reload database file in sync after compact' do
+    db = Daybreak::DB.new DB_PATH
+
+    @db['1'] = 'xy'
+    @db['1'] = 'z'
+    @db.compact
+    @db['1'] = '4'
+    @db['4'] = '1'
+    @db.flush
+
+    db.sync
+    assert_equal db['1'], '4'
+    assert_equal db['4'], '1'
+    db.close
   end
 
-  it "should compact cleanly" do
+  it 'should reload database file in sync after clear' do
+    db = Daybreak::DB.new DB_PATH
+
+    @db['1'] = 'xy'
+    @db['1'] = 'z'
+    @db.clear
+    @db['1'] = '4'
+    @db['4'] = '1'
+    @db.flush
+
+    db.sync
+    assert_equal db['1'], '4'
+    assert_equal db['4'], '1'
+    db.close
+  end
+
+  it 'should compact cleanly' do
     @db[1] = 1
     @db[1] = 1
-    @db.flush!
+    @db.sync
+
     size = File.stat(DB_PATH).size
-    @db.compact!
+    @db.compact
     assert_equal @db[1], 1
     assert size > File.stat(DB_PATH).size
   end
 
-  it "should allow for default values" do
-    default_db = Daybreak::DB.new(DB_PATH, 0)
+  it 'should allow for default values' do
+    default_db = Daybreak::DB.new(DB_PATH, :default => 0)
     assert_equal default_db[1], 0
     default_db[1] = 1
     assert_equal default_db[1], 1
+    default_db.close
   end
 
-  it "should handle default values that are procs" do
+  it 'should handle default values that are procs' do
     db = Daybreak::DB.new(DB_PATH) {|key| Set.new }
     assert db['foo'].is_a? Set
+    db.close
   end
 
-  it "should be able to sync competing writes" do
+  it 'should be able to sync competing writes' do
     @db.set! '1', 4
     db2 = Daybreak::DB.new DB_PATH
     db2.set! '1', 5
-    @db.read!
+    @db.sync
     assert_equal @db['1'], 5
+    db2.close
   end
 
-  it "should be able to handle another process's call to compact" do
-    20.times {|i| @db.set i, i, true }
+  it 'should be able to handle another process\'s call to compact' do
+    @db.lock { 20.times {|i| @db[i] = i } }
     db2 = Daybreak::DB.new DB_PATH
-    20.times {|i| @db.set i, i + 1, true }
-    @db.compact!
-    db2.read!
-    assert_equal 20, db2['19']
+    @db.lock { 20.times {|i| @db[i] = i } }
+    @db.compact
+    db2.sync
+    assert_equal 19, db2['19']
+    db2.close
   end
 
-  it "can empty the database" do
+  it 'can empty the database' do
     20.times {|i| @db[i] = i }
-    @db.empty!
+    @db.clear
     db2 = Daybreak::DB.new DB_PATH
     assert_equal nil, db2['19']
+    db2.close
   end
 
-  it "should compact subclassed dbs" do
-    class StringDB < Daybreak::DB
-      def serialize(it)
-        it.to_s
-      end
-
-      def parse(it)
-        it
-      end
-    end
-
-    db = StringDB.new 'string.db'
-    db[1] = 'one'
-    db[2] = 'two'
-    db.delete 2
-    db.compact!
-    assert_equal db[1], 'one'
-    assert_equal db[2], nil
-    db.empty!
-    db.close!
-  end
-
-  it "should handle deletions" do
+  it 'should handle deletions' do
     @db[1] = 'one'
     @db[2] = 'two'
-    @db.delete 'two'
+    @db.delete! 'two'
     assert !@db.has_key?('two')
     assert_equal @db['two'], nil
 
     db2 = Daybreak::DB.new DB_PATH
     assert !db2.has_key?('two')
     assert_equal db2['two'], nil
+    db2.close
   end
 
-  it "should close and reopen the file when clearing the database" do
+  it 'should close and reopen the file when clearing the database' do
     begin
       1000.times {@db.clear}
     rescue
@@ -123,8 +161,164 @@ describe "database functions" do
     end
   end
 
+  it 'should have threadsafe lock' do
+    @db[1] = 0
+    inc = proc { 1000.times { @db.lock { @db[1] += 1 } } }
+    a = Thread.new &inc
+    b = Thread.new &inc
+    a.join
+    b.join
+    assert_equal @db[1], 2000
+  end
+
+  it 'should synchronize across processes' do
+    @db[1] = 0
+    @db.flush
+    @db.close
+    begin
+      a = fork do
+        db = Daybreak::DB.new DB_PATH
+        1000.times do |i|
+          db.lock { db[1] += 1 }
+          db["a#{i}"] = i
+          sleep 0.01 if i % 100 == 0
+        end
+        db.close
+      end
+      b = fork do
+        db = Daybreak::DB.new DB_PATH
+        1000.times do |i|
+          db.lock { db[1] += 1 }
+          db["b#{i}"] = i
+          sleep 0.01 if i % 100 == 0
+        end
+      db.close
+    end
+      Process.wait a
+      Process.wait b
+      @db = Daybreak::DB.new DB_PATH
+      1000.times do |i|
+        assert_equal @db["a#{i}"], i
+        assert_equal @db["b#{i}"], i
+      end
+      assert_equal @db[1], 2000
+    rescue NotImplementedError
+      warn 'fork is not available: skipping multiprocess test'
+      @db = Daybreak::DB.new DB_PATH
+    end
+  end
+
+  it 'should synchronize across threads' do
+    @db[1] = 0
+    @db.flush
+    @db.close
+    a = Thread.new do
+      db = Daybreak::DB.new DB_PATH
+      1000.times do |i|
+        db.lock { db[1] += 1 }
+        db["a#{i}"] = i
+        sleep 0.01 if i % 100 == 0
+      end
+      db.close
+    end
+    b = Thread.new do
+      db = Daybreak::DB.new DB_PATH
+      1000.times do |i|
+        db.lock { db[1] += 1 }
+        db["b#{i}"] = i
+        sleep 0.01 if i % 100 == 0
+      end
+      db.close
+    end
+    a.join
+    b.join
+    @db = Daybreak::DB.new DB_PATH
+    1000.times do |i|
+      assert_equal @db["a#{i}"], i
+      assert_equal @db["b#{i}"], i
+    end
+    assert_equal @db[1], 2000
+  end
+
+  it 'should support background compaction' do
+    @db[1] = 0
+    @db.flush
+    @db.close
+    stop = false
+    a = Thread.new do
+      db = Daybreak::DB.new DB_PATH
+      1000.times do |i|
+        db.lock { db[1] += 1 }
+        db["a#{i}"] = i
+        sleep 0.01 if i % 100 == 0
+      end
+      db.close
+    end
+    b = Thread.new do
+      db = Daybreak::DB.new DB_PATH
+      1000.times do |i|
+        db.lock { db[1] += 1 }
+        db["b#{i}"] = i
+        sleep 0.01 if i % 100 == 0
+      end
+      db.close
+    end
+    c = Thread.new do
+      db = Daybreak::DB.new DB_PATH
+      db.compact until stop
+      db.close
+    end
+    d = Thread.new do
+      db = Daybreak::DB.new DB_PATH
+      db.compact until stop
+      db.close
+    end
+    stop = true
+    a.join
+    b.join
+    c.join
+    d.join
+    @db = Daybreak::DB.new DB_PATH
+    1000.times do |i|
+      assert_equal @db["a#{i}"], i
+      assert_equal @db["b#{i}"], i
+    end
+    assert_equal @db[1], 2000
+  end
+
+  it 'should support compact in lock' do
+    @db[1] = 2
+    @db.lock do
+      @db[1] = 2
+      @db.compact
+    end
+  end
+
+  it 'should support clear in lock' do
+    @db[1] = 2
+    @db.lock do
+      @db[1] = 2
+      @db.clear
+    end
+  end
+
+  it 'should support flush in lock' do
+    @db[1] = 2
+    @db.lock do
+      @db[1] = 2
+      @db.flush
+    end
+  end
+
+  it 'should support set! in lock' do
+    @db[1] = 2
+    @db.lock do
+      @db.set!(1, 2)
+    end
+  end
+
   after do
-    @db.empty!
-    @db.close!
+    @db.clear
+    @db.close
   end
 end

data/test/test_helper.rb

@@ -1,6 +1,3 @@
-require 'minitest/autorun'
-require 'minitest/benchmark'
-
 HERE = File.expand_path(File.dirname(__FILE__))
 DB_PATH = File.join HERE, "test.db"
 

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: daybreak
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
 - Jeff Larson
+- Daniel Mendler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-08 00:00:00.000000000 Z
+date: 2013-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -43,9 +44,10 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: A simple dimple key-value store for ruby.
+description: Incredibly fast pure-ruby key-value store
 email:
 - thejefflarson@gmail.com
+- mail@daniel-mendler.de
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -61,11 +63,12 @@ files:
 - daybreak.gemspec
 - lib/daybreak.rb
 - lib/daybreak/db.rb
-- lib/daybreak/record.rb
+- lib/daybreak/format.rb
+- lib/daybreak/queue.rb
+- lib/daybreak/serializer.rb
 - lib/daybreak/version.rb
-- lib/daybreak/writer.rb
-- test/bench.rb
-- test/compare.rb
+- script/bench
+- script/converter
 - test/prof.rb
 - test/test.rb
 - test/test_helper.rb
@@ -93,11 +96,9 @@ rubyforge_project:
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: Daybreak provides an in memory key-value store that is easily enumerable
-  in ruby.
+summary: Daybreak provides an incredibly fast pure-ruby in memory key-value store,
+  which is multi-process safe and uses a journal log to store the data.
 test_files:
-- test/bench.rb
-- test/compare.rb
 - test/prof.rb
 - test/test.rb
 - test/test_helper.rb

data/lib/daybreak/record.rb

@@ -1,62 +0,0 @@
-module Daybreak
-  # Records define how data is serialized and read from disk.
-  module Record
-    # Thrown when either key or data is missing
-    class UnnacceptableDataError < Exception; end
-
-    # Thrown when there is a CRC mismatch between the data from the disk
-    # and what was written to disk previously.
-    class CorruptDataError < Exception; end
-
-    extend self
-
-    # The mask a record uses to check for deletion.
-    DELETION_MASK = 1 << 31
-
-    # Read a record from an open io source, check the CRC, and set <tt>@key</tt>
-    # and <tt>@data</tt>.
-    # @param [#read] io an IO instance to read from
-
-    # The serialized representation of the key value pair plus the CRC.
-    # @return [String]
-    def serialize(record)
-      raise UnnacceptableDataError, 'key and data must be defined' unless record[0] && record[1]
-      s = key_data_string(record)
-      s << crc_string(s)
-    end
-
-    # Create a new record to read from IO.
-    # @param [#read] io an IO instance to read from
-    def deserialize(buf)
-      record = []
-      masked = read32(buf)
-      # Read the record's key bytes
-      record << buf.slice!(0, masked & (DELETION_MASK - 1)) <<
-        # Read the record's value bytes
-        buf.slice!(0, read32(buf)) <<
-        # Set the deletion flag
-        ((masked & DELETION_MASK) != 0)
-      raise CorruptDataError, 'CRC mismatch' unless buf.slice!(0, 4) == crc_string(key_data_string(record))
-      record
-    end
-
-    private
-
-    # Return the deletion flag plus two length prefixed cells
-    def key_data_string(record)
-      part(record[0], record[0].bytesize + (record[2] ? DELETION_MASK : 0)) << part(record[1], record[1].bytesize)
-    end
-
-    def crc_string(s)
-      [Zlib.crc32(s, 0)].pack('N')
-    end
-
-    def part(data, length)
-      [length].pack('N') << data
-    end
-
-    def read32(buf)
-      buf.slice!(0, 4).unpack('N')[0]
-    end
-  end
-end