daybreak 0.0.1

data/.gitignore

@@ -0,0 +1,8 @@
+*.db
+coverage
+doc
+profile
+.DS_Store
+.yardoc
+Gemfile.lock
+*.rbc

data/.yardopts

@@ -0,0 +1 @@
+--no-private --protected lib/**/*.rb - README EPIGRAPH LICENSE

data/EPIGRAPH

@@ -0,0 +1,6 @@
+Above the escutcheon was a declension of the place-names: Falconer Jail 1871,
+Falconer Reformatory, Falconer Federal Penitentiary, Falconer State Prison,
+Falconer Correctional Facility, and the last, which had never caught on:
+Daybreak House.
+
+John Cheever, Falconer

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in daybreak.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jeff Larson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,13 @@
+             ^^            |
+  daybreak     ^^        \ _ /
+                      -= /   \ =-
+~^~ ^ ^~^~ ~^~ ~ ~^~~^~^-=~=~=-~^~^~^~
+
+Daybreak is a simple key value store for ruby. It has user defined persistence,
+and all data is stored in a table in memory so ruby niceties are available.
+Daybreak's performance is on par with other options like pstore and dbm.
+
+$ gem install daybreak
+
+Docs: http://propublica.github.com/daybreak/
+Issue Tracker: http://propublica.github.com/daybreak/issues

data/Rakefile

@@ -0,0 +1,36 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+task :default do
+  require "./test/test.rb"
+end
+
+desc "Run benchmarks"
+task :bench do
+  require "./test/bench.rb"
+end
+
+desc "Run comparisons with other libraries"
+task :compare do
+  require "./test/compare.rb"
+end
+
+desc "Profile a simple run"
+task :prof do
+  require "./test/prof.rb"
+end
+
+require 'erb'
+
+desc "Write out docs to index.html"
+task :doc do |t|
+  File.open("index.html", 'w').write ERB.new(File.open("index.erb").read).result(binding)
+end
+
+desc "Publish the docs to gh-pages"
+task :publish do |t|
+  `git checkout gh-pages`
+  `git merge master`
+  `git push`
+  `git checkout master`
+end

data/daybreak.gemspec

@@ -0,0 +1,18 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/daybreak/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jeff Larson"]
+  gem.email         = ["thejefflarson@gmail.com"]
+  gem.description   = %q{A simple dimple key-value store for ruby.}
+  gem.summary       = %q{Daybreak provides an in memory key-value store that is easily enumerable in ruby.}
+  gem.homepage      = "http://propublica.github.com/daybreak/"
+
+  gem.files         = `git ls-files`.split($\).reject {|f| f =~ /^(index)/}
+  gem.test_files    = gem.files.grep(%r{^(test)/})
+  gem.name          = "daybreak"
+  gem.require_paths = ["lib"]
+  gem.version       = Daybreak::VERSION
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'ruby-prof'
+end

data/lib/daybreak.rb

@@ -0,0 +1,17 @@
+# Daybreak, a simple dimple key value store for ruby.
+module Daybreak
+  # The root path for Daybreak
+  ROOT = File.expand_path(File.dirname(__FILE__))
+end
+
+require 'tempfile'
+require 'thread'
+require 'zlib'
+require 'fcntl'
+
+require "#{Daybreak::ROOT}/daybreak/version"
+require "#{Daybreak::ROOT}/daybreak/locking"
+require "#{Daybreak::ROOT}/daybreak/record"
+require "#{Daybreak::ROOT}/daybreak/writer"
+require "#{Daybreak::ROOT}/daybreak/reader"
+require "#{Daybreak::ROOT}/daybreak/db"

data/lib/daybreak/db.rb

@@ -0,0 +1,163 @@
+module Daybreak
+  # Daybreak::DB contains the public api for Daybreak, you may extend it like
+  # any other Ruby class (i.e. to overwrite serialize and parse). It includes
+  # Enumerable for functional goodies like map, each, reduce and friends.
+  class DB
+    include Enumerable
+
+    # Create a new Daybreak::DB. The second argument is the default value
+    # to store when accessing a previously unset key, this follows the
+    # Hash standard.
+    # @param [String] file the path to the db file
+    # @param default the default value to store and return when a key is
+    #  not yet in the database.
+    # @yield [key] blk a block that will return the default value to store.
+    def initialize(file, default=nil, &blk)
+      @file_name = file
+      reset!
+      @default = default
+      @default = blk if block_given?
+      read!
+    end
+
+    # Set a key in the database to be written at some future date. If the data
+    # needs to be persisted immediately, call <tt>db.set(key, value, true)</tt>.
+    # @param [#to_s] key the key of the storage slot in the database
+    # @param value the value to store
+    # @param [Boolean] sync if true, sync this value immediately
+    def []=(key, value, sync = false)
+      key = key.to_s
+      @writer.write(Record.new(key, serialize(value)))
+      flush! if sync
+      @table[key] = value
+    end
+    alias_method :set, :"[]="
+
+    # set! flushes data immediately to disk.
+    # @param [#to_s] key the key of the storage slot in the database
+    # @param value the value to store
+    def set!(key, value)
+      set key, value, true
+    end
+
+    # Retrieve a value at key from the database. If the default value was specified
+    # when this database was created, that value will be set and returned. Aliased
+    # as <tt>get</tt>.
+    # @param [#to_s] key the value to retrieve from the database.
+    def [](key)
+      key = key.to_s
+      if @table.has_key? key
+        @table[key]
+      elsif default?
+        if @default.is_a? Proc
+          value = @default.call(key)
+        else
+          value = @default
+        end
+        set key, value
+      end
+    end
+    alias_method :get, :"[]"
+
+    # Iterate over the key, value pairs in the database.
+    def each(&blk)
+      keys.each { |k| blk.call(k, get(k)) }
+    end
+
+    # Does this db have a default value.
+    def default?
+      !@default.nil?
+    end
+
+    # Does this db have a value for this key?
+    def has_key?(key)
+      @table.has_key? key.to_s
+    end
+
+    # Return the keys in the db.
+    # @return [Array<String>]
+    def keys
+      @table.keys
+    end
+
+    # Return the number of stored items.
+    # @return [Integer]
+    def length
+      @table.keys.length
+    end
+
+    # Serialize the data for writing to disk, if you don't want to use <tt>Marshal</tt>
+    # overwrite this method.
+    # @param value the value to be serialized
+    # @return [String]
+    def serialize(value)
+      Marshal.dump(value)
+    end
+
+    # Parse the serialized value from disk, like serialize if you want to use a
+    # different serialization method overwrite this method.
+    # @param value the value to be parsed
+    # @return [String]
+    def parse(value)
+      Marshal.load(value)
+    end
+
+    # Reset and empty the database file.
+    def empty!
+      reset!
+      @writer.truncate!
+    end
+
+    # Force all queued commits to be written to disk.
+    def flush!
+      @writer.flush!
+    end
+
+    # Reset the state of the database, you should call <tt>read!</tt> after calling this.
+    def reset!
+      @table  = {}
+      @writer = Daybreak::Writer.new(@file_name)
+      @reader = Daybreak::Reader.new(@file_name)
+    end
+
+    # Close the database for reading and writing.
+    def close!
+      @writer.close!
+      @reader.close!
+    end
+
+    # Compact the database to remove stale commits and reduce the file size.
+    def compact!
+      # Create a new temporary file
+      tmp_file = Tempfile.new File.basename(@file_name)
+      copy_db  = DB.new tmp_file.path
+
+      # Copy the database key by key into the temporary table
+      each do |key, i|
+        copy_db.set(key, get(key))
+      end
+      copy_db.close!
+
+      # Empty this database
+      empty!
+
+      # Move the copy into place
+      tmp_file.close
+      FileUtils.mv tmp_file.path, @file_name
+      tmp_file.unlink
+
+      # Reset this database
+      close!
+      reset!
+      read!
+    end
+
+    # Read all values from the log file. If you want to check for changed data
+    # call this again.
+    def read!
+      @reader.read do |record|
+        @table[record.key] = parse record.data
+      end
+    end
+  end
+end

data/lib/daybreak/locking.rb

@@ -0,0 +1,14 @@
+module Daybreak
+  # File locking mixin
+  module Locking
+    # Lock a file with the type <tt>lock</tt>
+    def lock(fd, lock=File::LOCK_SH)
+      fd.flock lock
+      begin
+        yield
+      ensure
+        fd.flock File::LOCK_UN
+      end
+    end
+  end
+end

data/lib/daybreak/reader.rb

@@ -0,0 +1,39 @@
+module Daybreak
+  # Class for building out the table, you shouldn't need to access this
+  # class directly. Readers are responsible for reading each record in
+  # the file and yeilding the parsed records.
+  class Reader
+    # @param [String] file the file to read from
+    def initialize(file)
+      @file_name = file
+    end
+
+    # Close's the Reader's file descriptor
+    def close!
+      @fd.close unless @fd.nil?
+    end
+
+    # Right now this is really expensive, every call to read will
+    # close and reread the whole db file, but since cross process
+    # consistency is handled by the user, this should be fair warning.
+    def read(&blk)
+      reopen!
+      while !@fd.eof?
+        blk.call(Record.read(@fd))
+      end
+    end
+
+    private
+
+    def reopen!
+      close!
+      open!
+    end
+
+    def open!
+      @fd = File.open @file_name, 'r'
+      @fd.binmode
+      @fd.advise(:sequential) if @fd.respond_to? :advise
+    end
+  end
+end

data/lib/daybreak/record.rb

@@ -0,0 +1,64 @@
+module Daybreak
+  # Records define how data is serialized and read from disk.
+  class Record
+    # Thrown when either key or data is missing
+    class UnnacceptableDataError < Exception; end
+
+    # Thrown when there is a CRC mismatch between the data from the disk
+    # and what was written to disk previously.
+    class CorruptDataError < Exception; end
+    include Locking
+
+    attr_accessor :key, :data
+
+    def initialize(key = nil, data = nil)
+      @key  = key
+      @data = data
+    end
+
+    # Read a record from an open io source, check the CRC, and set @key and @data
+    # @param [#read] io an IO instance to read from
+    def read(io)
+      lock io do
+        @key  = read_bytes(io)
+        @data = read_bytes(io)
+        crc   = io.read(4)
+        raise CorruptDataError, "CRC mismatch #{crc} should be #{crc_string}" unless crc == crc_string
+      end
+      self
+    end
+
+    # The serialized representation of the key value pair plus the CRC
+    # @return [String]
+    def representation
+      raise UnnacceptableDataError, "key and data must be defined" if @key.nil? || @data.nil?
+      byte_string + crc_string
+    end
+
+    # Create a new record to read from IO.
+    # @param [#read] io an IO instance to read from
+    def self.read(io)
+      new.read(io)
+    end
+
+    private
+
+    def byte_string
+      @byte_string ||= part(@key) + part(@data)
+    end
+
+    def crc_string
+      [Zlib.crc32(byte_string, 0)].pack('N')
+    end
+
+    def read_bytes(io)
+      raw = io.read(4)
+      length = raw.unpack('N')[0]
+      io.read(length)
+    end
+
+    def part(data)
+      [data.bytesize].pack('N') + data
+    end
+  end
+end

data/lib/daybreak/version.rb

@@ -0,0 +1,4 @@
+module Daybreak
+  # Updated using SemVer
+  VERSION = "0.0.1"
+end

data/lib/daybreak/writer.rb

@@ -0,0 +1,110 @@
+module Daybreak
+  # Writer's handle the actually fiddly task of committing data to disk.
+  # They have a Worker instance that writes in a select loop.
+  class Writer
+    # Open up the file, ready it for binary and nonblocking writing.
+    def initialize(file)
+      @fd = File.open file, 'a'
+      @fd.binmode
+
+      f = @fd.fcntl(Fcntl::F_GETFL, 0)
+      @fd.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK | f)
+
+      @worker = Worker.new(@fd)
+    end
+
+    # Send a record to the workers queue.
+    def write(record)
+      @worker.enqueue record
+    end
+
+    # Finish writing
+    def finish!
+      @worker.finish!
+    end
+
+    # Flush pending commits, and restart the worker.
+    def flush!
+      @worker.flush!
+    end
+
+    # Finish writing and close the file descriptor
+    def close!
+      finish!
+      @fd.close
+    end
+
+    # Truncate the file.
+    def truncate!
+      @fd.truncate(0)
+    end
+
+    private
+
+    # Workers handle the actual fiddly bits of asynchronous io and
+    # and handle background writes.
+    class Worker
+      include Locking
+
+      def initialize(fd)
+        @queue  = Queue.new
+        @fd     = fd
+        @buffer = ""
+        @thread = Thread.new { work }
+        at_exit { finish! }
+      end
+
+      # Queue up a write to be committed later.
+      def enqueue(record)
+        @queue << record.representation
+      end
+
+      # Loop and block if we don't have work to do or if
+      # the file isn't ready for another write just yet.
+      def work
+        buf = ""
+        loop do
+          str = @queue.pop
+          if str.nil?
+            @fd.flush
+            break
+          end
+          buf << str
+          read, write = IO.select [], [@fd]
+          if write and fd = write.first
+            lock(fd, File::LOCK_EX) { buf = try_write fd, buf }
+          end
+        end
+      end
+
+      # Try and write the buffer to the file via non blocking file writes.
+      # If the write fails try again.
+      def try_write(fd, buf)
+        begin
+          s = fd.write_nonblock(buf)
+          if s < buf.length
+            buf = buf[s..-1] # didn't finish
+          else
+            buf = ""
+          end
+        rescue Errno::EAGAIN
+          buf = buf # try this again
+        end
+        buf
+      end
+
+      # finish! and start up another worker thread.
+      def flush!
+        finish!
+        @thread = Thread.new { work }
+        true
+      end
+
+      # Push a nil through the queue and block until the write loop is finished.
+      def finish!
+        @queue.push nil
+        @thread.join
+      end
+    end
+  end
+end

data/test/bench.rb

@@ -0,0 +1,28 @@
+require File.expand_path(File.dirname(__FILE__)) + '/test_helper.rb'
+
+describe "benchmarks" do
+  before do
+    @db = Daybreak::DB.new DB_PATH
+    1000.times {|i| @db[i] = i }
+    @db.flush!
+  end
+
+  bench_performance_constant "keys with sync" do |n|
+    n.times {|i| @db.set(i, 'i' * i, true) }
+  end
+
+  bench_performance_constant "inserting keys" do |n|
+    n.times {|i| @db[i] = 'i' * i }
+  end
+
+  bench_performance_constant "reading keys" do |n|
+    n.times {|i| assert_equal i % 1000, @db[i % 1000] }
+  end
+
+  after do
+    @db.empty!
+    @db.close!
+    File.unlink(DB_PATH)
+  end
+end
+

data/test/compare.rb

@@ -0,0 +1,47 @@
+require File.expand_path(File.dirname(__FILE__)) + '/test_helper.rb'
+require 'pstore'
+
+describe "compare with pstore" do
+  before do
+    @pstore = PStore.new(File.join(HERE, "test.pstore"))
+  end
+
+  bench_performance_constant "pstore bulk performance" do |n|
+    @pstore.transaction do
+      n.times do |i|
+        @pstore[i] = 'i' * i
+      end
+    end
+  end
+
+  after do
+    File.unlink File.join(HERE, "test.pstore")
+  end
+end
+
+require 'dbm'
+
+describe "compare with dbm" do
+  before do
+    @dbm = DBM.open(File.join(HERE, "test-dbm"), 666, DBM::WRCREAT)
+    1000.times {|i| @dbm[i.to_s] = i }
+  end
+
+  bench_performance_constant "DBM write performance" do |n|
+    n.times do |i|
+      @dbm[i.to_s] = 'i' * i
+    end
+  end
+
+  bench_performance_constant "DBM read performance" do |n|
+    n.times do |i|
+      assert_equal (i % 1000).to_s, @dbm[(i % 1000).to_s]
+    end
+  end
+
+  after do
+    @dbm.close
+
+    File.unlink File.join(HERE, "test-dbm.db")
+  end
+end

data/test/prof.rb

@@ -0,0 +1,14 @@
+require File.expand_path(File.dirname(__FILE__)) + '/test_helper.rb'
+
+require 'ruby-prof'
+
+result = RubyProf.profile do
+  db = Daybreak::DB.new './t.db'
+  1000.times {|n| db[n] = n}
+  db.flush!
+end
+
+File.unlink './t.db'
+printer = RubyProf::MultiPrinter.new(result)
+FileUtils.mkdir('./profile') unless File.exists? './profile'
+printer.print :path => './profile', :profile => 'profile'

data/test/test.rb

@@ -0,0 +1,81 @@
+require 'rubygems'
+require 'simplecov'
+require 'set'
+SimpleCov.start
+SimpleCov.command_name "Unit tests"
+
+require File.expand_path(File.dirname(__FILE__)) + '/test_helper.rb'
+
+describe "database functions" do
+  before do
+    @db = Daybreak::DB.new DB_PATH
+  end
+
+  it "should insert" do
+    @db[1] = 1
+    assert_equal @db[1], 1
+    assert @db.has_key?(1)
+    @db[1] = '2'
+    assert_equal @db[1], '2'
+    assert_equal @db.length, 1
+  end
+
+  it "should persist values" do
+    @db.set('1', '4', true)
+    @db.set('4', '1', true)
+
+    assert_equal @db['1'], '4'
+    db2 = Daybreak::DB.new DB_PATH
+    assert_equal db2['1'], '4'
+    assert_equal db2['4'], '1'
+    db2.close!
+  end
+
+  it "should compact cleanly" do
+    @db[1] = 1
+    @db[1] = 1
+    @db.flush!
+    size = File.stat(DB_PATH).size
+    @db.compact!
+    assert_equal @db[1], 1
+    assert size > File.stat(DB_PATH).size
+  end
+
+  it "should allow for default values" do
+    default_db = Daybreak::DB.new(DB_PATH, 0)
+    assert_equal default_db[1], 0
+    default_db[1] = 1
+    assert_equal default_db[1], 1
+  end
+
+  it "should handle default values that are procs" do
+    db = Daybreak::DB.new(DB_PATH) {|key| Set.new }
+    assert db['foo'].is_a? Set
+  end
+
+  it "should be able to sync competing writes" do
+    @db.set! '1', 4
+    db2 = Daybreak::DB.new DB_PATH
+    db2.set! '1', 5
+    @db.read!
+    assert_equal @db['1'], 5
+    @db.close!
+  end
+
+  it " should be able to handle another process's call to compact" do
+    20.times {|i| @db.set i, i, true }
+    db2 = Daybreak::DB.new DB_PATH
+    20.times {|i| @db.set i, i + 1, true }
+    @db.compact!
+    db2.read!
+    assert_equal 20, db2['19']
+  end
+
+  after do
+    @db.empty!
+    @db.close!
+    File.unlink(DB_PATH)
+  end
+end
+
+

data/test/test_helper.rb

@@ -0,0 +1,7 @@
+require 'minitest/autorun'
+require 'minitest/benchmark'
+
+HERE = File.expand_path(File.dirname(__FILE__))
+DB_PATH = File.join HERE, "test.db"
+
+require File.join HERE, '..', 'lib', 'daybreak'

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: daybreak
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jeff Larson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple dimple key-value store for ruby.
+email:
+- thejefflarson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .yardopts
+- EPIGRAPH
+- Gemfile
+- LICENSE
+- README
+- Rakefile
+- daybreak.gemspec
+- lib/daybreak.rb
+- lib/daybreak/db.rb
+- lib/daybreak/locking.rb
+- lib/daybreak/reader.rb
+- lib/daybreak/record.rb
+- lib/daybreak/version.rb
+- lib/daybreak/writer.rb
+- test/bench.rb
+- test/compare.rb
+- test/prof.rb
+- test/test.rb
+- test/test_helper.rb
+homepage: http://propublica.github.com/daybreak/
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Daybreak provides an in memory key-value store that is easily enumerable
+  in ruby.
+test_files:
+- test/bench.rb
+- test/compare.rb
+- test/prof.rb
+- test/test.rb
+- test/test_helper.rb
+has_rdoc: