bitcask 0.0.1 → 0.1.0

data/README.markdown

@@ -5,51 +5,81 @@ Utilities for reading the Bitcask file format. You can use this to recover
 deleted values (before they are compacted), recover from a backup, list keys
 to do read-repair when list-keys is malfunctioning, and so forth.
 
-# Open a bitcask.
-b = Bitcask.new '/var/lib/riak/bitcask/0'
-
-# Dump all keys and values, in cron order, excluding tombstones.
-# Data files go in cronological order, so this is in effect replaying history.
-b.data_files.each do |data_file|
-  data_file.each do |key, value|
-    next if value == Bitcask::TOMBSTONE
-    puts key
-    puts value
-  end
-end
-
-# If you know the offset, you can retrieve it directly.
-data_file[0] # => ["key", "value"]
-
-# And step through values one by one.
-data_file.read # => [k1, v1]
-data_file.read # => [k2, v2]
-
-# Seek, rewind, and pos are also supported.
-
-# In Riak, these are erlang terms.
-b.data_files.each do |data_file|
-  data_file.each do |key, value|
-    next if value == Bitcask::TOMBSTONE
-
-    bucket, key = BERT.decode key
-    value = BERT.decode value
-
-    # Store the object's value in riak
-    o = riak[bucket][key]
-    o.raw_data = value.last
-    o.store
-
-    # Or dump the entire value to a file for later inspection.
-    FileUtils.mkdir_p(bucket)
-    File.open(File.join(bucket, key), 'w') do |out|
-      out.write value.to_json
+Install
+-------
+
+    $ gem install bitcask
+
+Examples
+--------
+
+Open a bitcask.
+
+    b = Bitcask.new '/var/lib/riak/bitcask/0'
+
+Load the keydir, using hintfiles where possible.
+
+    b.load
+
+Get a specific entry:
+
+    b['test'] #=> 'value_of_test'
+
+Iterate over all values:
+
+    b.each do |key, value|
+      puts key
+      puts value
+    end
+
+In Riak, these are erlang terms.
+
+    b.each do |key, value|
+      next if value == Bitcask::TOMBSTONE
+
+      bucket, key = BERT.decode key
+      value = BERT.decode value
+
+      # Store the object's value in riak
+      o = riak[bucket][key]
+      o.raw_data = value.last
+      o.store
+
+      # Or dump the entire value to a file for later inspection.
+      FileUtils.mkdir_p(bucket)
+      File.open(File.join(bucket, key), 'w') do |out|
+        out.write value.to_json
+      end
+    end
+
+You can also work directly on the data files. Here's how to dump all keys and
+values, in cron order, excluding tombstones. Data files go in cronological
+order, so this is in effect replaying history since the last merge.
+
+    b.data_files.each do |data_file|
+      data_file.each do |entry|
+        next if entry.value == Bitcask::TOMBSTONE
+        puts entry.key
+        puts entry.value
+      end
     end
-  end
-end
+
+If you know the offset, you can retrieve it directly from a DataFile.
+
+    data_file[0] # => Struct {:key => 'key', :value => 'value'}
+
+And step through values one by one.
+
+    data_file.read # => [k1, v1]
+    data_file.read # => [k2, v2]
+
+Seek, rewind, and pos are also supported.
 
 You'd be surprised how fast this is. 10,000 values/sec, easy.
 
+Status
+------
+
 Anyone who wants to expand this, feel free. I've been using it for emergency
 recovery operations, but don't plan to reimplement bitcask in Ruby myself. I
 welcome pull requests.

data/lib/bitcask.rb

@@ -3,16 +3,29 @@ class Bitcask
 
   $LOAD_PATH << File.expand_path(File.dirname(__FILE__))
 
-  # A single data file
+  require 'bitcask/hint_file'
   require 'bitcask/data_file'
+  require 'bitcask/keydir'
   require 'bitcask/errors'
   require 'bitcask/version'
 
+  include Enumerable
+
   TOMBSTONE = "bitcask_tombstone"
 
   # Opens a bitcask backed by the given directory.
+  attr_accessor :keydir
+  attr_reader :dir
   def initialize(dir)
     @dir = dir
+    @keydir = Bitcask::Keydir.new
+  end
+
+  # Uses the keydir to get an object from the bitcask. Returns a
+  # value.
+  def [](key)
+    index = @keydir[key] or return nil
+    @keydir.data_files[index.file_id][index.value_pos, index.value_sz].value
   end
 
   # Returns a list of all data filenames in this bitcask, sorted from oldest
@@ -29,4 +42,74 @@ class Bitcask
       Bitcask::DataFile.new filename
     end
   end
+
+  # Iterates over all keys in keydir. Yields key, value pairs.
+  def each
+    @keydir.each do |key, index|
+      entry = @keydir.data_files[index.file_id][index.value_pos, index.value_sz]
+      yield [entry.key, entry.value]
+    end
+  end
+
+  # Keydir keys.
+  def keys
+    keydir.keys
+  end
+
+  # Populate the keydir.
+  def load
+    data_files.each do |d|
+      if h = d.hint_file
+        load_hint_file h
+      else
+        load_data_file d
+      end
+    end
+  end
+
+  # Load a DataFile into the keydir.
+  def load_data_file(data_file)
+    # Determine data_file index.
+    @keydir.data_files |= [data_file]
+    file_id = @keydir.data_files.index data_file
+   
+    pos = 0
+    data_file.each do |entry|
+      # Check for existing newer entry in keydir
+      if (cur = @keydir[entry.key]).nil? or entry.tstamp >= cur.tstamp
+        @keydir[entry.key] = Keydir::Entry.new(
+          file_id,
+          data_file.pos - pos,
+          pos,
+          entry.tstamp
+        )
+      end
+
+      pos = data_file.pos
+    end
+  end
+  
+  # Load a HintFile into the keydir.
+  def load_hint_file(hint_file)
+    # Determine data_file index.
+    @keydir.data_files |= [hint_file.data_file]
+    file_id = @keydir.data_files.index hint_file.data_file
+
+    hint_file.each do |entry|
+      # Check for existing newer entry in keydir
+      if (cur = @keydir[entry.key]).nil? or entry.tstamp >= cur.tstamp
+        @keydir[entry.key] = Keydir::Entry.new(
+          file_id,
+          entry.value_sz,
+          entry.value_pos,
+          entry.tstamp
+        )
+      end
+    end
+  end
+
+  # Keydir size.
+  def size
+    @keydir.size
+  end
 end

data/lib/bitcask/data_file.rb

@@ -4,18 +4,22 @@ class Bitcask::DataFile
   # This is most definitely not threadsafe, but it's so cheap you might as well
   # make lots of copies.
 
+  Entry = Struct.new :tstamp, :key, :value
+
+  include Enumerable
+
   def initialize(filename)
     @file = File.open(filename)
   end
 
   # Reads [key, value] from a particular offset.
   # Also advances the cursor.
-  def [](offset)
+  def [](offset, size = nil)
     seek offset
-    read
+    read size
   end
  
-  # Iterates over every entry in this file, yielding the key and value.
+  # Iterates over every entry in this file, yielding an Entry.
   # Options:
   #  :rewind (true) - Rewind the file to the beginning, instead of starting
   #                   right here.
@@ -43,28 +47,45 @@ class Bitcask::DataFile
     end
   end
 
+  def hint_file
+    @hint_file ||= begin
+      path = @file.path.sub(/\.data$/, '.hint')
+      if File.exists? path
+        h = Bitcask::HintFile.new path
+        h.data_file = self
+        h
+      end
+    end
+  end
+
   def pos
     @file.pos
   end
   alias tell pos
 
-  # Returns a single [key, value] pair read from the current offset,
-  # and advances to the next.
+  # Returns a single Entry read from the current offset, and advances to the
+  # next.
   #
   # Can raise Bitcask::ChecksumError
-  def read
+  def read(size = nil)
+    if size
+      f = StringIO.new @file.read(size)
+    else
+      f = @file
+    end
+
     # Parse header
-    header = @file.read(14) or return
+    header = f.read(14) or return
     crc, tstamp, ksz, value_sz = header.unpack "NNnN"
     
     # Read data
-    key = @file.read ksz
-    value = @file.read value_sz
+    key = f.read ksz
+    value = f.read value_sz
 
     # CRC check
     raise Bitcask::ChecksumError unless crc == Zlib.crc32(header[4..-1] + key + value)
 
-    [key, value]
+    Entry.new tstamp, key, value
   end
 
   # Rewinds the file.

data/lib/bitcask/hint_file.rb

@@ -0,0 +1,80 @@
+class Bitcask::HintFile
+  # A single Bitcask hint file.
+  #
+  # This is most definitely not threadsafe, but it's so cheap you might as well
+  # make lots of copies.
+
+  Entry = Struct.new :tstamp, :value_sz, :value_pos, :key
+
+  include Enumerable
+
+  attr_accessor :data_file
+  def initialize(filename)
+    @file = File.open(filename)
+  end
+
+  # Reads [key, value] from a particular offset.
+  # Also advances the cursor.
+  def [](offset)
+    seek offset
+    read
+  end
+ 
+  # Iterates over every entry in this file, yielding an Entry.
+  # Options:
+  #  :rewind (true) - Rewind the file to the beginning, instead of starting
+  #                   right here.
+  #  :raise_checksum (false) - Raise Bitcask::ChecksumError on crc failure, 
+  #                   instead of silently continuing.
+  def each(opts = {})
+    options = {
+      :rewind => true,
+      :raise_checksum => false
+    }.merge opts
+
+    rewind if options[:rewind]
+
+    loop do
+      o = read
+      if o
+        yield o
+      else
+        return self
+      end
+    end
+  end
+
+  def pos
+    @file.pos
+  end
+  alias tell pos
+
+  # Returns [timestamp, key, value_pos, value_size] read from the current
+  # offset, and advances to the next.
+  #
+  # Can raise Bitcask::ChecksumError
+  def read
+    # Parse header
+    header = @file.read(18) or return
+    tstamp, ksz, value_sz, value_pos1, value_pos2 = header.unpack "NnNNN"
+
+    # value_pos is an 8 byte big-endian number...
+    # For reference, reverse is [value_pos >> 32, value & 0xFFFFFFFF].pack("NN")
+    value_pos = (value_pos1 << 32) | value_pos2
+
+    # Read key
+    key = @file.read ksz
+
+    Entry.new tstamp, value_sz, value_pos, key
+  end
+
+  # Rewinds the file.
+  def rewind
+    @file.rewind
+  end
+
+  # Seek to a given offset.
+  def seek(offset)
+    @file.seek offset
+  end
+end

data/lib/bitcask/keydir.rb

@@ -0,0 +1,10 @@
+class Bitcask::Keydir < Hash
+  Entry = Struct.new :file_id, :value_sz, :value_pos, :tstamp
+
+  attr_accessor :data_files
+  def initialize(*a)
+    super *a
+
+    @data_files = []
+  end
+end

data/lib/bitcask/version.rb

@@ -1,3 +1,3 @@
 class Bitcask
-  VERSION = '0.0.1'
+  VERSION = '0.1.0'
 end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 0
   - 1
-  version: 0.0.1
+  - 0
+  version: 0.1.0
 platform: ruby
 authors: 
 - Kyle Kingsbury
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-18 00:00:00 -07:00
+date: 2011-06-11 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -31,10 +31,12 @@ files:
 - lib/bitcask/data_file.rb
 - lib/bitcask/version.rb
 - lib/bitcask/errors.rb
+- lib/bitcask/hint_file.rb
+- lib/bitcask/keydir.rb
 - LICENSE
 - README.markdown
 has_rdoc: true
-homepage: https://github.com/aphyr/bitcask
+homepage: https://github.com/aphyr/bitcask-ruby
 licenses: []
 
 post_install_message: