bitcask 0.0.1

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2011 Kyle Kingsbury
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,62 @@
+Bitcask
+=======
+
+Utilities for reading the Bitcask file format. You can use this to recover
+deleted values (before they are compacted), recover from a backup, list keys
+to do read-repair when list-keys is malfunctioning, and so forth.
+
+# Open a bitcask.
+b = Bitcask.new '/var/lib/riak/bitcask/0'
+
+# Dump all keys and values, in cron order, excluding tombstones.
+# Data files go in cronological order, so this is in effect replaying history.
+b.data_files.each do |data_file|
+  data_file.each do |key, value|
+    next if value == Bitcask::TOMBSTONE
+    puts key
+    puts value
+  end
+end
+
+# If you know the offset, you can retrieve it directly.
+data_file[0] # => ["key", "value"]
+
+# And step through values one by one.
+data_file.read # => [k1, v1]
+data_file.read # => [k2, v2]
+
+# Seek, rewind, and pos are also supported.
+
+# In Riak, these are erlang terms.
+b.data_files.each do |data_file|
+  data_file.each do |key, value|
+    next if value == Bitcask::TOMBSTONE
+
+    bucket, key = BERT.decode key
+    value = BERT.decode value
+
+    # Store the object's value in riak
+    o = riak[bucket][key]
+    o.raw_data = value.last
+    o.store
+
+    # Or dump the entire value to a file for later inspection.
+    FileUtils.mkdir_p(bucket)
+    File.open(File.join(bucket, key), 'w') do |out|
+      out.write value.to_json
+    end
+  end
+end
+
+You'd be surprised how fast this is. 10,000 values/sec, easy.
+
+Anyone who wants to expand this, feel free. I've been using it for emergency
+recovery operations, but don't plan to reimplement bitcask in Ruby myself. I
+welcome pull requests.
+
+License
+-------
+
+This software was written by Kyle Kingsbury <aphyr@aphyr.com>, at Remixation,
+Inc., for their iPad social video app "Showyou". Released under the MIT
+license.

data/lib/bitcask.rb

@@ -0,0 +1,32 @@
+class Bitcask
+  require 'zlib'
+
+  $LOAD_PATH << File.expand_path(File.dirname(__FILE__))
+
+  # A single data file
+  require 'bitcask/data_file'
+  require 'bitcask/errors'
+  require 'bitcask/version'
+
+  TOMBSTONE = "bitcask_tombstone"
+
+  # Opens a bitcask backed by the given directory.
+  def initialize(dir)
+    @dir = dir
+  end
+
+  # Returns a list of all data filenames in this bitcask, sorted from oldest
+  # to newest.
+  def data_file_names
+    Dir.glob(File.join(@dir, '*.data')).sort! do |a, b|
+      a.to_i <=> b.to_i
+    end
+  end
+
+  # Returns a list of Bitcask::DataFiles in chronological order.
+  def data_files
+    data_file_names.map! do |filename|
+      Bitcask::DataFile.new filename
+    end
+  end
+end

data/lib/bitcask/data_file.rb

@@ -0,0 +1,79 @@
+class Bitcask::DataFile
+  # A single Bitcask data file.
+  #
+  # This is most definitely not threadsafe, but it's so cheap you might as well
+  # make lots of copies.
+
+  def initialize(filename)
+    @file = File.open(filename)
+  end
+
+  # Reads [key, value] from a particular offset.
+  # Also advances the cursor.
+  def [](offset)
+    seek offset
+    read
+  end
+ 
+  # Iterates over every entry in this file, yielding the key and value.
+  # Options:
+  #  :rewind (true) - Rewind the file to the beginning, instead of starting
+  #                   right here.
+  #  :raise_checksum (false) - Raise Bitcask::ChecksumError on crc failure, 
+  #                   instead of silently continuing.
+  def each(opts = {})
+    options = {
+      :rewind => true,
+      :raise_checksum => false
+    }.merge opts
+
+    rewind if options[:rewind]
+
+    loop do
+      begin
+        o = read
+        if o
+          yield o
+        else
+          return self
+        end
+      rescue Bitcask::ChecksumError => e
+        raise e if options[:raise]
+      end
+    end
+  end
+
+  def pos
+    @file.pos
+  end
+  alias tell pos
+
+  # Returns a single [key, value] pair read from the current offset,
+  # and advances to the next.
+  #
+  # Can raise Bitcask::ChecksumError
+  def read
+    # Parse header
+    header = @file.read(14) or return
+    crc, tstamp, ksz, value_sz = header.unpack "NNnN"
+    
+    # Read data
+    key = @file.read ksz
+    value = @file.read value_sz
+
+    # CRC check
+    raise Bitcask::ChecksumError unless crc == Zlib.crc32(header[4..-1] + key + value)
+
+    [key, value]
+  end
+
+  # Rewinds the file.
+  def rewind
+    @file.rewind
+  end
+
+  # Seek to a given offset.
+  def seek(offset)
+    @file.seek offset
+  end
+end

data/lib/bitcask/errors.rb

@@ -0,0 +1,5 @@
+class Bitcask::Error < RuntimeError
+end
+
+class Bitcask::ChecksumError < Bitcask::Error
+end

data/lib/bitcask/version.rb

@@ -0,0 +1,3 @@
+class Bitcask
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification 
+name: bitcask
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Kyle Kingsbury
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-18 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: aphyr@aphyr.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/bitcask.rb
+- lib/bitcask/data_file.rb
+- lib/bitcask/version.rb
+- lib/bitcask/errors.rb
+- LICENSE
+- README.markdown
+has_rdoc: true
+homepage: https://github.com/aphyr/bitcask
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 8
+      - 6
+      version: 1.8.6
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: bitcask
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: An (incomplete) interface to the Bitcask storage system
+test_files: []
+