bitcask 0.2.0 → 0.2.1

data/bin/bitcask

@@ -0,0 +1,370 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("#{File.dirname(__FILE__)}/../lib/bitcask")
+require 'rubygems'
+require 'trollop'
+require 'ostruct'
+
+class Bitcask::Tool
+  K = Struct.new :key
+  KV = Struct.new :key, :value
+
+  def run
+    subcommands = %w(count last get dump)
+
+    opts = Trollop::options do
+      banner <<EOF
+Bitcask utility.
+
+bitcask <bitcask_dir> [options] <command> [subcommand options]
+
+Commands: #{subcommands.join(', ')}.
+
+Options:
+EOF
+
+      opt :key, "key", :type => :string
+      opt :bucket, "bucket", :type => :string
+      opt :value, "value", :type => :string
+      opt :color, "Colorize output"
+      opt :no_riak, "Do not interpret buckets and keys as Riak does"
+      opt :no_values, "Do not process values"
+      opt :no_regexp, "Do not interpret patterns as regular expressions."
+      opt :format, "Output format string.
+%b: Bucket
+%k: Key
+%v: Value
+      ", :type => :string
+      opt :verbose_values, "Display full riak values"
+      stop_on subcommands
+    end
+
+    # Set no_x options.
+    opts = opts.inject({}) do |opts, pair|
+      k,v = pair
+      if k.to_s =~ /^no_(\w+)/
+        opts[$1.to_sym] = ! v
+      else
+        opts[k] = v
+      end
+
+      opts
+    end
+
+    # Load bitcasks
+    bitcasks = bitcasks(ARGV.shift)
+    if bitcasks.empty?
+      Trollop::die "No bitcasks"
+    end
+
+    # Load libraries
+    require 'bert' if opts[:riak]
+    require 'ansi' if opts[:color]
+    require 'pp' if opts[:verbose_values]
+
+    # Subcommand options
+    command = ARGV.shift
+    opts.merge!(
+      case command
+        when 'all'
+          {}
+        when 'count'
+          {}
+        when 'dump'
+          {}
+        when 'get'
+          if opts[:riak]
+            {
+              :bucket => ARGV.shift,
+              :key =>  ARGV.shift
+            }
+          else
+            {:key => ARGV.shift}
+          end
+        when 'last'
+          Trollop::options do
+            opt :limit, 'results to return', :default => 1
+          end
+        else
+          Trollop::die "unknown command #{command.inspect}"
+      end
+    )
+
+    # Run commands
+    send command, bitcasks, opts
+  end
+
+  # Returns all the data in a set of bitcasks.
+  # This differs from dump in that it loads the keydir, so it shows
+  # only the most recent values.
+  def all(bitcasks, opts)
+    f = filter opts
+
+    bitcasks.each do |bitcask|
+      bitcask.load
+
+      if opts[:value]
+        bitcask.keydir.each_key do |key|
+          if f[key, nil]
+            # Get the value
+            value = bitcask[key]
+            if f[key, value]
+              out KV.new(key, value), opts
+            end
+          end
+        end
+      else
+        # Use the keydir alone.
+        bitcask.keydir.each_key do |key|
+          if f[key, nil]
+            out K.new(key), opts
+          end
+        end
+      end
+
+      # Let GC do its thing.
+      bitcask.keydir = nil
+    end
+  end
+
+  # Returns all bitcasks in directory.
+  def bitcasks(directory)
+    return [] unless File.directory? directory
+
+    entries = Dir.entries(directory).map do |f|
+      next if f == '.' or f == '..'
+      File.join(directory, f)
+    end.compact
+
+    if entries.any? { |f| File.file? f and f =~ /\.data$/ }
+      return [Bitcask.new(directory)]
+    end
+
+    entries.map do |e|
+      bitcasks e
+    end.flatten
+  end
+
+  # Colorizes a string.
+  def color(color, str, opts)
+    if opts[:color]
+      ANSI.style(color) { str }
+    else
+      str
+    end
+  end
+
+  # Count all the keys matching opts
+  def count(bitcasks, opts)
+    f = filter opts
+
+    require 'set'
+    keys = Set.new
+    bitcasks.each do |bitcask|
+      bitcask.data_files.each do |d|
+        if h = d.hint_file
+          h.each do |e|
+            if f[e.key, nil]
+              keys << e.key
+            end
+          end
+        else
+          d.each do |e|
+            if f[e.key, e.value]
+              keys << e.key
+            end
+          end
+        end
+      end
+    end
+
+    puts keys.size
+  end
+
+  # Dump every record in each bitcask, cronologically.
+  def dump(bitcasks, opts)
+    f = filter opts
+
+    bitcasks.each do |bitcask|
+      bitcask.data_files.each do |d|
+        if opts[:values] or not d.hint_file
+          # Iterate over data files.
+          d.each do |e|
+            if f[e.key, e.value]
+              out e, opts
+            end
+          end
+        else
+          # Use hint files where possible.
+          d.hint_file.each do |e|
+            if f[e.key, nil]
+              out e, opts
+            end
+          end
+        end
+      end
+    end
+  end
+
+  # Returns a proc to match bitcask entries.
+  def filter(opts)
+    pkey = pattern opts[:key], opts
+    pbucket = pattern opts[:bucket], opts
+    pvalue = pattern opts[:value], opts
+
+    # Trivial case: don't filter anything.
+    if pkey.nil? and pbucket.nil? and pvalue.nil?
+      return lambda { |k, v| true }
+    end
+
+    if opts[:riak]
+      # Taking advantage of the fact that erlang tuples are encoded as
+      # tuple, Size, string, Size, Data, string, Size, Data
+      bucket_part = pbucket.kind_of?(String) ? Regexp.escape(pbucket) : pbucket
+      key_part = pkey.kind_of?(String) ? Regexp.escape(pkey) : pkey
+      p_raw_key = /#{bucket_part}.+#{key_part}/
+  
+      lambda do |key, value|
+        if p_raw_key === key or (pkey.nil? and pvalue.nil? and pvalue)
+
+          # Confirm match.
+          bucket, key = BERT.decode key
+     
+          (not pkey or pkey === key) and
+          (not pbucket or pbucket === bucket) and
+          (not pvalue or !value or pvalue === value)
+        end
+      end
+    else
+      lambda do |key, value|
+        (not pkey or pkey === key) and
+        (not pvalue or !value or pvalue === value)
+      end
+    end
+  end
+ 
+  # Get a specific value.
+  def get(bitcasks, opts)
+    if opts[:riak]
+      t = BERT::Tuple.new
+      t << opts[:bucket]
+      t << opts[:key]
+      key = BERT::encode t
+    else
+      key = opts[:key]
+    end
+
+    key.force_encoding('BINARY')
+    value = bitcasks.inject(nil) do |value, bitcask|
+      value or begin
+        bitcask.load
+        bitcask[key]
+      end
+    end
+    
+    if value
+      out OpenStruct.new('key' => key, 'value' => value), opts
+    end
+  end
+
+  # Dump the last few items from some bitcasks.
+  def last(bitcasks, opts)
+    f = filter opts
+
+    buffers = bitcasks.map do |bitcask|
+      buffer = Array.new opts[:limit]
+
+      bitcask.data_files.each do |d|
+        # Get hint file
+        unless h = d.hint_file
+          warn "No hint file for #{d.inspect}"
+          next
+        end
+    
+        # Run over hintfile    
+        h.each do |e|
+          if f[e.key, nil]
+            buffer.shift
+            buffer << [d, e]
+          end
+        end
+      end
+
+      buffer.compact
+    end
+
+    # Merge buffers.
+    buffer = buffers.flatten(1).sort do |a, b|
+      a[1].tstamp <=> b[1].tstamp
+    end.last(opts[:limit])
+    
+    # Display buffers.  
+    buffer.map do |d, e|
+      if opts[:values]
+        d[e.value_pos, e.value_sz]
+      else
+        e
+      end
+    end.each do |e|
+      out e, opts
+    end
+  end
+
+  # Returns an object which uses === for comparison
+  def pattern(string, opts = {})
+    return nil unless string
+
+    if opts[:regexp] and string =~ /[^A-Za-z0-0_]/
+      /#{string}/
+    else
+      string
+    end
+  end
+
+  # Output an entry.
+  def out(entry, opts = {})
+    # Value
+    if entry.respond_to? :value and entry.value == Bitcask::TOMBSTONE
+      svalue = color :red, entry.value, opts
+    end
+ 
+    if opts[:riak]
+      # Riak
+      format = opts[:format] || "%b/%k\n%v"
+
+      bucket, key = BERT.decode entry.key
+      
+      sbucket = color :green, bucket, opts
+      skey = color :green, key, opts
+
+      svalue ||= begin
+        value = BERT.decode entry.value
+        if opts[:verbose_values]
+          svalue = PP.pp(value, '')
+        else
+          svalue = value.last.to_s
+        end
+      end
+    else
+      # Not riak
+      format = opts[:format] || "%k\n%v"
+      skey = color :green, entry.key, opts
+      svalue ||= entry.value
+      sbucket = ''
+    end
+  
+    # Format
+    s = format.gsub('%k', skey).gsub('%v', svalue).gsub('%b', sbucket)
+
+    puts s
+    s
+  end
+
+  def warn(s)
+    puts s
+  end
+end
+
+if $0 == __FILE__
+  Bitcask::Tool.new.run
+end

data/lib/bitcask/version.rb

@@ -1,3 +1,3 @@
 class Bitcask
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - Kyle Kingsbury
@@ -20,13 +20,14 @@ dependencies: []
 
 description: 
 email: aphyr@aphyr.com
-executables: []
-
+executables: 
+- bitcask
 extensions: []
 
 extra_rdoc_files: []
 
 files: 
+- bin/bitcask
 - lib/bitcask.rb
 - lib/bitcask/data_file.rb
 - lib/bitcask/version.rb