ok_hbase 0.0.5

data/lib/thrift/hbase/hbase_constants.rb

@@ -0,0 +1,14 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+module Apache
+  module Hadoop
+    module Hbase
+      module Thrift
+      end
+    end
+  end
+end

data/lib/thrift/hbase/hbase_types.rb

@@ -0,0 +1,252 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+module Apache
+  module Hadoop
+    module Hbase
+      module Thrift
+        # TCell - Used to transport a cell value (byte[]) and the timestamp it was
+# stored with together as a result for get and getRow methods. This promotes
+# the timestamp of a cell to a first-class value, making it easy to take
+# note of temporal data. Cell is used all the way from HStore up to HTable.
+        class TCell
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          VALUE = 1
+          TIMESTAMP = 2
+
+          FIELDS = {
+            VALUE => {:type => ::Thrift::Types::STRING, :name => 'value', :binary => true},
+            TIMESTAMP => {:type => ::Thrift::Types::I64, :name => 'timestamp'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An HColumnDescriptor contains information about a column family
+# such as the number of versions, compression settings, etc. It is
+# used as input when creating a table or adding a column.
+        class ColumnDescriptor
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          NAME = 1
+          MAXVERSIONS = 2
+          COMPRESSION = 3
+          INMEMORY = 4
+          BLOOMFILTERTYPE = 5
+          BLOOMFILTERVECTORSIZE = 6
+          BLOOMFILTERNBHASHES = 7
+          BLOCKCACHEENABLED = 8
+          TIMETOLIVE = 9
+
+          FIELDS = {
+            NAME => {:type => ::Thrift::Types::STRING, :name => 'name', :binary => true},
+            MAXVERSIONS => {:type => ::Thrift::Types::I32, :name => 'maxVersions', :default => 3},
+            COMPRESSION => {:type => ::Thrift::Types::STRING, :name => 'compression', :default => %q"NONE"},
+            INMEMORY => {:type => ::Thrift::Types::BOOL, :name => 'inMemory', :default => false},
+            BLOOMFILTERTYPE => {:type => ::Thrift::Types::STRING, :name => 'bloomFilterType', :default => %q"NONE"},
+            BLOOMFILTERVECTORSIZE => {:type => ::Thrift::Types::I32, :name => 'bloomFilterVectorSize', :default => 0},
+            BLOOMFILTERNBHASHES => {:type => ::Thrift::Types::I32, :name => 'bloomFilterNbHashes', :default => 0},
+            BLOCKCACHEENABLED => {:type => ::Thrift::Types::BOOL, :name => 'blockCacheEnabled', :default => false},
+            TIMETOLIVE => {:type => ::Thrift::Types::I32, :name => 'timeToLive', :default => -1}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A TRegionInfo contains information about an HTable region.
+        class TRegionInfo
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          STARTKEY = 1
+          ENDKEY = 2
+          ID = 3
+          NAME = 4
+          VERSION = 5
+
+          FIELDS = {
+            STARTKEY => {:type => ::Thrift::Types::STRING, :name => 'startKey', :binary => true},
+            ENDKEY => {:type => ::Thrift::Types::STRING, :name => 'endKey', :binary => true},
+            ID => {:type => ::Thrift::Types::I64, :name => 'id'},
+            NAME => {:type => ::Thrift::Types::STRING, :name => 'name', :binary => true},
+            VERSION => {:type => ::Thrift::Types::BYTE, :name => 'version'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A Mutation object is used to either update or delete a column-value.
+        class Mutation
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ISDELETE = 1
+          COLUMN = 2
+          VALUE = 3
+
+          FIELDS = {
+            ISDELETE => {:type => ::Thrift::Types::BOOL, :name => 'isDelete', :default => false},
+            COLUMN => {:type => ::Thrift::Types::STRING, :name => 'column', :binary => true},
+            VALUE => {:type => ::Thrift::Types::STRING, :name => 'value', :binary => true}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A BatchMutation object is used to apply a number of Mutations to a single row.
+        class BatchMutation
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ROW = 1
+          MUTATIONS = 2
+
+          FIELDS = {
+            ROW => {:type => ::Thrift::Types::STRING, :name => 'row', :binary => true},
+            MUTATIONS => {:type => ::Thrift::Types::LIST, :name => 'mutations', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Apache::Hadoop::Hbase::Thrift::Mutation}}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # Holds row name and then a map of columns to cells.
+        class TRowResult
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          ROW = 1
+          COLUMNS = 2
+
+          FIELDS = {
+            ROW => {:type => ::Thrift::Types::STRING, :name => 'row', :binary => true},
+            COLUMNS => {:type => ::Thrift::Types::MAP, :name => 'columns', :key => {:type => ::Thrift::Types::STRING, :binary => true}, :value => {:type => ::Thrift::Types::STRUCT, :class => ::Apache::Hadoop::Hbase::Thrift::TCell}}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # A Scan object is used to specify scanner parameters when opening a scanner.
+        class TScan
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          STARTROW = 1
+          STOPROW = 2
+          TIMESTAMP = 3
+          COLUMNS = 4
+          CACHING = 5
+          FILTERSTRING = 6
+
+          FIELDS = {
+            STARTROW => {:type => ::Thrift::Types::STRING, :name => 'startRow', :binary => true, :optional => true},
+            STOPROW => {:type => ::Thrift::Types::STRING, :name => 'stopRow', :binary => true, :optional => true},
+            TIMESTAMP => {:type => ::Thrift::Types::I64, :name => 'timestamp', :optional => true},
+            COLUMNS => {:type => ::Thrift::Types::LIST, :name => 'columns', :element => {:type => ::Thrift::Types::STRING, :binary => true}, :optional => true},
+            CACHING => {:type => ::Thrift::Types::I32, :name => 'caching', :optional => true},
+            FILTERSTRING => {:type => ::Thrift::Types::STRING, :name => 'filterString', :binary => true, :optional => true}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An IOError exception signals that an error occurred communicating
+# to the Hbase master or an Hbase region server.  Also used to return
+# more general Hbase error conditions.
+        class IOError < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An IllegalArgument exception indicates an illegal or invalid
+# argument was passed into a procedure.
+        class IllegalArgument < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+        # An AlreadyExists exceptions signals that a table with the specified
+# name already exists
+        class AlreadyExists < ::Thrift::Exception
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          def initialize(message=nil)
+            super()
+            self.message = message
+          end
+
+          MESSAGE = 1
+
+          FIELDS = {
+            MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
+      end
+    end
+  end
+end

data/ok-hbase.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ok_hbase/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "ok_hbase"
+  gem.version       = OkHbase::VERSION
+  gem.authors       = ["Nathan Keyes"]
+  gem.email         = ["keyes@okcupidlabs.com"]
+  gem.description   = %q{Lightweight Ruby Hbase Client}
+  gem.summary       = %q{Lightweight Ruby Hbase Client}
+  gem.homepage      = "http://okcwest.github.io/ok_hbase/"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+
+  gem.add_dependency 'thrift', '0.9.0'
+  gem.add_dependency 'activesupport'
+end

data/spec/ok_hbase/connection_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+module OkHbase
+  describe Connection do
+    describe ".create_table" do
+      let(:conn) { Connection.new auto_connect: true, timeout: 60 }
+
+      it "should create tables with the right column families" do
+        name = "ok_hbase_test_table"
+        column_families = {
+            'a' => {
+                'max_versions' => 5,
+                'compression' => 'GZ',
+                'in_memory' => true,
+                'bloom_filter_type' => 'ROW',
+                'block_cache_enabled' => true,
+
+                #TODO find out if these aren't being set properly or just aren't reported properly. 0 is the default
+                'bloom_filter_vector_size' => 0,
+                'bloom_filter_nb_hashes' => 0,
+
+                #TODO find out why this doesn't get reported properly. -1 is the default
+                'time_to_live' => -1
+            },
+            'b' => {
+                'max_versions' => 15,
+                'compression' => 'NONE',
+                'in_memory' => true,
+                'bloom_filter_type' => 'ROWCOL',
+                'block_cache_enabled' => true,
+
+                'bloom_filter_vector_size' => 0,
+                'bloom_filter_nb_hashes' => 0,
+
+                'time_to_live' => -1
+            }
+        }
+
+        expected_families = Hash[column_families.map { |cf, data| [cf, { 'name' => "#{cf}:" }.merge(data)] }]
+
+        #sanity check
+        conn.tables.should_not include(name)
+
+        conn.create_table(name, column_families)
+
+        conn.tables.should include(name)
+
+        table = conn.table(name)
+
+        table.families.should == expected_families
+
+        #cleanup
+        conn.delete_table(name, true)
+      end
+
+
+      it "should create tables with the right name" do
+        name = "ok_hbase_test_table"
+        column_families = {
+            'd' => {}
+        }
+
+        #sanity check
+        conn.tables.should_not include(name)
+
+        conn.create_table(name, column_families)
+
+        conn.tables.should include(name)
+
+        #cleanup
+        conn.delete_table(name, true)
+      end
+    end
+
+    describe ".open" do
+      let(:conn) { Connection.new }
+
+      it "should open a connection" do
+        expect { conn.open }.to change { conn.open? }.to(true)
+      end
+    end
+
+    describe ".close" do
+      let(:conn) { Connection.new auto_connect: true }
+
+      it "should close a connection" do
+        expect { conn.close }.to change { conn.open? }.to(false)
+      end
+    end
+
+    describe ".tables" do
+      let(:conn) { Connection.new auto_connect: true }
+
+      it "should return an array of table names" do
+        conn.tables.should be_an Array
+      end
+    end
+  end
+end

data/spec/ok_hbase/table_spec.rb

@@ -0,0 +1,149 @@
+require 'spec_helper'
+
+module OkHbase
+  describe Table do
+    let(:row_key1) { 'row1' }
+    let(:row_key2) { 'row2' }
+    let(:row_data1) { { 'd:foo' => 'Foo1', 'd:bar' => 'Bar1', 'd:baz' => 'Baz1' } }
+    let(:row_data2) { { 'd:foo' => 'Foo2', 'd:bar' => 'Bar2', 'd:baz' => 'Baz2' } }
+    let!(:timestamp) { (Time.now.to_f * 1000).to_i } # hbase timestamps are in milisecnds
+
+    test_table_name = 'ok_hbase_test_table'
+    conn = Connection.new(auto_connect: true, timeout: 60)
+
+    before(:all) do
+      conn.create_table(test_table_name, d: { max_versions: 3 })
+    end
+
+    after(:all) do
+      conn.delete_table(test_table_name, true)
+    end
+
+    subject { Table.new(test_table_name, conn) }
+
+    describe '#_scanner' do
+      let(:scanner_opts) { {
+          start_row: 'start',
+          stop_row: 'stop',
+          timestamp: Time.now,
+          columns: ['d:foo', 'd:bar', 'i:baz'],
+          caching: 1000,
+          filter_string: 'filter',
+      } }
+
+      it 'should set all the options' do
+        scanner = subject._scanner(scanner_opts)
+
+        scanner.startRow.should == scanner_opts[:start_row]
+        scanner.stopRow.should == scanner_opts[:stop_row]
+        scanner.timestamp.should == scanner_opts[:timestamp]
+        scanner.columns.should == scanner_opts[:columns]
+        scanner.caching.should == scanner_opts[:caching]
+        scanner.filterString.should == scanner_opts[:filter_string]
+      end
+    end
+
+    describe '.scan' do
+
+      it 'should convert a row prefix to a start and stop rows' do
+        subject.put('scan_row1', row_data1)
+        subject.put('scan_row2', row_data1)
+
+        opts = { row_prefix: 'scan' }
+        subject.should_receive(:_scanner).with(hash_including(start_row: 'scan', stop_row: 'scao')).and_return {
+          Apache::Hadoop::Hbase::Thrift::TScan.new(
+              startRow: 'scan',
+              stopRow: 'scao',
+              caching: 1000,
+          )
+        }
+
+        subject.scan(opts) do |row_key, cols|
+          cols.should == row_data1
+        end
+      end
+    end
+
+    describe '.regions' do
+      it 'should list all region having rows for the table' do
+        regions = subject.regions
+        regions.should be_an Array
+        regions.size.should be >= 1
+
+        regions.each do |region|
+          region.keys.should include('start_key', 'end_key', 'id', 'name', 'version')
+        end
+
+      end
+    end
+
+    describe 'CRUD' do
+
+      describe '.put' do
+
+        it 'should write to the row' do
+          subject.put(row_key1, row_data1)
+
+          subject.row(row_key1).should == row_data1
+        end
+
+        it 'should write to the row with a timestamp' do
+
+          subject.put(row_key1, row_data1, timestamp)
+
+          subject.row(row_key1, nil, nil, true).should == Hash[row_data1.map { |k, v| [k, [v, timestamp]] }]
+        end
+      end
+
+      describe '.cells' do
+        it 'should retrieve values' do
+
+          subject.put(row_key2, row_data2.merge('d:foo' => 'OldFoo1'), timestamp-10)
+          subject.put(row_key2, row_data2.merge('d:foo' => 'OldFoo2'), timestamp-5)
+          subject.put(row_key2, row_data2, timestamp)
+
+          subject.cells(row_key2, 'd:foo').should == ['Foo2', 'OldFoo2', 'OldFoo1']
+
+          subject.cells(row_key2, 'd:foo', nil, timestamp-1).should == ['OldFoo2', 'OldFoo1']
+          subject.cells(row_key2, 'd:foo', nil, timestamp-5).should == ['OldFoo1']
+
+          subject.cells(row_key2, 'd:foo', nil, nil, true).should == [['Foo2', timestamp], ['OldFoo2', timestamp-5], ['OldFoo1', timestamp-10]]
+          subject.cells(row_key2, 'd:foo', 2, nil, true).should == [['Foo2', timestamp], ['OldFoo2', timestamp-5]]
+          subject.cells(row_key2, 'd:foo', 1, nil, true).should == [['Foo2', timestamp]]
+
+          subject.cells(row_key2, 'd:foo', nil, nil, false).should == ['Foo2', 'OldFoo2', 'OldFoo1']
+          subject.cells(row_key2, 'd:foo', 2, nil, false).should == ['Foo2', 'OldFoo2']
+          subject.cells(row_key2, 'd:foo', 1, nil, false).should == ['Foo2']
+
+        end
+      end
+
+      describe ".rows" do
+        it 'should retrieve the specified rows' do
+
+          subject.put(row_key1, row_data1, timestamp-10)
+          subject.put(row_key2, row_data2.merge('d:foo' => 'OldFoo1'), timestamp-10)
+          subject.put(row_key2, row_data2.merge('d:foo' => 'OldFoo2'), timestamp-5)
+          subject.put(row_key2, row_data2, timestamp)
+
+          subject.rows([row_key1, row_key2]).should == [row_data1, row_data2]
+          subject.rows([row_key1, row_key2], nil, timestamp-9).should == [row_data1, row_data2.merge('d:foo' => 'OldFoo1')]
+
+          subject.rows([row_key1, row_key2], row_data1.keys - ['d:foo']).should == [row_data1.except('d:foo'), row_data2.except('d:foo')]
+          subject.rows([row_key1, row_key2], row_data1.keys - ['d:foo'], timestamp-5).should == [row_data1.except('d:foo'), row_data2.except('d:foo')]
+
+        end
+      end
+
+      describe ".delete" do
+        it 'should retrieve the specified rows' do
+          subject.put(row_key1, row_data1)
+
+          subject.delete(row_key1, ['d:bar', 'd:baz'])
+
+          subject.row(row_key1).should == row_data1.except('d:bar', 'd:baz')
+        end
+      end
+    end
+  end
+end

data/spec/ok_hbase_spec.rb

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+# require 'spec_helper'
+
+describe OkHbase do
+
+  describe "#increment_string" do
+    let(:normal_string) { "foo baq" }
+    let(:incremented_normal_string) { "foo bar" }
+
+    let(:utf8_string) { "Vulgar fraction one half: ¼".force_encoding(Encoding::UTF_8) }
+    let(:incremented_utf8_string) { "Vulgar fraction one half: ½".force_encoding(Encoding::UTF_8) }
+    let(:max_byte_string) { [255, 255, 255].pack('c*') }
+
+    it "should increment the last byte byte < 255" do
+      OkHbase::increment_string(normal_string).should == incremented_normal_string
+      OkHbase::increment_string(utf8_string).should == incremented_utf8_string
+    end
+
+    it "should return nil when all bytes are 255" do
+      OkHbase::increment_string(max_byte_string).should be_nil
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'support'))
+
+# needs to be at the top
+require 'simplecov'
+SimpleCov.start do
+  add_filter "/thrift/"
+end
+
+Bundler.require :default, :test
+
+
+require 'rubygems'
+require 'bundler'
+require 'awesome_print'
+
+require 'ok_hbase'
+
+

data/tasks/bump.rb

@@ -0,0 +1,30 @@
+desc 'shortcut for bump:patch'
+task :bump => 'bump:patch'
+
+namespace :bump do
+  desc 'Bump x.y.Z'
+  task :patch
+  
+  desc 'Bump x.Y.z'
+  task :minor
+  
+  desc 'Bump X.y.z'
+  task :major
+end
+
+# extracted and modified from https://github.com/grosser/project_template
+rule /^bump:.*/ do |t|
+  sh "git status | grep 'nothing to commit'" # ensure we are not dirty
+  index = ['major', 'minor','patch'].index(t.name.split(':').last)
+  file = 'lib/ok_hbase/version.rb'
+
+  version_file = File.read(file)
+  old_version, *version_parts = version_file.match(/(\d+)\.(\d+)\.(\d+)/).to_a
+  version_parts[index] = version_parts[index].to_i + 1
+  version_parts[2] = 0 if index < 2
+  version_parts[1] = 0 if index < 1
+  new_version = version_parts * '.'
+  File.open(file,'w'){|f| f.write(version_file.sub(old_version, new_version)) }
+
+  sh "bundle && git add #{file} && git commit -m 'bump version to #{new_version}'"
+end