hbase-ruby 1.1.0

data/History.txt

@@ -0,0 +1,11 @@
+v1.1.0 Nov 7, 2009
+* Bumped to version 1.1.0 to avoid confusion with an existing hbase-ruby on gemcutter that was at version 1.0
+* Reimplemented entirely with HBase Stargate (old REST API was obsoleted by this)
+* Since Stargate can return JSON, parsing is now done with JSON instead of XML
+* API change to open_scanner. Now expects open_scanner(table_name, options) with available options listed under HBase::Model::Scanner.AVAILABLE_OPTS 
+
+v0.4 Oct 12, 2008
+* support 'scanner'
+
+v0.1 Jul 29, 2008
+* initial version 

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2007 Dingding Ye
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.textile

@@ -0,0 +1,81 @@
+hbase-ruby is a pure ruby client for HBase (http://hadoop.apache.org/hbase) that works with the HBase Stargate interface.
+Stargate is the RESTful web service front end for HBase that can serve up a number of formats including XML, JSON, and protobufs.
+
+This was originally created by Dingding Ye <yedingding@gmail.com> at http://github.com/sishen/hbase-ruby
+
+
+h2. Versions
+
+* **"hbase-0.20.1 version" - rewritten for Stargate**: http://github.com/greglu/hbase-ruby/tree/master
+* **"hbase-0.20dev version"**: http://github.com/sishen/hbase-ruby/tree/master
+* **"hbase-0.19 version"**: http://github.com/sishen/hbase-ruby/tree/hbase-0.19
+
+
+h2. Installation
+
+<pre>
+<code>$ gem install hbase-ruby -s http://gemcutter.org</code>
+</pre>
+
+To work with this gem in your Rails application, add this to the environment.rb file:
+<pre><code>config.gem 'hbase-ruby', :lib => "hbase", :source => "http://gemcutter.org"</code></pre>
+
+To build the gem yourself:
+<pre><code>$ rake gem</code></pre>
+
+
+h2. Getting Started
+
+ # Download and unpack the most recent release of HBase from http://hadoop.apache.org/hbase/releases.html#Download
+ # Edit <hbase-dir>/conf/hbase-env.sh and uncomment/modify the following line to correspond to your Java home path:
+export JAVA_HOME=/usr/lib/jvm/java-6-sun
+ # Copy <hbase-dir>/contrib/stargate/hbase-<version>-stargate.jar into <hbase-dir>/lib
+ # Copy all the files in the <hbase-dir>/contrib/stargate/lib folder into <hbase-dir>/lib
+ # Start up HBase:
+$ <hbase-dir>/bin/start-hbase.sh
+ # Start up Stargate (append "-p 1234" at the end if you want to change the port):
+$ <hbase-dir>/bin/hbase org.apache.hadoop.hbase.stargate.Main
+
+
+h2. Usage
+
+Here are some examples:
+
+<pre><code>
+require 'hbase'
+
+client = HBase::Client.new("http://localhost:8080") # this url is the default for stargate.
+
+# Table Operation
+tables = client.list_tables                              # list available tables
+table = client.create_table('users', 'habbit')           # create a table whose column_family is habbit
+table = client.show_table('users')                       # show the meta info of table users 
+client.delete_table('users')                             # delete 'users' table
+
+# Row Operation
+row = client.show_row('users', 'sishen')                 # show the data of row 'sishen' in table 'users'
+row2 = client.create_row('users', 'sishen', Time.now.to_i, {:name => 'habbit:football', :value => 'i like football'}) # create the row 'sishen' with the data in the table 'users'
+client.delete_row('users', 'sishen', nil, 'habbit:football')  # delete the row 'sishen' of table 'users' with the optional column 'habbit:football'
+
+# Scanner Operation
+scanner = client.open_scanner('users', {:start_row => "row2", :batch => 5, :columns => ["habbit:"]}) # See more options from HBase::Model::Scanner.AVAILABLE_OPTS
+rows = client.get_rows(scanner)
+client.close_scanner(scanner)
+</code></pre>
+
+
+h2. Testing
+
+Run the specs with the following rake task:
+
+$ rake spec
+
+or pass it the URL to the HBase Stargate server as an argument:
+
+$ rake spec HBASE_URL=http://localhost:8080
+
+
+h2. Copyright
+
+Copyright (c) 2008 Dingding Ye <yedingding@gmail.com>
+Distributed under MIT License

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rubygems'
+
+begin
+  require 'jeweler'
+
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "hbase-ruby"
+    gemspec.authors = ['Ye Dingding', 'Greg Lu']
+    gemspec.email = 'greg.lu@gmail.com'
+    gemspec.homepage = "http://github.com/greglu/hbase-ruby"
+    gemspec.summary = "A pure ruby client for HBase using the Stargate interface."
+    gemspec.description = "A pure ruby client used to interact with HBase through its Stargate interface which serves up XML, JSON, protobuf, and more."
+    gemspec.files = FileList["{lib,spec,tasks}/**/*","Rakefile","VERSION","History.txt","MIT-LICENSE","README.textile"].to_a
+    gemspec.extra_rdoc_files = FileList["MIT-LICENSE","README.textile"].to_a
+
+    gemspec.add_development_dependency "rspec"
+    gemspec.add_dependency "json"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install jeweler"
+end
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/VERSION

@@ -0,0 +1 @@
+1.1.0

data/lib/hbase.rb

@@ -0,0 +1,44 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'hbase/client'
+require 'hbase/exception'
+require 'hbase/model'
+require 'hbase/request'
+require 'hbase/response'
+
+module HBase; end
+
+class Object
+  def to_proc
+    proc { |obj, *args| obj.send(self, *args) }
+  end
+
+  def blank?
+    respond_to?(:empty?) ? empty? : !self
+  end
+end
+
+class NilClass #:nodoc:
+  def blank?
+    true
+  end
+end
+
+class Array #:nodoc:
+  alias_method :blank?, :empty?
+end
+
+class Hash #:nodoc:
+  alias_method :blank?, :empty?
+end
+
+class String #:nodoc:
+  def blank?
+    self !~ /\S/
+  end
+
+  def to_b
+    self == "true" ? true : false
+  end
+end

data/lib/hbase/client.rb

@@ -0,0 +1,80 @@
+require 'net/http'
+require 'hbase/operation/meta_operation'
+require 'hbase/operation/table_operation'
+require 'hbase/operation/row_operation'
+require 'hbase/operation/scanner_operation'
+
+module HBase
+  class Client
+    include Operation::MetaOperation
+    include Operation::TableOperation
+    include Operation::RowOperation
+    include Operation::ScannerOperation
+
+    attr_reader :url, :connection
+
+    def initialize(url = "http://localhost:8080", opts = {})
+      @url = URI.parse(url)
+      unless @url.kind_of? URI::HTTP
+        raise "invalid http url: #{url}"
+      end
+
+      # Not actually opening the connection yet, just setting up the persistent connection.
+      @connection = Net::HTTP.new(@url.host, @url.port)
+      @connection.read_timeout = opts[:timeout] if opts[:timeout]
+    end
+
+    def get(path)
+      safe_request { @connection.get(@url.path + path, {"Accept" => "application/json"}) }
+    end
+
+    def post(path, data = nil)
+      safe_request { @connection.post(@url.path + path, data, {'Content-Type' => 'text/xml'}) }
+    end
+
+    # Needed for scanner functionality
+    def post_response(path, data = nil)
+      safe_response { @connection.post(@url.path + path, data, {'Content-Type' => 'text/xml'}) }
+    end
+
+    def delete(path)
+      safe_request { @connection.delete(@url.path + path) }
+    end
+
+    # Needed for scanner functionality
+    def delete_response(path)
+      safe_response { @connection.delete(@url.path + path) }
+    end
+
+    def put(path, data = nil)
+      safe_request { @connection.put(@url.path + path, data, {'Content-Type' => 'text/xml'}) }
+    end
+
+    private
+
+      # Part of safe_request was broken up into safe_response because when working with scanners
+      # in Stargate, you need to have access to the response itself, and not just the body.
+      def safe_response(&block)
+        begin
+          yield
+        rescue Errno::ECONNREFUSED
+          raise ConnectionNotEstablishedError, "can't connect to #{@url}"
+        rescue Timeout::Error => e
+          puts e.backtrace.join("\n")
+          raise ConnectionTimeoutError, "execution expired. Maybe query disabled tables"
+        end
+      end
+
+      def safe_request(&block)
+        response = safe_response{ yield block }
+
+        case response
+        when Net::HTTPSuccess
+          response.body
+        else
+          response.error!
+        end
+      end
+
+  end
+end

data/lib/hbase/exception.rb

@@ -0,0 +1,19 @@
+class HBase::Exception < StandardError; end
+
+class HBase::ConnectionNotEstablishedError < HBase::Exception; end
+
+class HBase::ConnectionTimeoutError < HBase::Exception; end
+
+class HBase::TableNotFoundError < HBase::Exception; end
+
+class HBase::TableExistsError < HBase::Exception; end
+
+class HBase::TableFailCreateError < HBase::Exception; end
+
+class HBase::TableNotDisabledError < HBase::Exception; end
+
+class HBase::TableFailDisableError < HBase::Exception; end
+
+class HBase::TableFailEnableError < HBase::Exception; end
+
+class HBase::RowNotFoundError < HBase::Exception; end

data/lib/hbase/model.rb

@@ -0,0 +1,19 @@
+module HBase
+  module Model
+    class Record
+      def initialize (params)
+        params.each do |key, value|
+          name = key.to_s
+          instance_variable_set("@#{name}", value) if respond_to?(name)
+        end
+      end
+    end
+  end
+end
+
+require 'hbase/model/column'
+require 'hbase/model/column_descriptor'
+require 'hbase/model/region_descriptor'
+require 'hbase/model/row'
+require 'hbase/model/table_descriptor'
+require 'hbase/model/scanner'

data/lib/hbase/model/column.rb

@@ -0,0 +1,9 @@
+module HBase
+  module Model
+    class Column < Record
+      attr_accessor :name
+      attr_accessor :value
+      attr_accessor :timestamp
+    end
+  end
+end

data/lib/hbase/model/column_descriptor.rb

@@ -0,0 +1,32 @@
+module HBase
+  module Model
+    module CompressionType
+      NONE = "NONE"
+      RECORD = "RECORD"
+      BLOCK = "BLOCK"
+
+      CTYPES = [NONE, RECORD, BLOCK]
+
+      def to_compression_type(type_string)
+        CTYPES.include?(type_string) ? type_string : NONE
+      end
+
+      module_function :to_compression_type
+    end
+
+    class ColumnDescriptor < Record
+      AVAILABLE_OPTS = {  :name => "name", :max_versions => "VERSIONS", :versions => "VERSIONS",
+                          :compression => "COMPRESSION", :in_memory => "IN_MEMORY",
+                          :block_cache => "BLOCKCACHE", :blockcache => "BLOCKCACHE",
+                          :blocksize => "BLOCKSIZE", :length => "LENGTH", :ttl => "TTL",
+                          :bloomfilter => "BLOOMFILTER"}
+      attr_accessor :name
+      attr_accessor :compression
+      attr_accessor :bloomfilter
+      attr_accessor :maximum_cell_size
+      attr_accessor :max_versions
+
+      attr_accessor :versions
+    end
+  end
+end

data/lib/hbase/model/region_descriptor.rb

@@ -0,0 +1,9 @@
+module HBase
+  module Model
+    class Region < Record
+    end
+
+    class RegionDescriptor < Record
+    end
+  end
+end

data/lib/hbase/model/row.rb

@@ -0,0 +1,11 @@
+module HBase
+  module Model
+    class Row < Record
+      attr_accessor :table_name
+      attr_accessor :name
+      attr_accessor :timestamp
+      attr_accessor :total_count
+      attr_accessor :columns
+    end
+  end
+end

data/lib/hbase/model/scanner.rb

@@ -0,0 +1,13 @@
+module HBase
+  module Model
+    class Scanner < Record
+      AVAILABLE_OPTS = {  :start_row => "startRow", :end_row => "endRow",
+                          :start_time => "startTime", :end_time => "endTime",
+                          :batch => "batch", :limit => "batch" }
+
+      attr_accessor :table_name
+      attr_accessor :scanner_id
+      attr_accessor :scanner_url
+    end
+  end
+end

data/lib/hbase/model/table_descriptor.rb

@@ -0,0 +1,8 @@
+module HBase
+  module Model
+    class TableDescriptor < Record
+      attr_accessor :name
+      attr_accessor :column_families
+    end
+  end
+end

data/lib/hbase/operation/meta_operation.rb

@@ -0,0 +1,10 @@
+module HBase
+  module Operation
+    module MetaOperation
+      def list_tables
+        request = Request::MetaRequest.new
+        tables = Response::MetaResponse.new(get(request.list_tables), :list_tables).parse
+      end
+    end
+  end
+end

data/lib/hbase/operation/row_operation.rb

@@ -0,0 +1,83 @@
+module HBase
+  module Operation
+    module RowOperation
+      Converter = {
+        '&' => '&',
+        '<' => '&lt;',
+        '>' => '&gt;',
+        "'" => '&apos;',
+        '"' => '&quot;'
+      }
+
+      def row_timestamps(table_name, name)
+        raise NotImplementedError, "Currently not supported in native hbase client"
+      end
+
+      def show_row(table_name, name, timestamp = nil, columns = nil, options = { })
+        begin
+          options[:version] ||= 1
+          request = Request::RowRequest.new(table_name, name, timestamp)
+          row = Response::RowResponse.new(get(request.show(columns, options))).parse.first
+          row.table_name = table_name
+          row.timestamp = timestamp
+          row
+        rescue Net::ProtocolError => e
+          if e.to_s.include?("Table")
+            raise TableNotFoundError, "Table '#{table_name}' Not Found"
+          elsif e.to_s.include?("Row")
+            raise RowNotFoundError, "Row '#{name}' Not Found"
+          end
+        end
+      end
+
+      def create_row(table_name, name, timestamp = nil, columns = nil)
+        begin
+          request = Request::RowRequest.new(table_name, name, timestamp)
+          data = []
+          if columns
+            if columns.instance_of? Array
+              data = columns
+            elsif columns.instance_of? Hash
+              data = [columns]
+            else
+              raise StandardError, "Only Array or Hash data accepted"
+            end
+          end
+
+          xml_data = "<?xml version='1.0' encoding='UTF-8' standalone='yes'?><CellSet>"
+          xml_data << "<Row key='#{[name].pack('m') rescue ''}'>"
+          data.each do |d|
+            escape_name = d[:name].gsub(/[&<>'"]/) { |match| Converter[match] }
+            xml_data << "<Cell "
+            xml_data << "timestamp='#{timestamp}'" if timestamp
+            xml_data << "column='#{[escape_name].pack('m') rescue ''}'>"
+            xml_data << "#{[d[:value]].pack("m") rescue ''}"
+            xml_data << "</Cell>"
+          end
+          xml_data << "</Row></CellSet>"
+
+          post(request.create(data.map{|col| col[:name]}), xml_data)
+        rescue Net::ProtocolError => e
+          if e.to_s.include?("Table")
+            raise TableNotFoundError, "Table '#{table_name}' Not Found"
+          elsif e.to_s.include?("Row")
+            raise RowNotFoundError, "Row '#{name}' Not Found"
+          end
+        end
+      end
+
+      def delete_row(table_name, name, timestamp = nil, columns = nil)
+        begin
+          request = Request::RowRequest.new(table_name, name, timestamp)
+          Response::RowResponse.new(delete(request.delete(columns)))
+        rescue Net::ProtocolError => e
+          if e.to_s.include?("Table")
+            raise TableNotFoundError, "Table '#{table_name}' Not Found"
+          elsif e.to_s.include?("Row")
+            raise RowNotFoundError, "Row '#{name}' Not Found"
+          end
+        end
+      end
+    end
+  end
+end