ok_hbase 0.0.5

data/examples/advanced/table_write.rb

@@ -0,0 +1,128 @@
+#!/usr/bin/env ruby
+#
+# table_write.rb - basic table batch and single write operations
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+$stdout.sync = true
+
+require 'awesome_print'
+require 'ok_hbase'
+require 'optparse'
+require 'logger'
+
+$options = {}
+$logger = Logger.new(STDOUT)
+$logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime} #{severity}: #{msg}\n" }
+$logger.level = Logger::FATAL
+
+def usage(error=nil)
+  puts "Error: #{error}\n\n" if error
+  puts $optparse
+  exit 1
+end
+
+def get_connection(table=nil)
+  $logger.debug "Setting up connection for table #{table}"
+  if table.nil?
+    $logger.fatal "Must specify a table"
+    return nil
+  end
+    
+  $logger.debug "Connecting to #{$options[:hostname]}"
+  conn = OkHbase::Connection.new(auto_connect: true, host: $options[:hostname], port: $options[:port],
+      timeout: $options[:timeout])
+  $logger.debug "Get instance for table #{table}"
+  OkHbase::Table.new(table, conn)
+end
+
+def write_test_row(conn, rowkey)
+  # set any column family shit
+  # use a pack method to build the binary sequence
+  puts 'wrote all the things'
+end
+
+def write_batch_row(conn, rowkey)
+  # set any column family shit
+
+  $options[:rowcount].times do |i|
+    # increment and write
+    puts 'wrote something things'
+  end
+end
+
+
+def get_rowkey()
+  # get a incrementor value if needed
+  # set attributes for a row key
+  # setup any time data
+  # use a pack method to build the binary sequence
+  # return binary sequence or decimal sequence to ok-hbase
+  puts "rowkey"
+end
+
+def main()
+  $optparse = OptionParser.new do|opts|
+    opts.banner = "Usage: #{__FILE__} [options]"
+
+    $options[:verbose] = false
+    $options[:port] = 9090
+    $options[:timeout] = 600
+    $options[:rowcount] = 1
+
+    opts.on('-h', '--help', 'Display this help') do
+      usage
+    end
+
+    opts.on('-v', '--verbose', 'Output json result') do
+      $options[:verbose] = true
+      $logger.level = Logger::DEBUG
+    end
+   
+    opts.on('-n', '--host HOSTNAME', 'hostname of RegionServer or  master') do |hostname|
+      $options[:hostname] = hostname
+    end
+
+    opts.on('-t', '--table TABLE', 'hbase table name') do |table|
+      $options[:table] = table
+    end
+
+    opts.on('-p', '--port PORT', "port number of thrift server, defaults to #{$options[:port]}") do |port|
+      $options[:port] = port.to_i
+    end
+
+    opts.on('--timeout TIMEOUT', "connect timeout, defaults to #{$options[:timeout]}") do |timeout|
+      $options[:timeout] = timeout.to_i
+    end
+
+    opts.on('-a', '--array ARRAY', Array, "array values for pack for rowkey, comma separated, no whitespace in the format of \"11111111,1,1,1,1\"") do |ar|
+      $options[:filter_array] = ar.map(&:to_i)
+    end
+
+    opts.on('-p', '--pack PACK', "template string to build binary sequence from literal passed to -a") do |pack|
+      $options[:filter_pack] = pack.to_s
+    end
+
+    opts.on('-w', '--write ROWS', "how many times to write with a row key defaults to #{$options[:rowcount]}") do |row|
+      $options[:rowcount] = row.to_i
+    end
+
+  end
+
+  usage "You didn't specify any options" if not ARGV[0]
+
+  $optparse.parse!
+
+  usage "You didn't specify a hostname" if not $options[:hostname]
+  usage "You didn't specify a table" if not $options[:table]
+  usage "You didn't specify an array literal" if not $options[:filter_array]
+  usage "You didn't specify a binary sequence template" if not $options[:filter_pack]
+
+  start_time = Time.now
+  c = get_connection($options[:table])
+  row_key = get_rowkey()
+  write_batch_row(c, row_key)
+  total_time = Time.now - start_time
+  puts "Wrote #{$options[:rowcount]} row(s) in #{total_time} second(s)"
+end
+
+main() if __FILE__ == $0

data/examples/table_scan.rb

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+#
+# perf_read.rb - basic read perf test
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+$stdout.sync = true
+
+require 'awesome_print'
+require 'ok_hbase'
+require 'optparse'
+require 'logger'
+
+$options = {}
+$logger = Logger.new(STDOUT)
+$logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime} #{severity}: #{msg}\n" }
+$logger.level = Logger::DEBUG
+
+def usage(error=nil)
+  puts "Error: #{error}\n\n" if error
+  puts $optparse
+  exit 1
+end
+
+def get_connection
+  $logger.debug 'Setting up connection'
+
+
+  $logger.debug "Connecting to #{$options[:host]}"
+  OkHbase::Connection.new(
+      auto_connect: true,
+      host: $options[:host],
+      port: $options[:port],
+      timeout: $options[:timeout]
+  )
+end
+
+def get_table(table, conn)
+  if table.nil?
+    $logger.fatal 'Must specify a table'
+    return nil
+  end
+  $logger.debug "Get instance for table #{table}"
+  OkHbase::Table.new(table, conn)
+end
+
+
+def main()
+  $optparse = OptionParser.new do |opts|
+    opts.banner = "Usage: #{__FILE__} [options]"
+
+    $options[:host] = 'localhost'
+    $options[:port] = 9090
+    $options[:timeout] = 10
+
+    opts.on('-h', '--help', 'Display this help') do
+      usage
+    end
+
+    opts.on('-H', '--host HOST', "host or ip address where thrift server is running, defaults to #{$options[:host]}") do |host|
+      $options[:host] = host
+    end
+
+    opts.on('-t', '--table TABLE', 'hbase table name') do |table|
+      $options[:table] = table
+    end
+
+    opts.on('-p', '--port PORT', "port number of thrift server, defaults to #{$options[:port]}") do |port|
+      $options[:port] = port.to_i
+    end
+
+    opts.on('--timeout TIMEOUT', "connect timeout, defaults to #{$options[:timeout]}") do |timeout|
+      $options[:timeout] = timeout.to_i
+    end
+
+    opts.on('-P', '--prefix ROW_PREFIX', "row prefix to use in scan") do |prefix|
+      $options[:prefix] = prefix
+    end
+
+
+  end
+
+  usage "You didn't specify any options" if not ARGV[0]
+
+  $optparse.parse!
+
+  usage "You didn't specify a table" if not $options[:table]
+  usage "You didn't specify a prefix" if not $options[:prefix]
+
+  connection = get_connection()
+  table = get_table($options[:table], connection)
+
+  table.scan(row_prefix: $options[:prefix]) do |row_key, columns|
+    ap row_key => columns
+  end
+end
+
+main() if __FILE__ == $0

data/examples/table_write.rb

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+#
+# perf_read.rb - basic read perf test
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+$stdout.sync = true
+
+require 'awesome_print'
+require 'ok_hbase'
+require 'optparse'
+require 'logger'
+
+$options = {}
+$logger = Logger.new(STDOUT)
+$logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime} #{severity}: #{msg}\n" }
+$logger.level = Logger::DEBUG
+
+def usage(error=nil)
+  puts "Error: #{error}\n\n" if error
+  puts $optparse
+  exit 1
+end
+
+def get_connection
+  $logger.debug 'Setting up connection'
+
+
+  $logger.debug "Connecting to #{$options[:host]}"
+  OkHbase::Connection.new(
+      auto_connect: true,
+      host: $options[:host],
+      port: $options[:port],
+      timeout: $options[:timeout]
+  )
+end
+
+def create_table(table, conn)
+  if table.nil?
+    $logger.fatal 'Must specify a table'
+    return nil
+  end
+  $logger.debug "Get instance for table #{table}"
+  if conn.tables.include? table
+    OkHbase::Table.new(table, conn)
+  else
+    conn.create_table(table, d: {})
+  end
+end
+
+
+def main()
+  $optparse = OptionParser.new do |opts|
+    opts.banner = "Usage: #{__FILE__} [options]"
+
+    $options[:host] = 'localhost'
+    $options[:port] = 9090
+    $options[:timeout] = 10
+
+    opts.on('-h', '--help', 'Display this help') do
+      usage
+    end
+
+    opts.on('-H', '--host HOST', "host or ip address where thrift server is running, defaults to #{$options[:host]}") do |host|
+      $options[:host] = host
+    end
+
+    opts.on('-t', '--table TABLE', 'hbase table name') do |table|
+      $options[:table] = table
+    end
+
+    opts.on('-p', '--port PORT', "port number of thrift server, defaults to #{$options[:port]}") do |port|
+      $options[:port] = port.to_i
+    end
+
+    opts.on('--timeout TIMEOUT', "connect timeout, defaults to #{$options[:timeout]}") do |timeout|
+      $options[:timeout] = timeout.to_i
+    end
+
+  end
+
+  usage "You didn't specify any options" if not ARGV[0]
+
+  $optparse.parse!
+
+  usage "You didn't specify a table" if not $options[:table]
+
+  connection = get_connection()
+  table = create_table($options[:table], connection)
+
+  ('a'..'zzz').each_with_index do |row_key, index|
+    $logger.debug "wrote row: #{row_key}"
+    table.put(row_key, {'d:row_number' => "#{index+1}", 'd:message' => "this is row number #{index+1}"})
+    $logger.debug "wrote row: #{row_key}"
+  end
+end
+
+main() if __FILE__ == $0

data/lib/ok_hbase/active_model.rb

@@ -0,0 +1,35 @@
+require 'ok_hbase/concerns'
+require 'ok_hbase/row'
+
+module OkHbase
+  class ActiveModel < OkHbase::Row
+    include OkHbase::Concerns::Table::ClassMethods
+    include OkHbase::Concerns::CustomRow::ClassMethods
+    include OkHbase::Concerns::Indexable::ClassMethods
+    include OkHbase::Concerns::Table::Instrumentation
+
+    def initialize(raw_data={})
+
+      raw_data = raw_data.with_indifferent_access
+      raw_data = raw_data[:raw_data] if raw_data[:raw_data]
+
+
+      options = {
+          table: self.class,
+          default_column_family: self.class.default_column_family,
+          raw_data: raw_data,
+      }
+      super(options)
+    end
+
+    def self.create(raw_data={})
+      instance = new(raw_data)
+      instance.save!
+      instance
+    end
+
+    def delete(indexes=[])
+      self.class.delete(row_key, nil, nil, indexes)
+    end
+  end
+end

data/lib/ok_hbase/client.rb

@@ -0,0 +1,42 @@
+require 'thrift'
+require 'thrift/transport/socket'
+require 'thrift/protocol/binary_protocol'
+
+require 'thrift/hbase/hbase_constants'
+require 'thrift/hbase/hbase_types'
+require 'thrift/hbase/hbase'
+
+module OkHbase
+  class Client < Apache::Hadoop::Hbase::Thrift::Hbase::Client
+
+    attr_accessor :max_tries
+
+    def initialize(iprot, oprot=nil, max_tries=nil)
+      @max_tries = max_tries || 0
+      super(iprot, oprot)
+    end
+
+    signatures = ['send_message(name, args_class, args = {})', 'receive_message(result_klass)']
+
+    signatures.each do |signature|
+      module_eval <<-RUBY, __FILE__, __LINE__
+        def #{signature}
+          tries = 0
+          begin
+            @iprot.trans.open unless @iprot.trans.open?
+            super
+          rescue => e
+            tries += 1
+            raise e unless tries < max_tries && recoverable?(e)
+            retry
+          end
+        end
+      RUBY
+    end
+
+    def recoverable?(e)
+      e.is_a?(Apache::Hadoop::Hbase::Thrift::IOError) ||
+          e.is_a?(Thrift::TransportException)
+    end
+  end
+end

data/lib/ok_hbase/concerns/custom_row/class_methods.rb

@@ -0,0 +1,13 @@
+module OkHbase
+  module Concerns
+    module CustomRow
+      module ClassMethods
+        extend ActiveSupport::Concern
+
+        module ClassMethods
+          include OkHbase::Concerns::CustomRow
+        end
+      end
+    end
+  end
+end

data/lib/ok_hbase/concerns/custom_row.rb

@@ -0,0 +1,40 @@
+module OkHbase
+  module Concerns
+    module CustomRow
+      extend ActiveSupport::Concern
+
+      def row(row_key, columns = nil, timestamp = nil, include_timestamp = false)
+        self.row_class.new table: self, default_column_family: self.default_column_family, raw_data: super
+      end
+
+      def rows(row_keys, columns = nil, timestamp = nil, include_timestamp = false)
+        super.map.with_index! { |data, i| self.row_class.new table: self, row_key: row_keys[i], default_column_family: self.default_column_family, raw_data: data }
+      end
+
+      def scan(opts={})
+        if block_given?
+          super { |row_key, data| yield self.row_class.new(table: self, row_key: row_key, default_column_family: self.default_column_family, raw_data: data) }
+        else
+          super.map { |row_key, data| self.row_class.new(table: self, row_key: row_key, default_column_family: self.default_column_family, raw_data: data) }
+        end
+
+      end
+
+      def row_class
+        @@_row_class
+      end
+
+      def default_column_family
+        @@default_column_family
+      end
+
+      def use_row_class(klass)
+        @@_row_class = klass
+      end
+
+      def use_default_column_family(column_family)
+        @@default_column_family = column_family
+      end
+    end
+  end
+end

data/lib/ok_hbase/concerns/indexable/class_methods.rb

@@ -0,0 +1,13 @@
+module OkHbase
+  module Concerns
+    module Indexable
+      module ClassMethods
+        extend ActiveSupport::Concern
+
+        module ClassMethods
+          include OkHbase::Concerns::Indexable
+        end
+      end
+    end
+  end
+end

data/lib/ok_hbase/concerns/indexable.rb

@@ -0,0 +1,101 @@
+module OkHbase
+  module Concerns
+    module Indexable
+      extend ActiveSupport::Concern
+
+      def use_index(index_name, opts={})
+        options = opts.with_indifferent_access
+        attributes = options[:attributes]
+        prefix_length = options[:prefix_length]
+        index_id = options[:index_id]
+        pack_pattern = options[:pack_pattern]
+        auto_create = options[:auto_create]
+
+        @@_indexes ||= {}
+        @@_indexes = @@_indexes.with_indifferent_access
+        @@_indexes[index_name] = options
+
+        define_singleton_method :indexes do
+          @@_indexes
+        end
+
+
+        define_singleton_method :encode_for_row_key do |value|
+          # coerce booleans to ints for packing
+          value = 1 if value.to_s.downcase == "true"
+          value = 0 if value.to_s.downcase == "false"
+
+          # coerce hbase i64s to Fixnum, Bignum
+          value = value.unpack('Q>').first if value.is_a?(String)
+
+          value
+        end
+
+        define_singleton_method :key_for_index do |index_name, data|
+
+          options = @@_indexes[index_name]
+
+          row = self.row_class.new(table: self, default_column_family: self.default_column_family, raw_data: data)
+          row_key_components = options[:attributes].map do |attribute|
+
+            value = if attribute == :index_id
+              options[:index_id]
+            else
+              row.attributes[attribute] || row.send(attribute)
+            end
+            encode_for_row_key(value)
+          end
+
+          row_key_components.pack(options[:pack_pattern].join(''))
+
+        end
+
+        define_singleton_method index_name do |idx_options, &block|
+          expected_option_keys = attributes[0...prefix_length]
+          prefix_pack_pattern = pack_pattern[0...prefix_length].join('')
+
+          prefix_components = expected_option_keys.map do |key|
+            value = key == :index_id ? index_id : idx_options[key]
+            encode_for_row_key(value)
+          end
+
+          row_prefix = prefix_components.pack(prefix_pack_pattern)
+
+          scan(row_prefix: row_prefix, &block)
+        end
+      end
+
+      def put(row_key, data, timestamp = nil, extra_indexes=[])
+        batch(timestamp).transaction do |batch|
+          @@_indexes.each_pair do |index_name, options|
+            next unless options[:auto_create] || extra_indexes.include?(index_name)
+
+            index_row_key = key_for_index(index_name, data)
+
+            batch.put(index_row_key, data)
+          end
+        end
+      end
+
+      def delete(row_key, columns=nil, timestamp=nil, indexes=[])
+        row = self.row(row_key)
+        attributes = row.attributes
+        if attributes[:row_key].blank? && attributes.except(:row_key).blank?
+          return
+        end
+
+        indexes = Array(indexes)
+
+        if indexes.empty?
+          indexes = @@_indexes.keys
+        end
+        self.batch(timestamp).transaction do |batch|
+          indexes.each do |index_name|
+            index_row_key = key_for_index(index_name, row.attributes)
+            batch.delete(index_row_key, columns)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ok_hbase/concerns/row.rb

@@ -0,0 +1,85 @@
+module OkHbase
+  module Concerns
+    module Row
+      attr_accessor :table, :row_key, :timestamp, :default_column_family
+      attr_reader :raw_data
+
+      def id
+        self.row_key
+      end
+
+      def id=(val)
+        self.row_key = val
+      end
+
+      def encoded_data
+        Hash[@raw_data.map { |k, v| [k, _encode(v)] }].with_indifferent_access
+      end
+
+      def attributes
+        hash = Hash[@raw_data.keys.map do |k|
+          k = k.split(':', 2).last
+          key_value = [k, send(k)]
+          key_value
+        end
+        ].with_indifferent_access
+
+        hash[:row_key] = @row_key
+        hash
+
+      end
+
+      def save!()
+        #raise ArgumentError.new "row_key must be a non-empty string" unless !@row_key.blank? && @row_key.is_a?(String)
+
+        table.put(row_key, encoded_data, timestamp)
+      end
+
+      def delete
+        table.delete(row_key)
+      end
+
+
+      def method_missing(method, *arguments, &block)
+        if method.to_s[-1, 1] == '='
+
+          key = method[0...-1]
+          val = arguments.last
+          unless key.to_s.include? ':'
+            key = "#{default_column_family}:#{key}"
+          else
+          end
+
+          ret_val = raw_data[key] = val
+
+          ret_val
+        else
+
+          unless method.to_s.include? ':'
+            key = "#{default_column_family}:#{method}"
+          else
+          end
+          return raw_data[key]
+        end
+
+      end
+
+      private
+      def _encode(value)
+        encoded = case value
+          when String
+            value.dup.force_encoding(Encoding::UTF_8)
+          when Bignum, Fixnum
+            [value].pack('Q>').force_encoding(Encoding::UTF_8)
+          when TrueClass, FalseClass
+            value.to_s.force_encoding(Encoding::UTF_8)
+          when NilClass
+            value
+        end
+
+        encoded
+      end
+
+    end
+  end
+end