ok_hbase 0.0.5

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.idea/*
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format=progress

data/.rvmrc

@@ -0,0 +1 @@
+rvm use --create --install ruby-1.9.3@ok_hbase

data/Gemfile

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ok_hbase.gemspec
+gemspec
+
+
+group :development, :test do
+  gem 'awesome_print'
+  gem 'bundler'
+end
+
+group :test do
+  gem 'json', '~> 1.7.7'
+  gem 'rake'
+  gem 'rspec'
+  gem 'simplecov', require: false
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Nathan Keyes
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# ok_hbase
+
+Welcome HBase cowboys.
+
+Read the [wiki](https://github.com/okcwest/ok-hbase/wiki)!
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'ok_hbase'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ok_hbase
+
+## Usage
+
+```bash
+$ bundle console
+Resolving dependencies...
+irb(main):030:0> connection = OkHbase::Connection.new(host: "hbase-dev")
+=> #<OkHbase::Connection:0x00000002440140 <snip>
+irb(main):031:0> table = OkHbase::Table.new(mytable, connection)
+=> #<OkHbase::Table:0x00000002449f38 <snip> 
+irb(main):032:0> count = 0
+=> 0
+irb(main):033:0> table.scan row_prefix: [ myid, 1, 5, 1, 0 ].pack("L>CCCC") do |row, col|
+irb(main):034:1*   count += 1
+irb(main):035:1> end
+=> nil
+irb(main):036:0> count
+=> 1072
+irb(main):037:0> connection.close
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'bundler'
+Bundler.setup :default, :test, :development
+
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec
+task :default => :spec
+
+Dir['tasks/**/*.{rb}'].each do |file|
+  require_relative file
+end

data/examples/README.md

@@ -0,0 +1,46 @@
+# okhbase examples
+Simple examples to illustrate using the api to do basic work. 
+
+## Usage
+
+### Create table and Write Data
+
+First we create a table, and put a bunch of data in it.
+This command will crate a table named 'ok_hbase_test', withs a single column family: 'd'.
+It then sequentially creates rows with row keys a through zzz, 18278 rows in total.
+That may take a few minutes, as we are not batching the writes.
+
+```bash
+$ ./table_write.rb  --host localhost --port 9090 --table ok_hbase_test
+```
+
+### Read Data
+Now that we have daa in the table, we want to read it.
+The following script will scan the table we just made for any rows that begin with 'hba'
+
+```bash
+$ ./table_scan.rb  --host localhost --table ok_hbase_test --prefix hba
+```
+
+The output should look like:
+```bash
+Nathans-MacBook-Pro-2:ok_hbase nkeyes$ ./examples/table_scan.rb  --host localhost --table ok_hbase_test --prefix hba
+2013-06-11 08:36:13 -0700 DEBUG: Setting up connection
+2013-06-11 08:36:13 -0700 DEBUG: Connecting to localhost
+2013-06-11 08:36:13 -0700 DEBUG: Get instance for table ok_hbase_test
+{
+    "hba" => {
+           "d:message" => "this is row number 5461",
+        "d:row_number" => "5461"
+    }
+}
+Nathans-MacBook-Pro-2:ok_hbase nkeyes$
+```
+
+Experiment with shorter prefixes to see more rows returned.
+An empty prefix will return all rows:
+
+```bash
+./table_scan.rb  --host localhost --table ok_hbase_test --prefix ''
+```
+

data/examples/advanced/README.md

@@ -0,0 +1,36 @@
+# okhbase examples
+Examples to illustrate using the api to do basic work.
+
+## Usage
+
+See: http://hbase.apache.org/book.html if you are new to hbase.
+
+With bundler up to date, usage():
+
+```bash
+$ bundle exec examples/perf_read.rb -h
+Usage: examples/perf_read.rb [options]
+    -h, --help                       Display this help
+    -v, --verbose                    Output json result
+    -n, --host HOSTNAME              hostname of RegionServer or  master
+    -t, --table TABLE                hbase table name
+        --port PORT                  port number of thrift server, defaults to 9090
+        --timeout TIMEOUT            connect timeout, defaults to 600
+    -a, --array ARRAY                array values for pack, in csv, no whitespace in the format of "11111111,1,1,1,1"
+    -p, --pack PACK                  template string to build binary sequence from literal passed to -a
+    -i, --iterations NUM             number of iterations to run the benchmark, defaults to 10
+```
+
+the table reading examples are built around the ruby Array api using Array.pack to build binary sequences to needle our data out of the hbase haystack. Most people use keys comprised of timestamps, IDs and other data.
+
+If we had a rowkey with a id and flags in a table called "chat" we could run the perf_read.rb example like this:
+
+```bash
+$ bundle exec examples/perf_read.rb -n 127.0.0.1 -t chat --array='22221111,7,3,4,0' -p "L>CCCC" -i 10 -v
+```
+
+## WIP
+* perf_write.rb: currently a wip , use at your own risk
+* table_write.rb: currently a wip, use at your own risk
+* table_read.rb: tested
+* perf_read.rb: tested

data/examples/advanced/perf_read.rb

@@ -0,0 +1,146 @@
+#!/usr/bin/env ruby
+#
+# perf_read.rb - basic read perf test
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+$stdout.sync = true
+
+require 'awesome_print'
+require 'ok_hbase'
+require 'optparse'
+require 'logger'
+
+$options = {}
+$logger = Logger.new(STDOUT)
+$logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime} #{severity}: #{msg}\n" }
+$logger.level = Logger::FATAL
+
+def usage(error=nil)
+  puts "Error: #{error}\n\n" if error
+  puts $optparse
+  exit 1
+end
+
+def get_connection(table=nil)
+  $logger.debug "Setting up connection for table #{table}"
+  if table.nil?
+    $logger.fatal "Must specify a table"
+    return nil
+  end
+    
+  $logger.debug "Connecting to #{$options[:hostname]}"
+  conn = OkHbase::Connection.new(auto_connect: true, host: $options[:hostname], port: $options[:port],
+      timeout: $options[:timeout])
+  $logger.debug "Get instance for table #{table}"
+  OkHbase::Table.new(table, conn)
+end
+
+def get_filter()
+  $logger.debug "Setup byte sequence using #{$options[:filter_array].to_s}.pack(#{$options[:filter_pack].to_s})"
+  $options[:filter_array].pack($options[:filter_pack].to_s)
+end
+
+def get_row_count(conn, prefix)
+  row_count = 0
+  conn.scan row_prefix: prefix, caching: 5000 do |row, cols|
+    row_count += 1
+  end
+
+  row_count
+end
+
+def perf_test_a()
+  c = get_connection($options[:table])
+  filter = get_filter()
+  Benchmark.bm(7, ">total:", ">avg:") do |x|
+    runs = []
+    $options[:iterations].times do |i|
+      runs << x.report("Run #{i+1}:") do
+        count = get_row_count(c, filter).to_s
+        $logger.debug "Found #{count} row(s)"
+      end
+    end
+    [runs.sum, runs.sum/runs.size]
+  end
+end
+
+def perf_test_b()
+  c = get_connection($options[:table])
+  filter = get_filter()
+  bench_times = []
+ 
+  $options[:iterations].times do |i|
+    bench_times.push(Benchmark.realtime { count = get_row_count(c, filter).to_s })
+    $logger.debug "Benchmark iteration ##{i+1}: #{bench_times[-1]} second(s)"
+  end
+
+  bench_times.each_with_index do |v,k|
+    puts "Run #{k+1}: #{v} second(s)"
+  end
+  puts "\nMedian: #{ bench_times[bench_times.length / 2] } second(s)"
+  puts "Average: #{ bench_times.inject(:+) / bench_times.length } second(s)"
+  puts "Higest Time: #{ bench_times.sort[-1] } second(s)"
+  puts "Lowest Time: #{ bench_times.sort[0] } second(s)"
+end
+
+def main()
+  $optparse = OptionParser.new do|opts|
+    opts.banner = "Usage: #{__FILE__} [options]"
+
+    $options[:verbose] = false
+    $options[:port] = 9090
+    $options[:timeout] = 600
+    $options[:iterations] = 10
+
+    opts.on('-h', '--help', 'Display this help') do
+      usage
+    end
+
+    opts.on('-v', '--verbose', 'Output json result') do
+      $options[:verbose] = true
+      $logger.level = Logger::DEBUG
+    end
+   
+    opts.on('-n', '--host HOSTNAME', 'hostname of RegionServer or  master') do |hostname|
+      $options[:hostname] = hostname
+    end
+
+    opts.on('-t', '--table TABLE', 'hbase table name') do |table|
+      $options[:table] = table
+    end
+
+    opts.on('-p', '--port PORT', "port number of thrift server, defaults to #{$options[:port]}") do |port|
+      $options[:port] = port.to_i
+    end
+
+    opts.on('--timeout TIMEOUT', "connect timeout, defaults to #{$options[:timeout]}") do |timeout|
+      $options[:timeout] = timeout.to_i
+    end
+
+    opts.on('-a', '--array ARRAY', Array, "array values for pack, in csv, no whitespace in the format of \"11111111,1,1,1,1\"") do |ar|
+      $options[:filter_array] = ar.map(&:to_i)
+    end
+
+    opts.on('-p', '--pack PACK', "template string to build binary sequence from literal passed to -a") do |pack|
+      $options[:filter_pack] = pack.to_s
+    end
+    
+    opts.on('-i', '--iterations NUM', "number of iterations to run the benchmark, defaults to #{$options[:iterations]}") do |num|
+      $options[:iterations] = num.to_i
+    end
+
+  end
+
+  usage "You didn't specify any options" if not ARGV[0]
+
+  $optparse.parse!
+
+  usage "You didn't specify a hostname" if not $options[:hostname]
+  usage "You didn't specify a table" if not $options[:table]
+  usage "You didn't specify an array literal" if not $options[:filter_array]
+  usage "You didn't specify a binary sequence template" if not $options[:filter_pack]
+
+  perf_test_b()
+end
+
+main() if __FILE__ == $0

data/examples/advanced/perf_write.rb

@@ -0,0 +1,143 @@
+#!/usr/bin/env ruby
+#
+# perf_write.rb - basic table write benchmarks
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+$stdout.sync = true
+
+require 'awesome_print'
+require 'ok_hbase'
+require 'optparse'
+require 'logger'
+
+$options = {}
+$logger = Logger.new(STDOUT)
+$logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime} #{severity}: #{msg}\n" }
+$logger.level = Logger::FATAL
+
+def usage(error=nil)
+  puts "Error: #{error}\n\n" if error
+  puts $optparse
+  exit 1
+end
+
+def get_connection(table=nil)
+  $logger.debug "Setting up connection for table #{table}"
+  if table.nil?
+    $logger.fatal "Must specify a table"
+    return nil
+  end
+    
+  $logger.debug "Connecting to #{$options[:hostname]}"
+  conn = OkHbase::Connection.new(auto_connect: true, host: $options[:hostname], port: $options[:port],
+      timeout: $options[:timeout])
+  $logger.debug "Get instance for table #{table}"
+  OkHbase::Table.new(table, conn)
+end
+
+def write_test_row(conn, rowkey)
+  # set any column family shit
+  # use a pack method to build the binary sequence
+  puts 'wrote all the things'
+end
+
+def write_batch_row(conn, rowkey)
+  # set any column family shit
+
+  $options[:rowcount].times do |i|
+    # increment and write
+    puts 'wrote something things'
+  end
+end
+
+
+def get_rowkey()
+  # get a incrementor value if needed
+  # set attributes for a row key
+  # setup any time data
+  # use a pack method to build the binary sequence
+  # return binary sequence or decimal sequence to ok-hbase
+  puts "rowkey"
+end
+
+def perf_test()
+  c = get_connection($options[:table])
+  row_key  = get_rowkey()
+  bench_times = []
+
+  $options[:iterations].times do |i|
+    bench_times.push(Benchmark.realtime { write_batch_row(c, row_key) })
+    $logger.debug "Benchmark iteration ##{i+1}: #{bench_times[-1]} second(s)"
+  end
+
+  bench_times.each_with_index do |v,k|
+    puts "Run #{k+1}: #{v} second(s)"
+  end
+  puts "\nMedian: #{ bench_times[bench_times.length / 2] } second(s)"
+  puts "Average: #{ bench_times.inject(:+) / bench_times.length } second(s)"
+  puts "Higest Time: #{ bench_times.sort[-1] } second(s)"
+  puts "Lowest Time: #{ bench_times.sort[0] } second(s)"
+end
+
+
+def main()
+  $optparse = OptionParser.new do|opts|
+    opts.banner = "Usage: #{__FILE__} [options]"
+
+    $options[:verbose] = false
+    $options[:port] = 9090
+    $options[:timeout] = 600
+    $options[:rowcount] = 1
+
+    opts.on('-h', '--help', 'Display this help') do
+      usage
+    end
+
+    opts.on('-v', '--verbose', 'Output json result') do
+      $options[:verbose] = true
+      $logger.level = Logger::DEBUG
+    end
+   
+    opts.on('-n', '--host HOSTNAME', 'hostname of RegionServer or  master') do |hostname|
+      $options[:hostname] = hostname
+    end
+
+    opts.on('-t', '--table TABLE', 'hbase table name') do |table|
+      $options[:table] = table
+    end
+
+    opts.on('-p', '--port PORT', "port number of thrift server, defaults to #{$options[:port]}") do |port|
+      $options[:port] = port.to_i
+    end
+
+    opts.on('--timeout TIMEOUT', "connect timeout, defaults to #{$options[:timeout]}") do |timeout|
+      $options[:timeout] = timeout.to_i
+    end
+
+    opts.on('-a', '--array ARRAY', Array, "array values for pack for rowkey, comma separated, no whitespace in the format of \"11111111,1,1,1,1\"") do |ar|
+      $options[:filter_array] = ar.map(&:to_i)
+    end
+
+    opts.on('-p', '--pack PACK', "template string to build binary sequence from literal passed to -a") do |pack|
+      $options[:filter_pack] = pack.to_s
+    end
+
+    opts.on('-w', '--write ROWS', "how many times to write with a row key defaults to #{$options[:rowcount]}") do |row|
+      $options[:rowcount] = row.to_i
+    end
+
+  end
+
+  usage "You didn't specify any options" if not ARGV[0]
+
+  $optparse.parse!
+
+  usage "You didn't specify a hostname" if not $options[:hostname]
+  usage "You didn't specify a table" if not $options[:table]
+  usage "You didn't specify an array literal" if not $options[:filter_array]
+  usage "You didn't specify a binary sequence template" if not $options[:filter_pack]
+
+  perf_test()
+end
+
+main() if __FILE__ == $0

data/examples/advanced/table_read.rb

@@ -0,0 +1,115 @@
+#!/usr/bin/env ruby
+#
+# table_read.rb - basic table read examples
+
+$:.unshift File.expand_path('../../lib', __FILE__)
+$stdout.sync = true
+
+require 'awesome_print'
+require 'ok_hbase'
+require 'optparse'
+require 'logger'
+
+$options = {}
+$logger = Logger.new(STDOUT)
+# $logger.formatter = proc do |severity, date, progname, msg| "#{date} #{severity}: #{msg}\n" end
+$logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime} #{severity}: #{msg}\n" }
+$logger.level = Logger::FATAL
+
+def usage(error=nil)
+  puts "Error: #{error}\n\n" if error
+  puts $optparse
+  exit 1
+end
+
+def get_connection(table=nil)
+  $logger.debug "Setting up connection for table #{table}"
+  if table.nil?
+    $logger.fatal "Must specify a table"
+    return nil
+  end
+    
+  $logger.debug "Connecting to #{$options[:hostname]}"
+  conn = OkHbase::Connection.new(auto_connect: true, host: $options[:hostname], port: $options[:port],
+      timeout: $options[:timeout])
+  $logger.debug "Get instance for table #{table}"
+  OkHbase::Table.new(table, conn)
+end
+
+def get_filter()
+  $logger.debug "Setup byte sequence using #{$options[:filter_array].to_s}.pack(#{$options[:filter_pack].to_s})"
+  $options[:filter_array].pack($options[:filter_pack].to_s)
+end
+
+def get_row_count(conn, prefix)
+  row_count = 0
+  $logger.debug "Getting row count"
+  conn.scan row_prefix: prefix, caching: 5000 do |row, cols|
+    $logger.debug row.to_s
+    row_count += 1
+  end
+
+  row_count
+end
+
+def main()
+  $optparse = OptionParser.new do|opts|
+    opts.banner = "Usage: #{__FILE__} [options]"
+
+    $options[:verbose] = false
+    $options[:port] = 9090
+    $options[:timeout] = 600
+
+    opts.on('-h', '--help', 'Display this help') do
+      usage
+    end
+
+    opts.on('-v', '--verbose', 'Output json result') do
+      $options[:verbose] = true
+      $logger.level = Logger::DEBUG
+    end
+   
+    opts.on('-n', '--host HOSTNAME', 'hostname of RegionServer or  master') do |hostname|
+      $options[:hostname] = hostname
+    end
+
+    opts.on('-t', '--table TABLE', 'hbase table name') do |table|
+      $options[:table] = table
+    end
+
+    opts.on('-p', '--port PORT', "port number of thrift server, defaults to #{$options[:port]}") do |port|
+      $options[:port] = port.to_i
+    end
+
+    opts.on('--timeout TIMEOUT', "connect timeout, defaults to #{$options[:timeout]}") do |timeout|
+      $options[:timeout] = timeout.to_i
+    end
+
+    opts.on('-a', '--array ARRAY', Array, "array values for pack, in csv, no whitespace in the format of \"11111111,1,1,1,1\"") do |ar|
+      $options[:filter_array] = ar.map(&:to_i)
+    end
+
+    opts.on('-p', '--pack PACK', "template string to build binary sequence from literal passed to -a") do |pack|
+      $options[:filter_pack] = pack.to_s
+    end
+
+  end
+
+  usage "You didn't specify any options" if not ARGV[0]
+
+  $optparse.parse!
+
+  usage "You didn't specify a hostname" if not $options[:hostname]
+  usage "You didn't specify a table" if not $options[:table]
+  usage "You didn't specify an array literal" if not $options[:filter_array]
+  usage "You didn't specify a binary sequence template" if not $options[:filter_pack]
+
+  start_time = Time.now
+  c = get_connection($options[:table])
+  filter = get_filter()
+  count = get_row_count(c, filter).to_s
+  total_time = Time.now - start_time
+  puts "Found #{count} row(s) in #{total_time} second(s)"
+end
+
+main() if __FILE__ == $0