hbase-stargate 1.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. data/LICENSE +20 -0
  2. data/README.textile +79 -0
  3. data/Rakefile +27 -0
  4. data/VERSION +1 -0
  5. data/lib/stargate.rb +7 -0
  6. data/lib/stargate/client.rb +93 -0
  7. data/lib/stargate/exception.rb +21 -0
  8. data/lib/stargate/model.rb +19 -0
  9. data/lib/stargate/model/column.rb +9 -0
  10. data/lib/stargate/model/column_descriptor.rb +32 -0
  11. data/lib/stargate/model/region_descriptor.rb +9 -0
  12. data/lib/stargate/model/row.rb +18 -0
  13. data/lib/stargate/model/scanner.rb +16 -0
  14. data/lib/stargate/model/table_descriptor.rb +8 -0
  15. data/lib/stargate/operation/meta_operation.rb +20 -0
  16. data/lib/stargate/operation/row_operation.rb +86 -0
  17. data/lib/stargate/operation/scanner_operation.rb +90 -0
  18. data/lib/stargate/operation/table_operation.rb +95 -0
  19. data/lib/stargate/request.rb +7 -0
  20. data/lib/stargate/request/basic_request.rb +27 -0
  21. data/lib/stargate/request/meta_request.rb +25 -0
  22. data/lib/stargate/request/row_request.rb +34 -0
  23. data/lib/stargate/request/scanner_request.rb +25 -0
  24. data/lib/stargate/request/table_request.rb +43 -0
  25. data/lib/stargate/response.rb +7 -0
  26. data/lib/stargate/response/basic_response.rb +25 -0
  27. data/lib/stargate/response/meta_response.rb +35 -0
  28. data/lib/stargate/response/row_response.rb +47 -0
  29. data/lib/stargate/response/scanner_response.rb +41 -0
  30. data/lib/stargate/response/table_response.rb +26 -0
  31. data/spec/hbase-stargate/model/column_descriptor_spec.rb +23 -0
  32. data/spec/hbase-stargate/model/column_spec.rb +12 -0
  33. data/spec/hbase-stargate/model/region_descriptor_spec.rb +4 -0
  34. data/spec/hbase-stargate/model/row_spec.rb +12 -0
  35. data/spec/hbase-stargate/model/scanner.rb +5 -0
  36. data/spec/hbase-stargate/model/table_descriptor_spec.rb +12 -0
  37. data/spec/hbase-stargate/operation/meta_operation_spec.rb +18 -0
  38. data/spec/hbase-stargate/operation/row_operation_spec.rb +75 -0
  39. data/spec/hbase-stargate/operation/scanner_operation_spec.rb +103 -0
  40. data/spec/hbase-stargate/operation/table_operation_spec.rb +43 -0
  41. data/spec/hbase-stargate/record_spec.rb +20 -0
  42. data/spec/hbase-stargate/request/meta_request_spec.rb +10 -0
  43. data/spec/hbase-stargate/request/row_request_spec.rb +4 -0
  44. data/spec/hbase-stargate/request/scanner_request_spec.rb +4 -0
  45. data/spec/hbase-stargate/request/table_request_spec.rb +4 -0
  46. data/spec/hbase-stargate/response/meta_response_spec.rb +4 -0
  47. data/spec/hbase-stargate/response/row_response_spec.rb +4 -0
  48. data/spec/hbase-stargate/response/scanner_response_spec.rb +4 -0
  49. data/spec/hbase-stargate/response/table_response_spec.rb +4 -0
  50. data/spec/spec.opts +4 -0
  51. data/spec/spec_helper.rb +3 -0
  52. metadata +144 -0
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2007 Dingding Ye
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.textile ADDED
@@ -0,0 +1,79 @@
1
+ h1. hbase-stargate
2
+
3
+ A Ruby client for HBase (http://hadoop.apache.org/hbase) that works with the Stargate interface.
4
+ Stargate is the RESTful web service front end for HBase that can serve up a number of formats including XML, JSON, and protobufs.
5
+
6
+ This project is based off the work of Dingding Ye <yedingding@gmail.com> at http://github.com/sishen/hbase-ruby
7
+
8
+
9
+ h2. Installation
10
+
11
+ <pre>
12
+ <code>$ gem install hbase-stargate -s http://gemcutter.org</code>
13
+ </pre>
14
+
15
+ To work with this gem in your Rails application, add this to the environment.rb file:
16
+ <pre><code>config.gem 'hbase-stargate', :lib => "stargate", :source => "http://gemcutter.org"</code></pre>
17
+
18
+ To build the gem yourself:
19
+ <pre><code>$ rake gem</code></pre>
20
+
21
+
22
+ h2. Getting Started
23
+
24
+ # Download and unpack the most recent release of HBase from http://hadoop.apache.org/hbase/releases.html#Download
25
+ # Edit <hbase-dir>/conf/hbase-env.sh and uncomment/modify the following line to correspond to your Java home path:
26
+ export JAVA_HOME=/usr/lib/jvm/java-6-sun
27
+ # Copy <hbase-dir>/contrib/stargate/hbase-<version>-stargate.jar into <hbase-dir>/lib
28
+ # Copy all the files in the <hbase-dir>/contrib/stargate/lib folder into <hbase-dir>/lib
29
+ # Start up HBase:
30
+ $ <hbase-dir>/bin/start-hbase.sh
31
+ # Start up Stargate (append "-p 1234" at the end if you want to change the port):
32
+ $ <hbase-dir>/bin/hbase org.apache.hadoop.hbase.stargate.Main
33
+
34
+
35
+ h2. Usage
36
+
37
+ Here are some examples:
38
+
39
+ <pre><code>
40
+ require 'stargate'
41
+
42
+ # Direct connection
43
+ # client = Stargate::Client.new("http://localhost:8080") # this url is the default for stargate.
44
+ # Connection through a proxy
45
+ client = Stargate::Client.new("http://localhost:8080", { :proxy => "10.2.3.4:8080" })
46
+
47
+ # Table Operation
48
+ tables = client.list_tables # list available tables
49
+ table = client.create_table('users', 'habbit') # create a table whose column_family is habbit
50
+ table = client.show_table('users') # show the meta info of table users
51
+ client.delete_table('users') # delete 'users' table
52
+
53
+ # Row Operation
54
+ row = client.show_row('users', 'sishen') # show the data of row 'sishen' in table 'users'
55
+ row2 = client.create_row('users', 'sishen', Time.now.to_i, {:name => 'habbit:football', :value => 'i like football'}) # create the row 'sishen' with the data in the table 'users'
56
+ client.delete_row('users', 'sishen', nil, 'habbit:football') # delete the row 'sishen' of table 'users' with the optional column 'habbit:football'
57
+
58
+ # Scanner Operation (see spec/stargate-client/operation/scanner_operation_spec.rb for more examples)
59
+ scanner = client.open_scanner('users', {:start_row => "row2", :batch => 5, :columns => ["habbit:"]}) # See more options from Stargate::Model::Scanner.AVAILABLE_OPTS
60
+ rows = client.get_rows(scanner)
61
+ client.close_scanner(scanner)
62
+ </code></pre>
63
+
64
+
65
+ h2. Testing
66
+
67
+ Run the specs with the following rake task:
68
+
69
+ $ rake spec
70
+
71
+ or pass it the URL to the HBase Stargate server as an argument:
72
+
73
+ $ rake spec STARGATE_URL=http://localhost:8080
74
+
75
+
76
+ h2. Copyright
77
+
78
+ Copyright (c) 2008 Dingding Ye <yedingding@gmail.com>
79
+ Distributed under MIT License
data/Rakefile ADDED
@@ -0,0 +1,27 @@
1
+ require 'spec/rake/spectask'
2
+
3
+ begin
4
+ require 'jeweler'
5
+
6
+ Jeweler::Tasks.new do |gemspec|
7
+ gemspec.name = "hbase-stargate"
8
+ gemspec.authors = ['Openplaces']
9
+ gemspec.email = 'greg.lu@gmail.com'
10
+ gemspec.homepage = "http://github.com/greglu/hbase-stargate"
11
+ gemspec.summary = "Ruby client for HBase's Stargate web service"
12
+ gemspec.description = "A Ruby client used to interact with HBase through its Stargate web service front-end"
13
+ gemspec.files = FileList["{lib,spec}/**/*","Rakefile","VERSION","LICENSE","README.textile"].to_a
14
+ gemspec.extra_rdoc_files = FileList["LICENSE","README.textile"].to_a
15
+
16
+ gemspec.add_development_dependency "rspec"
17
+ gemspec.add_dependency "json"
18
+ end
19
+ Jeweler::GemcutterTasks.new
20
+ rescue LoadError
21
+ puts "Jeweler not available. Install it with: sudo gem install jeweler"
22
+ end
23
+
24
+ Spec::Rake::SpecTask.new(:spec) do |t|
25
+ t.spec_files = FileList['spec/**/*_spec.rb']
26
+ t.spec_opts = File.open("spec/spec.opts").readlines.map{|x| x.chomp}
27
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.5.0
data/lib/stargate.rb ADDED
@@ -0,0 +1,7 @@
1
+ module Stargate end
2
+
3
+ require File.join(File.dirname(__FILE__), "stargate", "client")
4
+ require File.join(File.dirname(__FILE__), "stargate", "exception")
5
+ require File.join(File.dirname(__FILE__), "stargate", "model")
6
+ require File.join(File.dirname(__FILE__), "stargate", "request")
7
+ require File.join(File.dirname(__FILE__), "stargate", "response")
@@ -0,0 +1,93 @@
1
+ require 'net/http'
2
+ require File.dirname(__FILE__) + '/operation/meta_operation'
3
+ require File.dirname(__FILE__) + '/operation/table_operation'
4
+ require File.dirname(__FILE__) + '/operation/row_operation'
5
+ require File.dirname(__FILE__) + '/operation/scanner_operation'
6
+
7
+ module Stargate
8
+ class Client
9
+ VERSION = File.read(File.join(File.dirname(__FILE__), "..", "..", "VERSION")).chomp.freeze
10
+
11
+ include Operation::MetaOperation
12
+ include Operation::TableOperation
13
+ include Operation::RowOperation
14
+ include Operation::ScannerOperation
15
+
16
+ attr_reader :url, :connection
17
+
18
+ def initialize(url = "http://localhost:8080", opts = {})
19
+ @url = URI.parse(url)
20
+ unless @url.kind_of? URI::HTTP
21
+ raise "invalid http url: #{url}"
22
+ end
23
+
24
+ # Not actually opening the connection yet, just setting up the persistent connection.
25
+ if opts[:proxy]
26
+ proxy_address, proxy_port = opts[:proxy].split(':')
27
+ @connection = Net::HTTP.Proxy(proxy_address, proxy_port).new(@url.host, @url.port)
28
+ else
29
+ @connection = Net::HTTP.new(@url.host, @url.port)
30
+ end
31
+ @connection.read_timeout = opts[:timeout] if opts[:timeout]
32
+ end
33
+
34
+ def get(path, options = {})
35
+ safe_request { @connection.get(@url.path + path, {"Accept" => "application/json"}.merge(options)) }
36
+ end
37
+
38
+ def get_response(path, options = {})
39
+ safe_response { @connection.get(@url.path + path, {"Accept" => "application/json"}.merge(options)) }
40
+ end
41
+
42
+ def post(path, data = nil, options = {})
43
+ safe_request { @connection.post(@url.path + path, data, {'Content-Type' => 'text/xml'}.merge(options)) }
44
+ end
45
+
46
+ def post_response(path, data = nil, options = {})
47
+ safe_response { @connection.post(@url.path + path, data, {'Content-Type' => 'text/xml'}.merge(options)) }
48
+ end
49
+
50
+ def delete(path, options = {})
51
+ safe_request { @connection.delete(@url.path + path, options) }
52
+ end
53
+
54
+ def delete_response(path, options = {})
55
+ safe_response { @connection.delete(@url.path + path, options) }
56
+ end
57
+
58
+ def put(path, data = nil, options = {})
59
+ safe_request { @connection.put(@url.path + path, data, {'Content-Type' => 'text/xml'}.merge(options)) }
60
+ end
61
+
62
+ def put_response(path, data = nil, options = {})
63
+ safe_response { @connection.put(@url.path + path, data, {'Content-Type' => 'text/xml'}.merge(options)) }
64
+ end
65
+
66
+ private
67
+
68
+ # Part of safe_request was broken up into safe_response because when working with scanners
69
+ # in Stargate, you need to have access to the response itself, and not just the body.
70
+ def safe_response(&block)
71
+ begin
72
+ yield
73
+ rescue Errno::ECONNREFUSED
74
+ raise ConnectionNotEstablishedError, "can't connect to #{@url}"
75
+ rescue Timeout::Error => e
76
+ puts e.backtrace.join("\n")
77
+ raise ConnectionTimeoutError, "execution expired. Maybe query disabled tables"
78
+ end
79
+ end
80
+
81
+ def safe_request(&block)
82
+ response = safe_response{ yield block }
83
+
84
+ case response
85
+ when Net::HTTPSuccess
86
+ response.body
87
+ else
88
+ response.error!
89
+ end
90
+ end
91
+
92
+ end
93
+ end
@@ -0,0 +1,21 @@
1
+ class Stargate::Exception < StandardError; end
2
+
3
+ class Stargate::ConnectionNotEstablishedError < Stargate::Exception; end
4
+
5
+ class Stargate::ConnectionTimeoutError < Stargate::Exception; end
6
+
7
+ class Stargate::TableNotFoundError < Stargate::Exception; end
8
+
9
+ class Stargate::TableExistsError < Stargate::Exception; end
10
+
11
+ class Stargate::TableFailCreateError < Stargate::Exception; end
12
+
13
+ class Stargate::TableNotDisabledError < Stargate::Exception; end
14
+
15
+ class Stargate::TableFailDisableError < Stargate::Exception; end
16
+
17
+ class Stargate::TableFailEnableError < Stargate::Exception; end
18
+
19
+ class Stargate::RowNotFoundError < Stargate::Exception; end
20
+
21
+ class Stargate::ScannerError < Stargate::Exception; end
@@ -0,0 +1,19 @@
1
+ module Stargate
2
+ module Model
3
+ class Record
4
+ def initialize (params)
5
+ params.each do |key, value|
6
+ name = key.to_s
7
+ instance_variable_set("@#{name}", value) if respond_to?(name)
8
+ end
9
+ end
10
+ end
11
+ end
12
+ end
13
+
14
+ require File.dirname(__FILE__) + '/model/column'
15
+ require File.dirname(__FILE__) + '/model/column_descriptor'
16
+ require File.dirname(__FILE__) + '/model/region_descriptor'
17
+ require File.dirname(__FILE__) + '/model/row'
18
+ require File.dirname(__FILE__) + '/model/table_descriptor'
19
+ require File.dirname(__FILE__) + '/model/scanner'
@@ -0,0 +1,9 @@
1
+ module Stargate
2
+ module Model
3
+ class Column < Record
4
+ attr_accessor :name
5
+ attr_accessor :value
6
+ attr_accessor :timestamp
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,32 @@
1
+ module Stargate
2
+ module Model
3
+ module CompressionType
4
+ NONE = "NONE"
5
+ RECORD = "RECORD"
6
+ BLOCK = "BLOCK"
7
+
8
+ CTYPES = [NONE, RECORD, BLOCK]
9
+
10
+ def to_compression_type(type_string)
11
+ CTYPES.include?(type_string) ? type_string : NONE
12
+ end
13
+
14
+ module_function :to_compression_type
15
+ end
16
+
17
+ class ColumnDescriptor < Record
18
+ AVAILABLE_OPTS = { :name => "name", :max_versions => "VERSIONS", :versions => "VERSIONS",
19
+ :compression => "COMPRESSION", :in_memory => "IN_MEMORY",
20
+ :block_cache => "BLOCKCACHE", :blockcache => "BLOCKCACHE",
21
+ :blocksize => "BLOCKSIZE", :length => "LENGTH", :ttl => "TTL",
22
+ :bloomfilter => "BLOOMFILTER"}
23
+ attr_accessor :name
24
+ attr_accessor :compression
25
+ attr_accessor :bloomfilter
26
+ attr_accessor :maximum_cell_size
27
+ attr_accessor :max_versions
28
+
29
+ attr_accessor :versions
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,9 @@
1
+ module Stargate
2
+ module Model
3
+ class Region < Record
4
+ end
5
+
6
+ class RegionDescriptor < Record
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,18 @@
1
+ module Stargate
2
+ module Model
3
+ class Row < Record
4
+ attr_accessor :table_name
5
+ attr_accessor :name
6
+ attr_accessor :total_count
7
+ attr_accessor :columns
8
+
9
+ def timestamp
10
+ warn "[DEPRECATION] timestamp attribute will be removed from the Row model. "
11
+ end
12
+
13
+ def timestamp=
14
+ timestamp
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,16 @@
1
+ module Stargate
2
+ module Model
3
+ class Scanner < Record
4
+ AVAILABLE_OPTS = { :start_row => "startRow", :end_row => "endRow",
5
+ :start_time => "startTime", :end_time => "endTime",
6
+ :batch => "batch" }
7
+
8
+ attr_accessor :table_name
9
+ attr_accessor :scanner_url
10
+ attr_accessor :batch_size
11
+
12
+ # Deprecation: scanner_url is used instead of just the ID
13
+ attr_accessor :scanner_id
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,8 @@
1
+ module Stargate
2
+ module Model
3
+ class TableDescriptor < Record
4
+ attr_accessor :name
5
+ attr_accessor :column_families
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,20 @@
1
+ module Stargate
2
+ module Operation
3
+ module MetaOperation
4
+ def list_tables
5
+ request = Request::MetaRequest.new
6
+ Response::MetaResponse.new(get(request.list_tables), :list_tables).parse
7
+ end
8
+
9
+ def version
10
+ request = Request::MetaRequest.new
11
+ get(request.version, {"Accept" => "text/plain"}).strip
12
+ end
13
+
14
+ def cluster_version
15
+ request = Request::MetaRequest.new
16
+ get(request.cluster_version, {"Accept" => "text/plain"}).strip
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,86 @@
1
+ module Stargate
2
+ module Operation
3
+ module RowOperation
4
+ Converter = {
5
+ '&' => '&amp;',
6
+ '<' => '&lt;',
7
+ '>' => '&gt;',
8
+ "'" => '&apos;',
9
+ '"' => '&quot;'
10
+ }
11
+
12
+ def row_timestamps(table_name, name)
13
+ raise NotImplementedError, "Currently not supported in Stargate client"
14
+ end
15
+
16
+ def show_row(table_name, name, timestamp = nil, columns = nil, options = { })
17
+ begin
18
+ options[:version] ||= 1
19
+
20
+ request = Request::RowRequest.new(table_name, name, timestamp)
21
+ row = Response::RowResponse.new(get(request.show(columns, options)), :show_row).parse.first
22
+ row.table_name = table_name
23
+ row
24
+ rescue Net::ProtocolError => e
25
+ # TODO: Use better handling instead of this.
26
+ if e.to_s.include?("Table")
27
+ raise TableNotFoundError, "Table '#{table_name}' Not Found"
28
+ elsif e.to_s.include?("404")
29
+ raise RowNotFoundError, "Row '#{name}' Not Found"
30
+ end
31
+ end
32
+ end
33
+
34
+ def create_row(table_name, name, timestamp = nil, columns = nil)
35
+ begin
36
+ request = Request::RowRequest.new(table_name, name, timestamp)
37
+ data = []
38
+ if columns
39
+ if columns.instance_of? Array
40
+ data = columns
41
+ elsif columns.instance_of? Hash
42
+ data = [columns]
43
+ else
44
+ raise StandardError, "Only Array or Hash data accepted"
45
+ end
46
+ end
47
+
48
+ xml_data = "<?xml version='1.0' encoding='UTF-8' standalone='yes'?><CellSet>"
49
+ xml_data << "<Row key='#{[name].pack('m') rescue ''}'>"
50
+ data.each do |d|
51
+ escape_name = d[:name].gsub(/[&<>'"]/) { |match| Converter[match] }
52
+ xml_data << "<Cell "
53
+ xml_data << "timestamp='#{timestamp}' " if timestamp
54
+ xml_data << "column='#{[escape_name].pack('m') rescue ''}'>"
55
+ xml_data << "#{[d[:value]].pack("m") rescue ''}"
56
+ xml_data << "</Cell>"
57
+ end
58
+ xml_data << "</Row></CellSet>"
59
+
60
+ Response::RowResponse.new(post_response(request.create(data.map{|col| col[:name]}), xml_data), :create_row).parse
61
+ rescue Net::ProtocolError => e
62
+ if e.to_s.include?("Table")
63
+ raise TableNotFoundError, "Table '#{table_name}' Not Found"
64
+ elsif e.to_s.include?("Row")
65
+ raise RowNotFoundError, "Row '#{name}' Not Found"
66
+ else
67
+ raise StandardError, "Error encountered while trying to create row: #{e.message}"
68
+ end
69
+ end
70
+ end
71
+
72
+ def delete_row(table_name, name, timestamp = nil, columns = nil)
73
+ begin
74
+ request = Request::RowRequest.new(table_name, name, timestamp)
75
+ Response::RowResponse.new(delete_response(request.delete(columns)), :delete_row).parse
76
+ rescue Net::ProtocolError => e
77
+ if e.to_s.include?("Table")
78
+ raise TableNotFoundError, "Table '#{table_name}' Not Found"
79
+ elsif e.to_s.include?("Row")
80
+ raise RowNotFoundError, "Row '#{name}' Not Found"
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end