bigrecord-driver 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/bin/bigrecord-driver +41 -40
- data/bin/hbase-driver +12 -12
- data/lib/big_record_driver.rb +13 -6
- data/lib/big_record_driver/client.rb +32 -29
- data/lib/big_record_driver/column_descriptor.rb +13 -16
- data/lib/big_record_driver/exceptions.rb +12 -10
- data/lib/big_record_driver/hbase_driver/server.rb +351 -320
- data/lib/big_record_driver/server.rb +123 -0
- data/lib/big_record_driver/version.rb +4 -2
- data/test/abstract_test_client.rb +5 -6
- data/test/test_client_hbase.rb +13 -17
- metadata +3 -5
- data/lib/big_record_driver/bigrecord_server.rb +0 -119
- data/lib/big_record_driver/driver_manager.rb +0 -34
- data/test/test_driver_manager.rb +0 -46
@@ -0,0 +1,123 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/exceptions'
|
2
|
+
require File.dirname(__FILE__) + '/column_descriptor'
|
3
|
+
require 'drb'
|
4
|
+
|
5
|
+
# The name of the java String class conflicts with ruby's String class.
|
6
|
+
module Java
|
7
|
+
java_import "java.lang.String"
|
8
|
+
java_import "java.lang.Exception"
|
9
|
+
end
|
10
|
+
|
11
|
+
class String
|
12
|
+
def to_bytes
|
13
|
+
Java::String.new(self).getBytes
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
module BigRecord
|
19
|
+
module Driver
|
20
|
+
|
21
|
+
class Server
|
22
|
+
java_import "java.io.IOException"
|
23
|
+
|
24
|
+
def configure(config = {})
|
25
|
+
raise NotImplementedError
|
26
|
+
end
|
27
|
+
|
28
|
+
def update(table_name, row, values, timestamp=nil)
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
|
32
|
+
def get(table_name, row, column, options={})
|
33
|
+
raise NotImplementedError
|
34
|
+
end
|
35
|
+
|
36
|
+
def get_columns(table_name, row, columns, options={})
|
37
|
+
raise NotImplementedError
|
38
|
+
end
|
39
|
+
|
40
|
+
def get_consecutive_rows(table_name, start_row, limit, columns, stop_row = nil)
|
41
|
+
raise NotImplementedError
|
42
|
+
end
|
43
|
+
|
44
|
+
def delete(table_name, row)
|
45
|
+
raise NotImplementedError
|
46
|
+
end
|
47
|
+
|
48
|
+
def create_table(table_name, column_descriptors)
|
49
|
+
raise NotImplementedError
|
50
|
+
end
|
51
|
+
|
52
|
+
def drop_table(table_name)
|
53
|
+
raise NotImplementedError
|
54
|
+
end
|
55
|
+
|
56
|
+
def truncate_table(table_name)
|
57
|
+
raise NotImplementedError
|
58
|
+
end
|
59
|
+
|
60
|
+
def ping
|
61
|
+
raise NotImplementedError
|
62
|
+
end
|
63
|
+
|
64
|
+
def table_exists?(table_name)
|
65
|
+
raise NotImplementedError
|
66
|
+
end
|
67
|
+
|
68
|
+
def table_names
|
69
|
+
raise NotImplementedError
|
70
|
+
end
|
71
|
+
|
72
|
+
def method_missing(method, *args)
|
73
|
+
super
|
74
|
+
rescue NoMethodError
|
75
|
+
raise NoMethodError, "undefined method `#{method}' for \"#{self}\":#{self.class}"
|
76
|
+
end
|
77
|
+
|
78
|
+
def respond_to?(method)
|
79
|
+
super
|
80
|
+
end
|
81
|
+
|
82
|
+
protected
|
83
|
+
|
84
|
+
def to_ruby_string(byte_string)
|
85
|
+
Java::String.new(byte_string).to_s
|
86
|
+
end
|
87
|
+
|
88
|
+
# Try to recover from network related exceptions. e.g. hbase has been restarted and the
|
89
|
+
# cached connections in @tables are no longer valid. Every method in this class (except connect_table)
|
90
|
+
# should have its code wrapped by a call to this method.
|
91
|
+
def safe_exec
|
92
|
+
yield
|
93
|
+
rescue IOException => e
|
94
|
+
puts "A network error occured: #{e.message}. Trying to recover..."
|
95
|
+
init_connection
|
96
|
+
begin
|
97
|
+
yield
|
98
|
+
rescue Exception, Java::Exception => e2
|
99
|
+
if e2.class == e.class
|
100
|
+
puts "Failed to recover the connection."
|
101
|
+
else
|
102
|
+
puts "Failed to recover the connection but got a different error this time: #{e2.message}."
|
103
|
+
end
|
104
|
+
puts "Stack trace:"
|
105
|
+
puts e2.backtrace.join("\n")
|
106
|
+
|
107
|
+
if e2.kind_of?(NativeException)
|
108
|
+
raise BigRecord::Driver::JavaError, e2.message
|
109
|
+
else
|
110
|
+
raise e2
|
111
|
+
end
|
112
|
+
end
|
113
|
+
puts "Connection recovered successfully..."
|
114
|
+
rescue Exception => e
|
115
|
+
puts "\n#{e.class.name}: #{e.message}"
|
116
|
+
puts e.backtrace.join("\n")
|
117
|
+
raise e
|
118
|
+
end
|
119
|
+
|
120
|
+
end
|
121
|
+
|
122
|
+
end
|
123
|
+
end
|
@@ -1,9 +1,8 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'rubygems'
|
3
2
|
require 'test/unit'
|
4
|
-
require 'big_record_driver'
|
3
|
+
require File.dirname(__FILE__) + '/../lib/big_record_driver'
|
5
4
|
|
6
|
-
module AbstractTestClient
|
5
|
+
module AbstractTestClient
|
7
6
|
|
8
7
|
TABLE_NAME = :animals
|
9
8
|
|
@@ -286,7 +285,7 @@ module AbstractTestClient #< Test::Unit::TestCase
|
|
286
285
|
def test_ping
|
287
286
|
db = nil
|
288
287
|
assert_nothing_raised("Couldn't initialize the client") do
|
289
|
-
db =
|
288
|
+
db = BigRecord::Driver::Client.new(:drb_port => PORT)
|
290
289
|
end
|
291
290
|
assert_not_nil db, "Couldn't initialize the client"
|
292
291
|
assert db.ping, "The client was initialized but we cannot communicate with the db itself"
|
@@ -308,7 +307,7 @@ module AbstractTestClient #< Test::Unit::TestCase
|
|
308
307
|
end
|
309
308
|
|
310
309
|
def test_invalid_column_family
|
311
|
-
assert_raises
|
310
|
+
assert_raises BigRecord::Driver::JavaError do
|
312
311
|
@big_db.get(TABLE_NAME, 'dog-key', 'nonexistentcolumnfamily:name')
|
313
312
|
end
|
314
313
|
end
|
data/test/test_client_hbase.rb
CHANGED
@@ -1,26 +1,22 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/abstract_test_client'
|
2
2
|
|
3
|
-
|
3
|
+
PORT = (ARGV[0] || 40000).to_i
|
4
|
+
|
5
|
+
class TestHbaseClient < Test::Unit::TestCase
|
4
6
|
include AbstractTestClient
|
5
|
-
# Prepare the connection and the test tables.
|
6
|
-
def setup
|
7
|
-
unless @big_db
|
8
|
-
unless BigRecordDriver::DriverManager.running?(40005)
|
9
|
-
BigRecordDriver::DriverManager.restart(40005)
|
10
|
-
end
|
11
7
|
|
12
|
-
|
13
|
-
|
14
|
-
end
|
8
|
+
def setup
|
9
|
+
@big_db = BigRecord::Driver::Client.new({:zookeeper_quorum=> 'localhost', :zookeeper_client_port => '2181', :drb_port => PORT}) unless @big_db
|
15
10
|
|
16
11
|
@big_db.drop_table(TABLE_NAME) if @big_db.table_exists?(TABLE_NAME)
|
17
12
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
13
|
+
columns_descriptors = []
|
14
|
+
columns_descriptors << BigRecord::Driver::ColumnDescriptor.new(:columnfamily1)
|
15
|
+
columns_descriptors << BigRecord::Driver::ColumnDescriptor.new(:columnfamily2)
|
16
|
+
@big_db.create_table(TABLE_NAME, columns_descriptors)
|
17
|
+
end
|
18
|
+
|
19
|
+
def teardown
|
20
|
+
@big_db.drop_table(TABLE_NAME) if @big_db.table_exists?(TABLE_NAME)
|
25
21
|
end
|
26
22
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bigrecord-driver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- openplaces.org
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date:
|
12
|
+
date: 2010-02-12 00:00:00 -05:00
|
13
13
|
default_executable: hbase-driver
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -29,17 +29,15 @@ files:
|
|
29
29
|
- bin/hbase-driver
|
30
30
|
- conf/log4j.properties
|
31
31
|
- lib/big_record_driver.rb
|
32
|
-
- lib/big_record_driver/bigrecord_server.rb
|
33
32
|
- lib/big_record_driver/client.rb
|
34
33
|
- lib/big_record_driver/column_descriptor.rb
|
35
|
-
- lib/big_record_driver/driver_manager.rb
|
36
34
|
- lib/big_record_driver/exceptions.rb
|
37
35
|
- lib/big_record_driver/hbase_driver/server.rb
|
36
|
+
- lib/big_record_driver/server.rb
|
38
37
|
- lib/big_record_driver/version.rb
|
39
38
|
- lib/bigrecord_driver.rb
|
40
39
|
- test/abstract_test_client.rb
|
41
40
|
- test/test_client_hbase.rb
|
42
|
-
- test/test_driver_manager.rb
|
43
41
|
- vendor/java/hbase/commons-logging-1.0.4.jar
|
44
42
|
- vendor/java/hbase/commons-logging-api-1.0.4.jar
|
45
43
|
- vendor/java/hbase/hadoop-0.20.1-hdfs127-core.jar
|
@@ -1,119 +0,0 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/exceptions'
|
2
|
-
require File.dirname(__FILE__) + '/column_descriptor'
|
3
|
-
require 'drb'
|
4
|
-
# The name of the java String class conflicts with ruby's String class.
|
5
|
-
module Java
|
6
|
-
include_class "java.lang.String"
|
7
|
-
include_class "java.lang.Exception"
|
8
|
-
|
9
|
-
end
|
10
|
-
|
11
|
-
class String
|
12
|
-
def to_bytes
|
13
|
-
Java::String.new(self).getBytes
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
|
18
|
-
module BigRecordDriver
|
19
|
-
|
20
|
-
class BigRecordServer
|
21
|
-
include_class "java.io.IOException"
|
22
|
-
|
23
|
-
def configure(config = {})
|
24
|
-
raise NotImplementedError
|
25
|
-
end
|
26
|
-
|
27
|
-
def update(table_name, row, values, timestamp=nil)
|
28
|
-
raise NotImplementedError
|
29
|
-
end
|
30
|
-
|
31
|
-
def get(table_name, row, column, options={})
|
32
|
-
raise NotImplementedError
|
33
|
-
end
|
34
|
-
|
35
|
-
def get_columns(table_name, row, columns, options={})
|
36
|
-
raise NotImplementedError
|
37
|
-
end
|
38
|
-
|
39
|
-
def get_consecutive_rows(table_name, start_row, limit, columns, stop_row = nil)
|
40
|
-
raise NotImplementedError
|
41
|
-
end
|
42
|
-
|
43
|
-
def delete(table_name, row)
|
44
|
-
raise NotImplementedError
|
45
|
-
end
|
46
|
-
|
47
|
-
def create_table(table_name, column_descriptors)
|
48
|
-
raise NotImplementedError
|
49
|
-
end
|
50
|
-
|
51
|
-
def drop_table(table_name)
|
52
|
-
raise NotImplementedError
|
53
|
-
end
|
54
|
-
|
55
|
-
def truncate_table(table_name)
|
56
|
-
raise NotImplementedError
|
57
|
-
end
|
58
|
-
|
59
|
-
def ping
|
60
|
-
raise NotImplementedError
|
61
|
-
end
|
62
|
-
|
63
|
-
def table_exists?(table_name)
|
64
|
-
raise NotImplementedError
|
65
|
-
end
|
66
|
-
|
67
|
-
def table_names
|
68
|
-
raise NotImplementedError
|
69
|
-
end
|
70
|
-
|
71
|
-
def method_missing(method, *args)
|
72
|
-
super
|
73
|
-
rescue NoMethodError
|
74
|
-
raise NoMethodError, "undefined method `#{method}' for \"#{self}\":#{self.class}"
|
75
|
-
end
|
76
|
-
|
77
|
-
def respond_to?(method)
|
78
|
-
super
|
79
|
-
end
|
80
|
-
|
81
|
-
protected
|
82
|
-
|
83
|
-
def to_ruby_string(cell)
|
84
|
-
Java::String.new(cell.getValue).to_s
|
85
|
-
end
|
86
|
-
# Try to recover from network related exceptions. e.g. hbase has been restarted and the
|
87
|
-
# cached connections in @tables are no longer valid. Every method in this class (except connect_table)
|
88
|
-
# should have its code wrapped by a call to this method.
|
89
|
-
def safe_exec
|
90
|
-
yield
|
91
|
-
rescue IOException => e
|
92
|
-
puts "A network error occured: #{e.message}. Trying to recover..."
|
93
|
-
init_connection
|
94
|
-
begin
|
95
|
-
yield
|
96
|
-
rescue Exception, Java::Exception => e2
|
97
|
-
if e2.class == e.class
|
98
|
-
puts "Failed to recover the connection."
|
99
|
-
else
|
100
|
-
puts "Failed to recover the connection but got a different error this time: #{e2.message}."
|
101
|
-
end
|
102
|
-
puts "Stack trace:"
|
103
|
-
puts e2.backtrace.join("\n")
|
104
|
-
|
105
|
-
if e2.kind_of?(NativeException)
|
106
|
-
raise BigRecordDriver::JavaError, e2.message
|
107
|
-
else
|
108
|
-
raise e2
|
109
|
-
end
|
110
|
-
end
|
111
|
-
puts "Connection recovered successfully..."
|
112
|
-
rescue Exception => e
|
113
|
-
puts "\n#{e.class.name}: #{e.message}"
|
114
|
-
puts e.backtrace.join("\n")
|
115
|
-
raise e
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
end
|
@@ -1,34 +0,0 @@
|
|
1
|
-
module BigRecordDriver
|
2
|
-
|
3
|
-
class DriverManager
|
4
|
-
class << self
|
5
|
-
|
6
|
-
def set_cmd(db = 'hbase')
|
7
|
-
@@CMD = File.dirname(__FILE__) + "/../../bin/#{db}-driver"
|
8
|
-
end
|
9
|
-
DriverManager.set_cmd
|
10
|
-
def start(port = 40005)
|
11
|
-
`ruby #{@@CMD} start -p #{port.to_s}`
|
12
|
-
end
|
13
|
-
|
14
|
-
def restart(port = 40005)
|
15
|
-
`ruby #{@@CMD} restart -p #{port.to_s}`
|
16
|
-
end
|
17
|
-
|
18
|
-
def stop(port = 40005)
|
19
|
-
`ruby #{@@CMD} stop -p #{port.to_s}`
|
20
|
-
end
|
21
|
-
|
22
|
-
def running?(port = 40005)
|
23
|
-
status = `ruby #{@@CMD} status -p #{port.to_s}`
|
24
|
-
status == "Running.\n"
|
25
|
-
end
|
26
|
-
|
27
|
-
def silent_start(port = 40005)
|
28
|
-
start(port) unless running?(port)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
data/test/test_driver_manager.rb
DELETED
@@ -1,46 +0,0 @@
|
|
1
|
-
$:.unshift(File.dirname(__FILE__) + '/../lib')
|
2
|
-
|
3
|
-
require 'test/unit'
|
4
|
-
require 'big_record_driver'
|
5
|
-
|
6
|
-
class TestDriverManager < Test::Unit::TestCase
|
7
|
-
|
8
|
-
# everything is in a sequence and therefore there's only 1 test
|
9
|
-
def test_all
|
10
|
-
port = 40000
|
11
|
-
|
12
|
-
# stop the driver if it's already running
|
13
|
-
assert_nothing_raised do
|
14
|
-
BigRecordDriver::DriverManager.stop(port) if BigRecordDriver::DriverManager.running?(port)
|
15
|
-
end
|
16
|
-
assert !BigRecordDriver::DriverManager.running?(port), "The driver is already running and it couldn't be stopped"
|
17
|
-
|
18
|
-
# start the real tests
|
19
|
-
assert_nothing_raised do
|
20
|
-
BigRecordDriver::DriverManager.start(port)
|
21
|
-
end
|
22
|
-
assert BigRecordDriver::DriverManager.running?(port), "The driver couldn't be started"
|
23
|
-
|
24
|
-
assert_nothing_raised do
|
25
|
-
BigRecordDriver::DriverManager.restart(port)
|
26
|
-
end
|
27
|
-
assert BigRecordDriver::DriverManager.running?(port), "The driver couldn't be restarted"
|
28
|
-
|
29
|
-
assert_nothing_raised("The driver should be able to do a silent start when it's already running") do
|
30
|
-
BigRecordDriver::DriverManager.silent_start(port)
|
31
|
-
end
|
32
|
-
assert BigRecordDriver::DriverManager.running?(port), "The driver stopped during a silent start instead of staying alive"
|
33
|
-
|
34
|
-
assert_nothing_raised do
|
35
|
-
BigRecordDriver::DriverManager.stop(port)
|
36
|
-
end
|
37
|
-
assert !BigRecordDriver::DriverManager.running?(port), "The driver couldn't be stopped"
|
38
|
-
|
39
|
-
assert_nothing_raised("The driver should be able to do a silent start when it's not running") do
|
40
|
-
BigRecordDriver::DriverManager.silent_start(port)
|
41
|
-
end
|
42
|
-
assert BigRecordDriver::DriverManager.running?(port), "The driver couldn't be started silently"
|
43
|
-
|
44
|
-
end
|
45
|
-
|
46
|
-
end
|