hbase-jruby 0.4.6-java → 0.5.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +31 -14
- data/lib/hbase-jruby/dependency.rb +4 -2
- data/lib/hbase-jruby/hbase.rb +27 -9
- data/lib/hbase-jruby/pom/pom.xml.erb +40 -1
- data/lib/hbase-jruby/version.rb +1 -1
- data/test/test_aggregation.rb +1 -2
- data/test/test_hbase.rb +22 -12
- data/test/test_table_admin.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 38719a45c5f33a6b3ea5841b32325eec7e709ebf
|
4
|
+
data.tar.gz: cc3634b8218d60809d86256819eb8b6657dcad63
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 11aebd9b28bce00bb7130197c9b9b55d59311cecd38d1711fef0703d62833db234a3e224d365df153c63c1fd9d1b638df57185dc988412d6c91c1596b1c476ae
|
7
|
+
data.tar.gz: a375d7bc7e45c61c8c3f2613feacd05e83714b7992dd061f8c13ece2c8587e7f6590b9e18d02252e6aa8c979ae9fb9fdc439f035487feb7e6fb7b66004c8fe1e
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,14 @@
|
|
1
1
|
Changelog
|
2
2
|
=========
|
3
3
|
|
4
|
+
0.5.0
|
5
|
+
-----
|
6
|
+
- Added CDH5.0 (HBase 0.96.1.1) and CDH4.6 dependency profiles
|
7
|
+
- If you share HBase instance between threads, you may need to set
|
8
|
+
`hbase.hconnection.threads.core` due to
|
9
|
+
[a bug](https://issues.apache.org/jira/browse/HBASE-10449) in CDH5.5.0.
|
10
|
+
- Use HConnection instead of deprecated HTablePool when possible
|
11
|
+
|
4
12
|
0.4.6
|
5
13
|
-----
|
6
14
|
- [#29 Fix possible HTable leaks](https://github.com/junegunn/hbase-jruby/issues/29)
|
data/README.md
CHANGED
@@ -152,20 +152,21 @@ or by `require`ing relevant JAR files after launching JRuby.
|
|
152
152
|
Well, there's an easier way.
|
153
153
|
Call `HBase.resolve_dependency!` helper method passing one of the arguments listed below.
|
154
154
|
|
155
|
-
| Argument | Dependency | Default version
|
156
|
-
| ---------- | ------------------------ | ---------------
|
157
|
-
|
|
158
|
-
| cdh4.
|
159
|
-
| cdh4.
|
160
|
-
| cdh4.
|
161
|
-
| cdh4.
|
162
|
-
|
|
163
|
-
|
|
164
|
-
| 0.
|
165
|
-
| 0.
|
166
|
-
| 0.
|
167
|
-
|
|
168
|
-
|
|
155
|
+
| Argument | Dependency | Default version | Required executable |
|
156
|
+
| ---------- | ------------------------ | --------------- | ------------------- |
|
157
|
+
| cdh5.0[.*] | Cloudera CDH5.0 | cdh5.0.0 | mvn |
|
158
|
+
| cdh4.5[.*] | Cloudera CDH4.5 | cdh4.5.0 | mvn |
|
159
|
+
| cdh4.4[.*] | Cloudera CDH4.4 | cdh4.4.0 | mvn |
|
160
|
+
| cdh4.3[.*] | Cloudera CDH4.3 | cdh4.3.2 | mvn |
|
161
|
+
| cdh4.2[.*] | Cloudera CDH4.2 | cdh4.2.2 | mvn |
|
162
|
+
| cdh4.1[.*] | Cloudera CDH4.1 | cdh4.1.5 | mvn |
|
163
|
+
| cdh3[u*] | Cloudera CDH3 | cdh3u6 | mvn |
|
164
|
+
| 0.98[.*] | Apache HBase 0.98 | 0.98.0-hadoop2 | mvn |
|
165
|
+
| 0.96[.*] | Apache HBase 0.96 | 0.96.2-hadoop2 | mvn |
|
166
|
+
| 0.94[.*] | Apache HBase 0.94 | 0.94.18 | mvn |
|
167
|
+
| 0.92[.*] | Apache HBase 0.92 | 0.92.2 | mvn |
|
168
|
+
| *POM PATH* | Custom Maven POM file | - | mvn |
|
169
|
+
| `:local` | Local HBase installation | - | hbase |
|
169
170
|
|
170
171
|
(Default version is used when an argument prefix is given without specific patch version.
|
171
172
|
e.g. `cdh4.2` defaults to `cdh4.2.2`)
|
@@ -1100,6 +1101,22 @@ end
|
|
1100
1101
|
|
1101
1102
|
## Advanced topics
|
1102
1103
|
|
1104
|
+
### Thread-safety
|
1105
|
+
|
1106
|
+
You can freely share a `HBase::Table` instance among threads, as it is backed by
|
1107
|
+
thread-local HTable instances. ([HTable instance in itself is not
|
1108
|
+
thread-safe](https://hbase.apache.org/book/client.html))
|
1109
|
+
|
1110
|
+
```ruby
|
1111
|
+
table = hbase[:my_table]
|
1112
|
+
|
1113
|
+
10.times.map do |i|
|
1114
|
+
Thread.new do
|
1115
|
+
table.put i, data
|
1116
|
+
end
|
1117
|
+
end.each(&:join)
|
1118
|
+
```
|
1119
|
+
|
1103
1120
|
### Lexicographic scan order
|
1104
1121
|
|
1105
1122
|
HBase stores rows in the lexicographic order of the rowkeys in their byte array
|
@@ -12,6 +12,8 @@ class HBase
|
|
12
12
|
# https://ccp.cloudera.com/display/SUPPORT/CDH+Downloads
|
13
13
|
SUPPORTED_PROFILES = {
|
14
14
|
# Prefix => Latest known version
|
15
|
+
'cdh5.0' => 'cdh5.0.0',
|
16
|
+
'cdh4.6' => 'cdh4.6.0',
|
15
17
|
'cdh4.5' => 'cdh4.5.0',
|
16
18
|
'cdh4.4' => 'cdh4.4.0',
|
17
19
|
'cdh4.3' => 'cdh4.3.2',
|
@@ -19,9 +21,9 @@ class HBase
|
|
19
21
|
'cdh4.1' => 'cdh4.1.5',
|
20
22
|
'cdh3' => 'cdh3u6',
|
21
23
|
'0.98' => '0.98.0-hadoop2',
|
22
|
-
'0.96' => '0.96.
|
24
|
+
'0.96' => '0.96.2-hadoop2',
|
23
25
|
'0.95' => '0.95.2-hadoop2',
|
24
|
-
'0.94' => '0.94.
|
26
|
+
'0.94' => '0.94.18',
|
25
27
|
'0.92' => '0.92.2',
|
26
28
|
}
|
27
29
|
|
data/lib/hbase-jruby/hbase.rb
CHANGED
@@ -67,13 +67,26 @@ class HBase
|
|
67
67
|
end
|
68
68
|
end
|
69
69
|
end
|
70
|
-
@
|
70
|
+
@connection = HConnectionManager.createConnection @config
|
71
|
+
@htable_pool =
|
72
|
+
if @connection.respond_to?(:getTable)
|
73
|
+
nil
|
74
|
+
else
|
75
|
+
HTablePool.new @config, java.lang.Integer::MAX_VALUE
|
76
|
+
end
|
71
77
|
@threads = Set.new
|
72
78
|
@mutex = Mutex.new
|
73
79
|
@schema = Schema.new
|
74
80
|
@closed = false
|
75
81
|
end
|
76
82
|
|
83
|
+
# Returns if this instance is backed by an HTablePool which is deprecated
|
84
|
+
# in the recent versions of HBase
|
85
|
+
# @return [Boolean]
|
86
|
+
def use_table_pool?
|
87
|
+
!@htable_pool.nil?
|
88
|
+
end
|
89
|
+
|
77
90
|
# Returns an HBaseAdmin object for administration
|
78
91
|
# @yield [admin] An HBaseAdmin object
|
79
92
|
# @yieldparam [org.apache.hadoop.hbase.client.HBaseAdmin] admin
|
@@ -93,13 +106,17 @@ class HBase
|
|
93
106
|
@mutex.synchronize do
|
94
107
|
unless @closed
|
95
108
|
@closed = true
|
96
|
-
|
109
|
+
@htable_pool.close if use_table_pool?
|
110
|
+
clear_thread_locals
|
111
|
+
@connection.close
|
112
|
+
|
113
|
+
# To be deprecated
|
97
114
|
begin
|
98
115
|
HConnectionManager.deleteConnection(@config)
|
99
116
|
rescue ArgumentError
|
100
117
|
# HBase 0.92 or below
|
101
118
|
HConnectionManager.deleteConnection(@config, true)
|
102
|
-
end
|
119
|
+
end if use_table_pool?
|
103
120
|
end
|
104
121
|
end
|
105
122
|
end
|
@@ -167,10 +184,14 @@ class HBase
|
|
167
184
|
end
|
168
185
|
|
169
186
|
# Reset underlying HTablePool
|
187
|
+
# @deprecated
|
170
188
|
# @return [nil]
|
171
189
|
def reset_table_pool
|
190
|
+
raise RuntimeError, 'Not using table pool' unless use_table_pool?
|
191
|
+
|
172
192
|
@mutex.synchronize do
|
173
|
-
|
193
|
+
clear_thread_locals
|
194
|
+
@htable_pool.close
|
174
195
|
@htable_pool = HTablePool.new @config, java.lang.Integer::MAX_VALUE
|
175
196
|
end
|
176
197
|
nil
|
@@ -196,10 +217,7 @@ private
|
|
196
217
|
end
|
197
218
|
end
|
198
219
|
|
199
|
-
def
|
200
|
-
# Close all the HTable instances in the pool
|
201
|
-
@htable_pool.close
|
202
|
-
|
220
|
+
def clear_thread_locals
|
203
221
|
# Cleanup thread-local references
|
204
222
|
@threads.each do |thr|
|
205
223
|
thr[:hbase_jruby].delete self
|
@@ -207,7 +225,7 @@ private
|
|
207
225
|
end
|
208
226
|
|
209
227
|
def get_htable name
|
210
|
-
@htable_pool.get_table name
|
228
|
+
(@htable_pool || @connection).get_table name
|
211
229
|
end
|
212
230
|
|
213
231
|
def check_closed
|
@@ -10,7 +10,46 @@
|
|
10
10
|
<name>hbase-jruby</name>
|
11
11
|
|
12
12
|
<profiles>
|
13
|
-
<% %w[
|
13
|
+
<% %w[cdh5.0].each do |cdh| %>
|
14
|
+
<profile>
|
15
|
+
<id><%= cdh %></id>
|
16
|
+
<properties>
|
17
|
+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
18
|
+
<hadoop.version>2.3.0-<%= profiles[cdh] %></hadoop.version>
|
19
|
+
<hbase.version>0.96.1.1-<%= profiles[cdh] %></hbase.version>
|
20
|
+
</properties>
|
21
|
+
<repositories>
|
22
|
+
<repository>
|
23
|
+
<id>cloudera-releases</id>
|
24
|
+
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
|
25
|
+
<releases>
|
26
|
+
<enabled>true</enabled>
|
27
|
+
</releases>
|
28
|
+
<snapshots>
|
29
|
+
<enabled>false</enabled>
|
30
|
+
</snapshots>
|
31
|
+
</repository>
|
32
|
+
</repositories>
|
33
|
+
|
34
|
+
<dependencies>
|
35
|
+
<dependency>
|
36
|
+
<groupId>org.apache.hadoop</groupId>
|
37
|
+
<artifactId>hadoop-common</artifactId>
|
38
|
+
<version>${hadoop.version}</version>
|
39
|
+
<scope>compile</scope>
|
40
|
+
</dependency>
|
41
|
+
|
42
|
+
<dependency>
|
43
|
+
<groupId>org.apache.hbase</groupId>
|
44
|
+
<artifactId>hbase-client</artifactId>
|
45
|
+
<version>${hbase.version}</version>
|
46
|
+
<scope>compile</scope>
|
47
|
+
</dependency>
|
48
|
+
</dependencies>
|
49
|
+
</profile>
|
50
|
+
<% end %>
|
51
|
+
|
52
|
+
<% %w[cdh4.6 cdh4.5 cdh4.4 cdh4.3].each do |cdh| %>
|
14
53
|
<profile>
|
15
54
|
<!-- http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/latest/CDH-Version-and-Packaging-Information/cdhvd_topic_8.html -->
|
16
55
|
<id><%= cdh %></id>
|
data/lib/hbase-jruby/version.rb
CHANGED
data/test/test_aggregation.rb
CHANGED
@@ -5,8 +5,7 @@ require 'helper'
|
|
5
5
|
|
6
6
|
class TestAggregation < TestHBaseJRubyBase
|
7
7
|
def test_aggregation
|
8
|
-
|
9
|
-
return unless @aggregation
|
8
|
+
omit "AggregationClient is removed in 0.96" unless @aggregation
|
10
9
|
|
11
10
|
(1..100).each do |idx|
|
12
11
|
@table.put idx, 'cf1:a' => idx, 'cf1:b' => idx * 2
|
data/test/test_hbase.rb
CHANGED
@@ -86,15 +86,22 @@ class TestHBase < TestHBaseJRubyBase
|
|
86
86
|
# Thread-local htable cache has now been created
|
87
87
|
assert Thread.current[:hbase_jruby][hbase2][TABLE]
|
88
88
|
|
89
|
+
sleeping = {}
|
90
|
+
mutex = Mutex.new
|
89
91
|
threads = 4.times.map { |i|
|
90
92
|
Thread.new {
|
91
|
-
|
93
|
+
Thread.current[:htable] = hbase2[TABLE].htable
|
94
|
+
mutex.synchronize { sleeping[Thread.current] = true }
|
92
95
|
sleep
|
93
96
|
}
|
94
97
|
}
|
98
|
+
sleep 0.1 while mutex.synchronize { sleeping.length } < 4
|
95
99
|
threads.each do |t|
|
96
|
-
t
|
100
|
+
assert t[:htable]
|
97
101
|
assert t[:hbase_jruby][hbase2][TABLE]
|
102
|
+
assert_equal t[:htable], t[:hbase_jruby][hbase2][TABLE]
|
103
|
+
|
104
|
+
t.kill
|
98
105
|
end
|
99
106
|
|
100
107
|
# Now close the connection
|
@@ -113,20 +120,23 @@ class TestHBase < TestHBaseJRubyBase
|
|
113
120
|
|
114
121
|
def test_reset_pool
|
115
122
|
hbase2 = HBase.new @hbase.config
|
116
|
-
|
123
|
+
omit_unless(hbase2.use_table_pool?) do
|
124
|
+
table = hbase2[TABLE]
|
117
125
|
|
118
|
-
|
119
|
-
|
120
|
-
|
126
|
+
htable = table.htable
|
127
|
+
assert_equal htable, table.htable
|
128
|
+
assert_equal htable, hbase2[TABLE.to_sym].htable
|
121
129
|
|
122
|
-
|
130
|
+
assert_nil hbase2.reset_table_pool
|
123
131
|
|
124
|
-
|
125
|
-
|
132
|
+
assert_not_equal htable, table.htable
|
133
|
+
assert_not_equal htable, hbase2[TABLE].htable
|
126
134
|
|
127
|
-
|
128
|
-
|
129
|
-
|
135
|
+
htable = table.htable
|
136
|
+
assert_equal htable, table.htable
|
137
|
+
assert_equal htable, hbase2[TABLE.to_sym].htable
|
138
|
+
end
|
139
|
+
hbase2.close
|
130
140
|
end
|
131
141
|
end
|
132
142
|
|
data/test/test_table_admin.rb
CHANGED
@@ -143,7 +143,7 @@ class TestTableAdmin < TestHBaseJRubyBase
|
|
143
143
|
end
|
144
144
|
|
145
145
|
def test_add_coprocessor!
|
146
|
-
|
146
|
+
omit "AggregationClient is removed in 0.96" unless @aggregation
|
147
147
|
|
148
148
|
coproc = 'org.apache.hadoop.hbase.coprocessor.AggregateImplementation'
|
149
149
|
assert_false @table.has_coprocessor? coproc
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hbase-jruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Junegunn Choi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -102,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
102
102
|
version: '0'
|
103
103
|
requirements: []
|
104
104
|
rubyforge_project:
|
105
|
-
rubygems_version: 2.2.
|
105
|
+
rubygems_version: 2.2.2
|
106
106
|
signing_key:
|
107
107
|
specification_version: 4
|
108
108
|
summary: A JRuby binding for HBase
|