hbase-jruby 0.6.2-java → 0.6.3-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -0
- data/README.md +41 -26
- data/lib/hbase-jruby/dependency.rb +3 -2
- data/lib/hbase-jruby/pom/pom.xml.erb +4 -2
- data/lib/hbase-jruby/row.rb +41 -44
- data/lib/hbase-jruby/table/admin.rb +16 -4
- data/lib/hbase-jruby/version.rb +1 -1
- data/test/helper.rb +3 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0654a082ed7102051b33a0d27f9b317870f59eb2
|
|
4
|
+
data.tar.gz: a050c546803000e36f4daadc511beff5c3b24b51
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a8bb69c1d2909ad7ddff0062087eedf78552dbe75fbfcb9d95f5ad4e3c0e6a28b77b1a0529e4426aed8fab6ac267568a7d085938933419bd1354b8dff5285871
|
|
7
|
+
data.tar.gz: b0b872f8559795f34b2b546f55e7386f3763e7b2a14d7de61a7909a8ed3de68e7f606534f37ae156bb3f823fbca482c29dd13412cfa28a9cfb62b0caa4c63762
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,28 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
+
0.6.3
|
|
5
|
+
-----
|
|
6
|
+
|
|
7
|
+
- Refactoring
|
|
8
|
+
- Changed `HBase::Row#to_h` call to return a Hash extended with
|
|
9
|
+
`HBase::Row::HashExtension` module which makes it possible to extend the
|
|
10
|
+
returned Hash by monkey-patching the module
|
|
11
|
+
- `HBase::Row::HashExtension` overrides `#[]` method to allow looking up data
|
|
12
|
+
with both Array and String notation of column keys: `[CF, CQ]` or `CF:CQ`
|
|
13
|
+
- Hash for describing column and table options can now contain XML entries
|
|
14
|
+
using additional `:config` key.
|
|
15
|
+
|
|
16
|
+
```ruby
|
|
17
|
+
hbase[:my_table].create! :d,
|
|
18
|
+
# HTableDescriptor#setValue
|
|
19
|
+
'MAX_FILESIZE' => 1 << 33,
|
|
20
|
+
:config => {
|
|
21
|
+
# HTableDescriptor#setConfiguration
|
|
22
|
+
'hbase.hstore.compaction.max.size' => 1 << 30
|
|
23
|
+
}
|
|
24
|
+
```
|
|
25
|
+
|
|
4
26
|
0.6.2
|
|
5
27
|
-----
|
|
6
28
|
|
data/README.md
CHANGED
|
@@ -47,7 +47,7 @@ hbase[:my_table].get(100).double('f:c') # Returns 3.14
|
|
|
47
47
|
require 'hbase-jruby'
|
|
48
48
|
|
|
49
49
|
# Load required JAR files from CDH distribution using Maven
|
|
50
|
-
HBase.resolve_dependency! '
|
|
50
|
+
HBase.resolve_dependency! 'cdh5.2.0'
|
|
51
51
|
|
|
52
52
|
# Connect to HBase on localhost
|
|
53
53
|
hbase = HBase.new
|
|
@@ -143,31 +143,44 @@ table.delete 1
|
|
|
143
143
|
|
|
144
144
|
### Resolving Hadoop/HBase dependency
|
|
145
145
|
|
|
146
|
-
To be able to access HBase from JRuby, Hadoop/HBase dependency must be
|
|
147
|
-
This can be done by either setting up CLASSPATH variable beforehand
|
|
146
|
+
To be able to access HBase from JRuby, Hadoop/HBase dependency must be
|
|
147
|
+
satisfied. This can be done by either setting up CLASSPATH variable beforehand
|
|
148
148
|
or by `require`ing relevant JAR files after launching JRuby.
|
|
149
149
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
150
|
+
You might be able to find the right uberjar required for using HBase client
|
|
151
|
+
API from [hbase-client-dep releases page](https://github.com/junegunn/hbase-client-dep/releases).
|
|
152
|
+
(If you don't find one, send me a pull request.)
|
|
153
|
+
|
|
154
|
+
```ruby
|
|
155
|
+
require 'hbase-jruby'
|
|
156
|
+
require 'hbase-client-dep-cdh5.2.jar'
|
|
157
|
+
|
|
158
|
+
HBase.log4j = { 'log4j.threshold' => 'ERROR' }
|
|
159
|
+
hbase = HBase.new
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### `HBase.resolve_dependency!` (*to be deprecated!*)
|
|
163
|
+
|
|
164
|
+
hbase-jruby is shipped with `HBase.resolve_dependency!` helper method that can
|
|
165
|
+
be used to load JAR files from Maven repository.
|
|
166
|
+
|
|
167
|
+
| Argument | Dependency | Default version | Required executable |
|
|
168
|
+
| ---------- | ------------------------ | --------------- | ------------------- |
|
|
169
|
+
| cdh5.2[.*] | Cloudera CDH5.2 | cdh5.2.1 | mvn |
|
|
170
|
+
| cdh5.1[.*] | Cloudera CDH5.1 | cdh5.1.4 | mvn |
|
|
171
|
+
| cdh5.0[.*] | Cloudera CDH5.0 | cdh5.0.5 | mvn |
|
|
172
|
+
| cdh4.5[.*] | Cloudera CDH4.5 | cdh4.5.0 | mvn |
|
|
173
|
+
| cdh4.4[.*] | Cloudera CDH4.4 | cdh4.4.0 | mvn |
|
|
174
|
+
| cdh4.3[.*] | Cloudera CDH4.3 | cdh4.3.2 | mvn |
|
|
175
|
+
| cdh4.2[.*] | Cloudera CDH4.2 | cdh4.2.2 | mvn |
|
|
176
|
+
| cdh4.1[.*] | Cloudera CDH4.1 | cdh4.1.5 | mvn |
|
|
177
|
+
| cdh3[u*] | Cloudera CDH3 | cdh3u6 | mvn |
|
|
178
|
+
| 0.98[.*] | Apache HBase 0.98 | 0.98.0-hadoop2 | mvn |
|
|
179
|
+
| 0.96[.*] | Apache HBase 0.96 | 0.96.2-hadoop2 | mvn |
|
|
180
|
+
| 0.94[.*] | Apache HBase 0.94 | 0.94.18 | mvn |
|
|
181
|
+
| 0.92[.*] | Apache HBase 0.92 | 0.92.2 | mvn |
|
|
182
|
+
| *POM PATH* | Custom Maven POM file | - | mvn |
|
|
183
|
+
| `:local` | Local HBase installation | - | hbase |
|
|
171
184
|
|
|
172
185
|
(Default version is used when an argument prefix is given without specific patch version.
|
|
173
186
|
e.g. `cdh4.2` defaults to `cdh4.2.2`)
|
|
@@ -994,7 +1007,7 @@ http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HColumnDescriptor.html
|
|
|
994
1007
|
Some of the properties are only available on recent versions of HBase.
|
|
995
1008
|
|
|
996
1009
|
| Property | Type | Description |
|
|
997
|
-
|
|
1010
|
+
| ------------------------ | ------------- | ------------------------------------------------------------------------------------------------------------------ |
|
|
998
1011
|
| `:blockcache` | Boolean | If MapFile blocks should be cached |
|
|
999
1012
|
| `:blocksize` | Fixnum | Blocksize to use when writing out storefiles/hfiles on this column family |
|
|
1000
1013
|
| `:bloomfilter` | Symbol/String | Bloom filter type: `:none`, `:row`, `:rowcol`, or uppercase Strings |
|
|
@@ -1012,6 +1025,7 @@ Some of the properties are only available on recent versions of HBase.
|
|
|
1012
1025
|
| `:replication_scope` | Fixnum | Replication scope |
|
|
1013
1026
|
| `:ttl` | Fixnum | Time-to-live of cell contents, in seconds |
|
|
1014
1027
|
| `:versions` | Fixnum | The maximum number of versions. (By default, all available versions are retrieved.) |
|
|
1028
|
+
| `:config` | Hash | Additional XML configuration |
|
|
1015
1029
|
|
|
1016
1030
|
#### List of table properties
|
|
1017
1031
|
|
|
@@ -1026,6 +1040,7 @@ http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HTableDescriptor.html
|
|
|
1026
1040
|
| `:durability` | Symbol/String | Durability setting of the table |
|
|
1027
1041
|
| `:split_policy` | String/Class | Region split policy |
|
|
1028
1042
|
| `:splits` | Array | Region split points |
|
|
1043
|
+
| `:config` | Hash | Additional XML configuration |
|
|
1029
1044
|
|
|
1030
1045
|
### Managing column families
|
|
1031
1046
|
|
|
@@ -1238,7 +1253,7 @@ ba.java # Returns the native Java byte array (byte[])
|
|
|
1238
1253
|
|
|
1239
1254
|
## API documentation
|
|
1240
1255
|
|
|
1241
|
-
[http://rubydoc.info/gems/hbase-jruby/
|
|
1256
|
+
[http://www.rubydoc.info/gems/hbase-jruby/](http://www.rubydoc.info/gems/hbase-jruby/)
|
|
1242
1257
|
|
|
1243
1258
|
## Test
|
|
1244
1259
|
|
|
@@ -12,8 +12,9 @@ class HBase
|
|
|
12
12
|
# https://ccp.cloudera.com/display/SUPPORT/CDH+Downloads
|
|
13
13
|
SUPPORTED_PROFILES = {
|
|
14
14
|
# Prefix => Latest known version
|
|
15
|
-
'cdh5.
|
|
16
|
-
'cdh5.
|
|
15
|
+
'cdh5.2' => 'cdh5.2.1',
|
|
16
|
+
'cdh5.1' => 'cdh5.1.4',
|
|
17
|
+
'cdh5.0' => 'cdh5.0.5',
|
|
17
18
|
'cdh4.6' => 'cdh4.6.0',
|
|
18
19
|
'cdh4.5' => 'cdh4.5.0',
|
|
19
20
|
'cdh4.4' => 'cdh4.4.0',
|
|
@@ -10,12 +10,14 @@
|
|
|
10
10
|
<name>hbase-jruby</name>
|
|
11
11
|
|
|
12
12
|
<profiles>
|
|
13
|
-
<% { 'cdh5.0' => '
|
|
13
|
+
<% { 'cdh5.0' => ['2.3.0', '0.96.1.1'],
|
|
14
|
+
'cdh5.1' => ['2.3.0', '0.98.1' ],
|
|
15
|
+
'cdh5.2' => ['2.5.0', '0.98.6' ]}.each do |cdh, (hadoop_version, hbase_version)| %>
|
|
14
16
|
<profile>
|
|
15
17
|
<id><%= cdh %></id>
|
|
16
18
|
<properties>
|
|
17
19
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
18
|
-
<hadoop.version
|
|
20
|
+
<hadoop.version><%= hadoop_version %>-<%= profiles[cdh] %></hadoop.version>
|
|
19
21
|
<hbase.version><%= hbase_version %>-<%= profiles[cdh] %></hbase.version>
|
|
20
22
|
</properties>
|
|
21
23
|
<repositories>
|
data/lib/hbase-jruby/row.rb
CHANGED
|
@@ -38,45 +38,58 @@ class Row
|
|
|
38
38
|
end
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
module HashExtension
|
|
42
|
+
def [] key
|
|
43
|
+
# %w[cf x]
|
|
44
|
+
if key.is_a?(Array) && key.length == 2
|
|
45
|
+
key = [key[0].to_sym, ByteArray[key[1]]]
|
|
46
|
+
# %[cf:x]
|
|
47
|
+
elsif key.is_a?(String) && key.index(':')
|
|
48
|
+
cf, cq = key.split(':', 2)
|
|
49
|
+
key = [cf.to_sym, ByteArray[cq]]
|
|
50
|
+
end
|
|
51
|
+
super key
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
42
55
|
# @return [Hash]
|
|
43
56
|
def to_h
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
end
|
|
57
|
+
ret = {}
|
|
58
|
+
@result.getNoVersionMap.each do |cf, cqmap|
|
|
59
|
+
cf = Util.from_bytes :string, cf
|
|
60
|
+
cqmap.each do |cq, val|
|
|
61
|
+
cqs = Util.from_bytes(:string, cq) rescue nil
|
|
62
|
+
f, q, t = @table.lookup_schema(cqs)
|
|
63
|
+
t = nil if f != cf
|
|
64
|
+
name = t ? q : [cf.to_sym, ByteArray[cq]]
|
|
65
|
+
|
|
66
|
+
ret[name] = Util.from_bytes(t, val)
|
|
55
67
|
end
|
|
56
68
|
end
|
|
69
|
+
ret.extend(HashExtension)
|
|
57
70
|
end
|
|
58
71
|
alias to_hash to_h
|
|
59
72
|
|
|
60
73
|
# @return [Hash]
|
|
61
74
|
def to_H
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
end
|
|
75
|
+
ret = {}
|
|
76
|
+
@result.getMap.each do |cf, cqmap|
|
|
77
|
+
cf = Util.from_bytes :string, cf
|
|
78
|
+
cqmap.each do |cq, tsmap|
|
|
79
|
+
cqs = Util.from_bytes(:string, cq) rescue nil
|
|
80
|
+
f, q, t = @table.lookup_schema(cqs)
|
|
81
|
+
t = nil if f != cf
|
|
82
|
+
name = t ? q : [cf.to_sym, ByteArray[cq]]
|
|
83
|
+
|
|
84
|
+
ret[name] =
|
|
85
|
+
Hash[
|
|
86
|
+
tsmap.map { |ts, val|
|
|
87
|
+
[ ts, Util.from_bytes(t, val) ]
|
|
88
|
+
}
|
|
89
|
+
]
|
|
78
90
|
end
|
|
79
91
|
end
|
|
92
|
+
ret.extend(HashExtension)
|
|
80
93
|
end
|
|
81
94
|
alias to_hash_with_versions to_H
|
|
82
95
|
|
|
@@ -273,22 +286,6 @@ private
|
|
|
273
286
|
end
|
|
274
287
|
end
|
|
275
288
|
|
|
276
|
-
HASH_TEMPLATE = {}.tap { |h|
|
|
277
|
-
h.instance_eval do
|
|
278
|
-
def [] key
|
|
279
|
-
# %w[cf x]
|
|
280
|
-
if key.is_a?(Array) && key.length == 2
|
|
281
|
-
key = [key[0].to_sym, ByteArray[key[1]]]
|
|
282
|
-
# %[cf:x]
|
|
283
|
-
elsif key.is_a?(String) && key.index(':')
|
|
284
|
-
cf, cq = key.split(':', 2)
|
|
285
|
-
key = [cf.to_sym, ByteArray[cq]]
|
|
286
|
-
end
|
|
287
|
-
super key
|
|
288
|
-
end
|
|
289
|
-
end
|
|
290
|
-
}
|
|
291
|
-
|
|
292
289
|
# @param [HBase::Table] table
|
|
293
290
|
# @param [org.apache.hadoop.hbase.client.Result] java_result
|
|
294
291
|
def initialize table, java_result
|
|
@@ -44,7 +44,12 @@ class Table
|
|
|
44
44
|
# :min_versions => 2,
|
|
45
45
|
# :replication_scope => 0,
|
|
46
46
|
# :ttl => 100,
|
|
47
|
-
# :versions => 5
|
|
47
|
+
# :versions => 5,
|
|
48
|
+
#
|
|
49
|
+
# # XML config can be specified in :config Hash
|
|
50
|
+
# :config => {
|
|
51
|
+
# 'hbase.hstore.compaction.max.size' => 1 << 30,
|
|
52
|
+
# }
|
|
48
53
|
# }
|
|
49
54
|
# )
|
|
50
55
|
# @overload create!(table_descriptor)
|
|
@@ -320,8 +325,11 @@ private
|
|
|
320
325
|
def hcd name, opts
|
|
321
326
|
HColumnDescriptor.new(name.to_s).tap do |hcd|
|
|
322
327
|
opts.each do |key, val|
|
|
323
|
-
|
|
324
|
-
|
|
328
|
+
if key == :config
|
|
329
|
+
val.each do |k, v|
|
|
330
|
+
hcd.setConfiguration k, v.to_s
|
|
331
|
+
end
|
|
332
|
+
elsif method = COLUMN_PROPERTIES[key] && COLUMN_PROPERTIES[key][:set]
|
|
325
333
|
hcd.send method,
|
|
326
334
|
({
|
|
327
335
|
:bloomfilter => proc { |v|
|
|
@@ -374,7 +382,11 @@ private
|
|
|
374
382
|
props.each do |key, value|
|
|
375
383
|
next if key == :splits
|
|
376
384
|
|
|
377
|
-
if
|
|
385
|
+
if key == :config
|
|
386
|
+
value.each do |k, v|
|
|
387
|
+
htd.setConfiguration k, v.to_s
|
|
388
|
+
end
|
|
389
|
+
elsif method = TABLE_PROPERTIES[key] && TABLE_PROPERTIES[key][:set]
|
|
378
390
|
if method.is_a? Symbol
|
|
379
391
|
htd.send method, value
|
|
380
392
|
else
|
data/lib/hbase-jruby/version.rb
CHANGED
data/test/helper.rb
CHANGED
|
@@ -15,7 +15,9 @@ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
|
|
|
15
15
|
$rowkey = Time.now.to_i
|
|
16
16
|
|
|
17
17
|
require "hbase-jruby"
|
|
18
|
-
if
|
|
18
|
+
if jar = ENV['HBASE_JRUBY_TEST_JAR']
|
|
19
|
+
require jar
|
|
20
|
+
elsif dist = ENV['HBASE_JRUBY_TEST_DIST']
|
|
19
21
|
HBase.resolve_dependency!(dist, :verbose => true)
|
|
20
22
|
end
|
|
21
23
|
HBase.log4j = { 'log4j.threshold' => 'ERROR' }
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: hbase-jruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.6.
|
|
4
|
+
version: 0.6.3
|
|
5
5
|
platform: java
|
|
6
6
|
authors:
|
|
7
7
|
- Junegunn Choi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2014-
|
|
11
|
+
date: 2014-12-05 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: test-unit
|