hbase-jruby 0.1.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +623 -0
- data/Rakefile +7 -0
- data/hbase-jruby.gemspec +23 -0
- data/lib/hbase-jruby.rb +16 -0
- data/lib/hbase-jruby/admin.rb +29 -0
- data/lib/hbase-jruby/byte_array.rb +39 -0
- data/lib/hbase-jruby/cell.rb +122 -0
- data/lib/hbase-jruby/column_key.rb +63 -0
- data/lib/hbase-jruby/dependency.rb +69 -0
- data/lib/hbase-jruby/hbase.rb +77 -0
- data/lib/hbase-jruby/pom/cdh3u5.xml +40 -0
- data/lib/hbase-jruby/pom/cdh4.1.2.xml +47 -0
- data/lib/hbase-jruby/result.rb +382 -0
- data/lib/hbase-jruby/scoped.rb +489 -0
- data/lib/hbase-jruby/table.rb +486 -0
- data/lib/hbase-jruby/util.rb +171 -0
- data/lib/hbase-jruby/version.rb +5 -0
- data/test/helper.rb +53 -0
- data/test/test_byte_array.rb +40 -0
- data/test/test_cell.rb +51 -0
- data/test/test_column_key.rb +49 -0
- data/test/test_hbase.rb +36 -0
- data/test/test_scoped.rb +318 -0
- data/test/test_table.rb +211 -0
- data/test/test_table_admin.rb +148 -0
- data/test/test_util.rb +80 -0
- metadata +116 -0
@@ -0,0 +1,486 @@
|
|
1
|
+
require 'bigdecimal'
|
2
|
+
require 'thread'
|
3
|
+
|
4
|
+
class HBase
|
5
|
+
# @!attribute [r] name
|
6
|
+
# @return [String] The name of the table
|
7
|
+
class Table
|
8
|
+
attr_reader :name
|
9
|
+
|
10
|
+
include Enumerable
|
11
|
+
include Admin
|
12
|
+
|
13
|
+
# Returns a read-only org.apache.hadoop.hbase.HTableDescriptor object
|
14
|
+
# @return [org.apache.hadoop.hbase.client.UnmodifyableHTableDescriptor]
|
15
|
+
def descriptor
|
16
|
+
htable.get_table_descriptor
|
17
|
+
end
|
18
|
+
|
19
|
+
# Closes the table and returns HTable object back to the HTablePool.
|
20
|
+
# @return [nil]
|
21
|
+
def close
|
22
|
+
Thread.current[:htable] ||= {}
|
23
|
+
ht = Thread.current[:htable].delete(@name)
|
24
|
+
ht.close if ht
|
25
|
+
nil
|
26
|
+
end
|
27
|
+
|
28
|
+
# Checks if the table of the name exists
|
29
|
+
# @return [true, false] Whether table exists
|
30
|
+
def exists?
|
31
|
+
with_admin { |admin| admin.tableExists @name }
|
32
|
+
end
|
33
|
+
|
34
|
+
# Checks if the table is enabled
|
35
|
+
# @return [true, false] Whether table is enabled
|
36
|
+
def enabled?
|
37
|
+
with_admin { |admin| admin.isTableEnabled(@name) }
|
38
|
+
end
|
39
|
+
|
40
|
+
# Checks if the table is disabled
|
41
|
+
# @return [true, false] Whether table is disabled
|
42
|
+
def disabled?
|
43
|
+
!enabled?
|
44
|
+
end
|
45
|
+
|
46
|
+
# @overload create!(column_family_name, props = {})
|
47
|
+
# Create the table with one column family of the given name
|
48
|
+
# @param [#to_s] The name of the column family
|
49
|
+
# @param [Hash] props Table properties
|
50
|
+
# @return [nil]
|
51
|
+
# @overload create!(column_family_hash, props = {})
|
52
|
+
# Create the table with the specified column families
|
53
|
+
# @param [Hash] Column family properties
|
54
|
+
# @param [Hash] props Table properties
|
55
|
+
# @return [nil]
|
56
|
+
# @example
|
57
|
+
# table.create!(
|
58
|
+
# # Column family with default options
|
59
|
+
# :cf1 => {},
|
60
|
+
# # Another column family with custom properties
|
61
|
+
# :cf2 => {
|
62
|
+
# :blockcache => true,
|
63
|
+
# :blocksize => 128 * 1024,
|
64
|
+
# :bloomfilter => :row,
|
65
|
+
# :compression => :snappy,
|
66
|
+
# :in_memory => true,
|
67
|
+
# :keep_deleted_cells => true,
|
68
|
+
# :min_versions => 2,
|
69
|
+
# :replication_scope => 0,
|
70
|
+
# :ttl => 100,
|
71
|
+
# :versions => 5
|
72
|
+
# }
|
73
|
+
# )
|
74
|
+
# @overload create!(table_descriptor)
|
75
|
+
# Create the table with the given HTableDescriptor
|
76
|
+
# @param [org.apache.hadoop.hbase.HTableDescriptor] Table descriptor
|
77
|
+
# @return [nil]
|
78
|
+
def create! desc, props = {}
|
79
|
+
todo = nil
|
80
|
+
with_admin do |admin|
|
81
|
+
raise RuntimeError, 'Table already exists' if admin.tableExists(@name)
|
82
|
+
|
83
|
+
case desc
|
84
|
+
when HTableDescriptor
|
85
|
+
patch_table_descriptor! desc, props
|
86
|
+
admin.createTable desc
|
87
|
+
when Symbol, String
|
88
|
+
todo = lambda { create!({desc => {}}, props) }
|
89
|
+
when Hash
|
90
|
+
htd = HTableDescriptor.new(@name.to_java_bytes)
|
91
|
+
patch_table_descriptor! htd, props
|
92
|
+
desc.each do |name, opts|
|
93
|
+
htd.addFamily hcd(name, opts)
|
94
|
+
end
|
95
|
+
|
96
|
+
admin.createTable htd
|
97
|
+
else
|
98
|
+
raise ArgumentError, 'Invalid table description'
|
99
|
+
end
|
100
|
+
end
|
101
|
+
todo.call if todo # Avoids mutex relocking
|
102
|
+
end
|
103
|
+
|
104
|
+
# Alters the table
|
105
|
+
# @param [Hash] props Table properties
|
106
|
+
# @return [nil]
|
107
|
+
# @example
|
108
|
+
# table.alter!(
|
109
|
+
# :max_filesize => 512 * 1024 ** 2,
|
110
|
+
# :memstore_flushsize => 64 * 1024 ** 2,
|
111
|
+
# :readonly => false,
|
112
|
+
# :deferred_log_flush => true
|
113
|
+
# )
|
114
|
+
def alter! props
|
115
|
+
with_admin do |admin|
|
116
|
+
htd = admin.get_table_descriptor(@name.to_java_bytes)
|
117
|
+
patch_table_descriptor! htd, props
|
118
|
+
while_disabled(admin) do
|
119
|
+
admin.modifyTable @name.to_java_bytes, htd
|
120
|
+
wait_async_admin(admin)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
# Adds the column family
|
126
|
+
# @param [#to_s] name The name of the column family
|
127
|
+
# @param [Hash] opts Column family properties
|
128
|
+
# @return [nil]
|
129
|
+
def add_family! name, opts
|
130
|
+
with_admin do |admin|
|
131
|
+
while_disabled(admin) do
|
132
|
+
admin.addColumn @name, hcd(name.to_s, opts)
|
133
|
+
wait_async_admin(admin)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
# Alters the column family
|
139
|
+
# @param [#to_s] name The name of the column family
|
140
|
+
# @param [Hash] opts Column family properties
|
141
|
+
# @return [nil]
|
142
|
+
def alter_family! name, opts
|
143
|
+
with_admin do |admin|
|
144
|
+
while_disabled(admin) do
|
145
|
+
admin.modifyColumn @name, hcd(name.to_s, opts)
|
146
|
+
wait_async_admin(admin)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
# Removes the column family
|
152
|
+
# @param [#to_s] name The name of the column family
|
153
|
+
# @return [nil]
|
154
|
+
def delete_family! name
|
155
|
+
with_admin do |admin|
|
156
|
+
while_disabled(admin) do
|
157
|
+
admin.deleteColumn @name, name.to_s
|
158
|
+
wait_async_admin(admin)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
# Enables the table
|
164
|
+
# @return [nil]
|
165
|
+
def enable!
|
166
|
+
with_admin do |admin|
|
167
|
+
admin.enableTable @name unless admin.isTableEnabled(@name)
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
# Disables the table
|
172
|
+
# @return [nil]
|
173
|
+
def disable!
|
174
|
+
with_admin do |admin|
|
175
|
+
admin.disableTable @name if admin.isTableEnabled(@name)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
# Truncates the table by dropping it and recreating it.
|
180
|
+
# @return [nil]
|
181
|
+
def truncate!
|
182
|
+
htd = htable.get_table_descriptor
|
183
|
+
drop!
|
184
|
+
create! htd
|
185
|
+
end
|
186
|
+
|
187
|
+
# Drops the table
|
188
|
+
# @return [nil]
|
189
|
+
def drop!
|
190
|
+
with_admin do |admin|
|
191
|
+
raise RuntimeError, 'Table does not exist' unless admin.tableExists @name
|
192
|
+
|
193
|
+
admin.disableTable @name if admin.isTableEnabled(@name)
|
194
|
+
admin.deleteTable @name
|
195
|
+
close
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
# @overload get(rowkey)
|
200
|
+
# Single GET.
|
201
|
+
# Gets a record with the given rowkey. If the record is not found, nil is returned.
|
202
|
+
# @param [Object] rowkey Rowkey
|
203
|
+
# @return [HBase::Result, nil]
|
204
|
+
# @overload get(rowkeys)
|
205
|
+
# Batch GET. Gets an array of records with the given rowkeys.
|
206
|
+
# Nonexistent records will be returned as nils.
|
207
|
+
# @param [Array<Object>] rowkeys Rowkeys
|
208
|
+
# @return [Array<HBase::Result>]
|
209
|
+
def get rowkeys
|
210
|
+
each.get rowkeys
|
211
|
+
end
|
212
|
+
|
213
|
+
# @overload put(rowkey, data)
|
214
|
+
# Put operation on a rowkey
|
215
|
+
# @param [Object] rowkey Rowkey
|
216
|
+
# @param [Hash] data Data to put
|
217
|
+
# @return [Fixnum] Number of puts succeeded
|
218
|
+
# @overload put(data)
|
219
|
+
# Put operation on multiple rowkeys
|
220
|
+
# @param [Hash<Hash>] data Data to put indexed by rowkeys
|
221
|
+
# @return [Fixnum] Number of puts succeeded
|
222
|
+
def put *args
|
223
|
+
return put(args.first => args.last) if args.length == 2
|
224
|
+
|
225
|
+
puts = args.first.map { |rowkey, props| putify rowkey, props }
|
226
|
+
htable.put puts
|
227
|
+
puts.length
|
228
|
+
end
|
229
|
+
|
230
|
+
# @overload delete(rowkey)
|
231
|
+
# Deletes a row with the given rowkey
|
232
|
+
# @param [Object] rowkey
|
233
|
+
# @return [nil]
|
234
|
+
# @example
|
235
|
+
# table.delete('a000')
|
236
|
+
# @overload delete(rowkey, column_family)
|
237
|
+
# Deletes columns with the given column family from the row
|
238
|
+
# @param [Object] rowkey
|
239
|
+
# @param [String] column_family
|
240
|
+
# @return [nil]
|
241
|
+
# @example
|
242
|
+
# table.delete('a000', 'cf1')
|
243
|
+
# @overload delete(rowkey, column)
|
244
|
+
# Deletes a column
|
245
|
+
# @param [Object] rowkey
|
246
|
+
# @param [String, Array] column Column expression in String "FAMILY:QUALIFIER", or in Array [FAMILY, QUALIFIER]
|
247
|
+
# @return [nil]
|
248
|
+
# @example
|
249
|
+
# table.delete('a000', 'cf1:col1')
|
250
|
+
# @overload delete(rowkey, column, timestamp)
|
251
|
+
# Deletes a version of a column
|
252
|
+
# @param [Object] rowkey
|
253
|
+
# @param [String, Array] column Column expression in String "FAMILY:QUALIFIER", or in Array [FAMILY, QUALIFIER]
|
254
|
+
# @param [Fixnum] timestamp Timestamp.
|
255
|
+
# @return [nil]
|
256
|
+
# @example
|
257
|
+
# table.delete('a000', 'cf1:col1', 1352978648642)
|
258
|
+
# @overload delete(*delete_specs)
|
259
|
+
# Batch deletion
|
260
|
+
# @param [Array<Array>] delete_specs
|
261
|
+
# @return [nil]
|
262
|
+
# @example
|
263
|
+
# table.delete(
|
264
|
+
# ['a000', 'cf1:col1', 1352978648642],
|
265
|
+
# ['a001', 'cf1:col1'],
|
266
|
+
# ['a002', 'cf1'],
|
267
|
+
# ['a003'])
|
268
|
+
def delete *args
|
269
|
+
specs = args.first.is_a?(Array) ? args : [args]
|
270
|
+
|
271
|
+
htable.delete specs.map { |spec|
|
272
|
+
rowkey, cfcq, *ts = spec
|
273
|
+
cf, cq = Util.parse_column_name(cfcq) if cfcq
|
274
|
+
|
275
|
+
Delete.new(Util.to_bytes rowkey).tap { |del|
|
276
|
+
if !ts.empty?
|
277
|
+
ts.each do |t|
|
278
|
+
del.deleteColumn cf, cq, t
|
279
|
+
end
|
280
|
+
elsif cq
|
281
|
+
# Delete all versions
|
282
|
+
del.deleteColumns cf, cq
|
283
|
+
elsif cf
|
284
|
+
del.deleteFamily cf
|
285
|
+
end
|
286
|
+
}
|
287
|
+
}
|
288
|
+
end
|
289
|
+
|
290
|
+
# @overload increment(rowkey, column, by)
|
291
|
+
# Atomically increase column value by the specified amount
|
292
|
+
# @param [Object] rowkey Rowkey
|
293
|
+
# @param [String, Array] column Column expression in String "FAMILY:QUALIFIER", or in Array [FAMILY, QUALIFIER]
|
294
|
+
# @param [Fixnum] by Increment amount
|
295
|
+
# @return [Fixnum] Column value after increment
|
296
|
+
# @example
|
297
|
+
# table.increment('a000', 'cf1:col1', 1)
|
298
|
+
# @overload increment(rowkey, column_by_hash)
|
299
|
+
# Atomically increase values of multiple columns
|
300
|
+
# @param [Object] rowkey Rowkey
|
301
|
+
# @param [Hash] column_by_hash Column expression to increment amount pairs
|
302
|
+
# @example
|
303
|
+
# table.increment('a000', 'cf1:col1' => 1, 'cf1:col2' => 2)
|
304
|
+
def increment rowkey, *args
|
305
|
+
if args.first.is_a?(Hash)
|
306
|
+
cols = args.first
|
307
|
+
htable.increment Increment.new(Util.to_bytes rowkey).tap { |inc|
|
308
|
+
cols.each do |col, by|
|
309
|
+
cf, cq = Util.parse_column_name(col)
|
310
|
+
inc.addColumn cf, cq, by
|
311
|
+
end
|
312
|
+
}
|
313
|
+
else
|
314
|
+
col, by = args
|
315
|
+
cf, cq = Util.parse_column_name(col)
|
316
|
+
htable.incrementColumnValue Util.to_bytes(rowkey), cf, cq, by || 1
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
# Returns the count of the rows in the table
|
321
|
+
# @return [Fixnum]
|
322
|
+
def count
|
323
|
+
each.count
|
324
|
+
end
|
325
|
+
|
326
|
+
# Scan through the table
|
327
|
+
# @yield [HBase::Result] Yields each row in the scope
|
328
|
+
# @return [HBase::Scoped]
|
329
|
+
def each
|
330
|
+
if block_given?
|
331
|
+
Scoped.send(:new, self).each { |r| yield r }
|
332
|
+
else
|
333
|
+
Scoped.send(:new, self)
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
# @see HBase::Scoped#range
|
338
|
+
# @return [HBase::Scoped]
|
339
|
+
def range *key_range
|
340
|
+
each.range(*key_range)
|
341
|
+
end
|
342
|
+
|
343
|
+
# @see HBase::Scoped#project
|
344
|
+
# @return [HBase::Scoped]
|
345
|
+
def project *columns
|
346
|
+
each.project(*columns)
|
347
|
+
end
|
348
|
+
|
349
|
+
# @see HBase::Scoped#filter
|
350
|
+
# @return [HBase::Scoped]
|
351
|
+
def filter *filters
|
352
|
+
each.filter(*filters)
|
353
|
+
end
|
354
|
+
|
355
|
+
# @see HBase::Scoped#limit
|
356
|
+
# @return [HBase::Scoped]
|
357
|
+
def limit rows
|
358
|
+
each.limit rows
|
359
|
+
end
|
360
|
+
|
361
|
+
# @see HBase::Scoped#versions
|
362
|
+
# @return [HBase::Scoped]
|
363
|
+
def versions vs
|
364
|
+
each.versions vs
|
365
|
+
end
|
366
|
+
|
367
|
+
# @see HBase::Scoped#caching
|
368
|
+
# @return [HBase::Scoped]
|
369
|
+
def caching rows
|
370
|
+
each.caching rows
|
371
|
+
end
|
372
|
+
|
373
|
+
# @see HBase::Scoped#batch
|
374
|
+
# @return [HBase::Scoped]
|
375
|
+
def batch b
|
376
|
+
each.batch b
|
377
|
+
end
|
378
|
+
|
379
|
+
# Returns the underlying org.apache.hadoop.hbase.client.HTable object (local to current thread)
|
380
|
+
# @return [org.apache.hadoop.hbase.client.HTable]
|
381
|
+
def htable
|
382
|
+
# @htable ||= @pool.get_table(@name)
|
383
|
+
(local_htables = Thread.current[:htable] ||= {})[@name] ||
|
384
|
+
(local_htables[@name] = @pool.get_table(@name))
|
385
|
+
end
|
386
|
+
|
387
|
+
# Returns table description
|
388
|
+
# @return [String] Table description
|
389
|
+
def inspect
|
390
|
+
if exists?
|
391
|
+
htable.get_table_descriptor.to_s
|
392
|
+
else
|
393
|
+
# FIXME
|
394
|
+
"{NAME => '#{@name}'}"
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
398
|
+
private
|
399
|
+
def initialize config, htable_pool, name
|
400
|
+
@config = config
|
401
|
+
@pool = htable_pool
|
402
|
+
@name = name.to_s
|
403
|
+
@htable = nil
|
404
|
+
end
|
405
|
+
|
406
|
+
def while_disabled admin
|
407
|
+
begin
|
408
|
+
admin.disableTable @name if admin.isTableEnabled(@name)
|
409
|
+
yield
|
410
|
+
ensure
|
411
|
+
admin.enableTable @name
|
412
|
+
end
|
413
|
+
end
|
414
|
+
|
415
|
+
def putify rowkey, props
|
416
|
+
Put.new(Util.to_bytes rowkey).tap { |put|
|
417
|
+
props.each do |col, val|
|
418
|
+
cf, cq = Util.parse_column_name(col)
|
419
|
+
put.add cf, cq, Util.to_bytes(val)
|
420
|
+
end
|
421
|
+
}
|
422
|
+
end
|
423
|
+
|
424
|
+
def hcd name, opts
|
425
|
+
method_map = {
|
426
|
+
:blockcache => :setBlockCacheEnabled,
|
427
|
+
:blocksize => :setBlocksize,
|
428
|
+
:bloomfilter => :setBloomFilterType,
|
429
|
+
:compression => :setCompressionType,
|
430
|
+
:data_block_encoding => :setDataBlockEncoding,
|
431
|
+
:encode_on_disk => :setEncodeOnDisk,
|
432
|
+
:in_memory => :setInMemory,
|
433
|
+
:keep_deleted_cells => :setKeepDeletedCells,
|
434
|
+
:min_versions => :setMinVersions,
|
435
|
+
:replication_scope => :setScope,
|
436
|
+
:ttl => :setTimeToLive,
|
437
|
+
:versions => :setMaxVersions,
|
438
|
+
}
|
439
|
+
HColumnDescriptor.new(name.to_s).tap do |hcd|
|
440
|
+
opts.each do |key, val|
|
441
|
+
if method_map[key]
|
442
|
+
hcd.send method_map[key],
|
443
|
+
({
|
444
|
+
:bloomfilter => proc { |v|
|
445
|
+
const_shortcut StoreFile::BloomType, v, "Invalid bloom filter type"
|
446
|
+
},
|
447
|
+
:compression => proc { |v|
|
448
|
+
const_shortcut Compression::Algorithm, v, "Invalid compression algorithm"
|
449
|
+
}
|
450
|
+
}[key] || proc { |a| a }).call(val)
|
451
|
+
else
|
452
|
+
raise ArgumentError, "Invalid option: #{key}"
|
453
|
+
end
|
454
|
+
end#opts
|
455
|
+
end
|
456
|
+
end
|
457
|
+
|
458
|
+
def const_shortcut base, v, message
|
459
|
+
vs = v.to_s.upcase.to_sym
|
460
|
+
#if base.constants.map { |c| base.const_get c }.include?(v)
|
461
|
+
if base.constants.map { |c| base.const_get c }.any? { |cv| v == cv }
|
462
|
+
v
|
463
|
+
elsif base.constants.include? vs
|
464
|
+
base.const_get vs
|
465
|
+
else
|
466
|
+
raise ArgumentError, [message, v.to_s].join(': ')
|
467
|
+
end
|
468
|
+
end
|
469
|
+
|
470
|
+
def patch_table_descriptor! htd, props
|
471
|
+
props.each do |key, value|
|
472
|
+
method = {
|
473
|
+
:max_filesize => :setMaxFileSize,
|
474
|
+
:readonly => :setReadOnly,
|
475
|
+
:memstore_flushsize => :setMemStoreFlushSize,
|
476
|
+
:deferred_log_flush => :setDeferredLogFlush
|
477
|
+
}[key]
|
478
|
+
raise ArgumentError, "Invalid table property: #{key}" unless method
|
479
|
+
|
480
|
+
htd.send method, value
|
481
|
+
end
|
482
|
+
htd
|
483
|
+
end
|
484
|
+
end#Table
|
485
|
+
end#HBase
|
486
|
+
|