sequel-hive-adapter 0.2.4 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -1
- data/Gemfile.lock +3 -3
- data/README.rdoc +4 -2
- data/VERSION +1 -1
- data/lib/sequel/adapters/hive.rb +15 -39
- data/sequel-hive-adapter.gemspec +9 -9
- data/spec/sequel-hive-adapter_spec.rb +32 -4
- metadata +6 -6
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -8,7 +8,7 @@ GEM
|
|
8
8
|
git (>= 1.2.5)
|
9
9
|
rake
|
10
10
|
rake (0.8.7)
|
11
|
-
rbhive (0.
|
11
|
+
rbhive (0.2.5)
|
12
12
|
thrift (>= 0.4.0)
|
13
13
|
rcov (0.9.9)
|
14
14
|
redgreen (1.2.2)
|
@@ -21,7 +21,7 @@ GEM
|
|
21
21
|
diff-lcs (~> 1.1.2)
|
22
22
|
rspec-mocks (2.3.0)
|
23
23
|
sequel (3.23.0)
|
24
|
-
thrift (0.
|
24
|
+
thrift (0.7.0)
|
25
25
|
yard (0.6.8)
|
26
26
|
|
27
27
|
PLATFORMS
|
@@ -30,7 +30,7 @@ PLATFORMS
|
|
30
30
|
DEPENDENCIES
|
31
31
|
bundler (~> 1.0.0)
|
32
32
|
jeweler (~> 1.6.0)
|
33
|
-
rbhive
|
33
|
+
rbhive (~> 0.2.2)
|
34
34
|
rcov
|
35
35
|
redgreen
|
36
36
|
rspec (~> 2.3.0)
|
data/README.rdoc
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
= sequel-hive-adapter
|
2
2
|
|
3
|
-
A Hadoop Hive adapter for Sequel. Uses rbhive and thrift.
|
3
|
+
A Hadoop Hive adapter for Sequel. Uses rbhive[https://github.com/forward/rbhive] and thrift[http://thrift.apache.org/].
|
4
4
|
|
5
5
|
= Installation
|
6
6
|
|
7
7
|
gem install sequel-hive-adapter
|
8
|
+
|
9
|
+
RubyGems.org: http://rubygems.org/gems/sequel-hive-adapter
|
8
10
|
|
9
11
|
= Usage
|
10
12
|
|
@@ -20,7 +22,7 @@ Also from the command-line:
|
|
20
22
|
Your database is stored in DB...
|
21
23
|
> DB.tables
|
22
24
|
Executing Hive Query: SHOW TABLES
|
23
|
-
=> [[
|
25
|
+
=> [[:table1,:table2,...]]
|
24
26
|
|
25
27
|
== Contributing to sequel-hive-adapter
|
26
28
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.3.0
|
data/lib/sequel/adapters/hive.rb
CHANGED
@@ -26,15 +26,23 @@ module Sequel
|
|
26
26
|
end
|
27
27
|
alias_method :do, :execute
|
28
28
|
|
29
|
+
#
|
30
|
+
# Returns the schema for the given table as an array with all members being arrays of length 2,
|
31
|
+
# the first member being the column name, and the second member being a hash of column
|
32
|
+
# information.
|
33
|
+
#
|
29
34
|
def schema(table, opts={})
|
30
35
|
hero = execute("DESCRIBE #{table}")
|
31
|
-
hero
|
32
|
-
[h.
|
33
|
-
end
|
36
|
+
hero.map do |h|
|
37
|
+
[ h[:col_name].to_sym, { :db_type => h[:data_type] , :comment => h[:comment] } ]
|
38
|
+
end
|
34
39
|
end
|
35
40
|
|
41
|
+
#
|
42
|
+
# Returns a list of tables as symbols.
|
43
|
+
#
|
36
44
|
def tables(opts={})
|
37
|
-
execute('SHOW TABLES')
|
45
|
+
execute('SHOW TABLES').map{|i| i.values}.reduce(:+).map{|i| i.to_sym}
|
38
46
|
end
|
39
47
|
|
40
48
|
private
|
@@ -47,46 +55,18 @@ module Sequel
|
|
47
55
|
class Dataset < Sequel::Dataset
|
48
56
|
SELECT_CLAUSE_METHODS = clause_methods(:select, %w'distinct columns from join where group having compounds order limit')
|
49
57
|
|
50
|
-
#TODO better type conversion
|
51
|
-
CONVERT_FROM = { :boolean => :to_s, :string => :to_s, :bigint => :to_i, :float => :to_f, :double => :to_f, :int => :to_i, :smallint => :to_i, :tinyint => :to_i }
|
52
|
-
|
53
58
|
def schema
|
54
59
|
@schema ||= @db.schema(@opts[:from].first)
|
55
60
|
end
|
56
61
|
|
57
62
|
def columns
|
58
|
-
|
59
|
-
if needs_schema_check?
|
60
|
-
@columns = schema.map{|c| c.first.to_sym}
|
61
|
-
else
|
62
|
-
@columns = @opts[:select].map do |col|
|
63
|
-
col.respond_to?(:aliaz) ? col.aliaz : col
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
# Returns the function symbol that converts a column to the correct datatype
|
69
|
-
def convert_type(column)
|
70
|
-
return :to_s unless needs_schema_check?
|
71
|
-
db_type = schema.select do |a|
|
72
|
-
a.first == column
|
73
|
-
end.flatten!
|
74
|
-
CONVERT_FROM[db_type.last[:db_type]]
|
63
|
+
@columns ||= schema.map{|c| c.first.to_sym}
|
75
64
|
end
|
76
65
|
|
77
66
|
def fetch_rows(sql)
|
78
67
|
execute(sql) do |result|
|
79
|
-
|
80
|
-
|
81
|
-
result.each do |r|
|
82
|
-
row = {}
|
83
|
-
r.each_with_index do |v, i|
|
84
|
-
row[columns[i]] = v.send(convert_type(columns[i]))
|
85
|
-
end
|
86
|
-
yield row
|
87
|
-
end
|
88
|
-
ensure
|
89
|
-
# result.close
|
68
|
+
result.each do |r|
|
69
|
+
yield r
|
90
70
|
end
|
91
71
|
end
|
92
72
|
self
|
@@ -94,10 +74,6 @@ module Sequel
|
|
94
74
|
|
95
75
|
private
|
96
76
|
|
97
|
-
def needs_schema_check?
|
98
|
-
@opts[:select].nil? || @opts[:select].include?(:*)
|
99
|
-
end
|
100
|
-
|
101
77
|
def select_clause_methods
|
102
78
|
SELECT_CLAUSE_METHODS
|
103
79
|
end
|
data/sequel-hive-adapter.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{sequel-hive-adapter}
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.3.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
-
s.authors = [
|
12
|
-
s.date = %q{2011-
|
11
|
+
s.authors = [%q{hrp}]
|
12
|
+
s.date = %q{2011-08-20}
|
13
13
|
s.description = %q{A Hadoop Hive adapter for Sequel. Uses RBHive and Thrift.}
|
14
14
|
s.email = %q{hrparmar@gmail.com}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -31,9 +31,9 @@ Gem::Specification.new do |s|
|
|
31
31
|
"spec/spec_helper.rb"
|
32
32
|
]
|
33
33
|
s.homepage = %q{http://github.com/hrp/sequel-hive-adapter}
|
34
|
-
s.licenses = [
|
35
|
-
s.require_paths = [
|
36
|
-
s.rubygems_version = %q{1.
|
34
|
+
s.licenses = [%q{MIT}]
|
35
|
+
s.require_paths = [%q{lib}]
|
36
|
+
s.rubygems_version = %q{1.8.8}
|
37
37
|
s.summary = %q{Hive adapter for Sequel.}
|
38
38
|
|
39
39
|
if s.respond_to? :specification_version then
|
@@ -41,7 +41,7 @@ Gem::Specification.new do |s|
|
|
41
41
|
|
42
42
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
43
43
|
s.add_runtime_dependency(%q<sequel>, [">= 0"])
|
44
|
-
s.add_runtime_dependency(%q<rbhive>, ["
|
44
|
+
s.add_runtime_dependency(%q<rbhive>, ["~> 0.2.2"])
|
45
45
|
s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
|
46
46
|
s.add_development_dependency(%q<yard>, ["~> 0.6.0"])
|
47
47
|
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
@@ -50,7 +50,7 @@ Gem::Specification.new do |s|
|
|
50
50
|
s.add_development_dependency(%q<redgreen>, [">= 0"])
|
51
51
|
else
|
52
52
|
s.add_dependency(%q<sequel>, [">= 0"])
|
53
|
-
s.add_dependency(%q<rbhive>, ["
|
53
|
+
s.add_dependency(%q<rbhive>, ["~> 0.2.2"])
|
54
54
|
s.add_dependency(%q<rspec>, ["~> 2.3.0"])
|
55
55
|
s.add_dependency(%q<yard>, ["~> 0.6.0"])
|
56
56
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
@@ -60,7 +60,7 @@ Gem::Specification.new do |s|
|
|
60
60
|
end
|
61
61
|
else
|
62
62
|
s.add_dependency(%q<sequel>, [">= 0"])
|
63
|
-
s.add_dependency(%q<rbhive>, ["
|
63
|
+
s.add_dependency(%q<rbhive>, ["~> 0.2.2"])
|
64
64
|
s.add_dependency(%q<rspec>, ["~> 2.3.0"])
|
65
65
|
s.add_dependency(%q<yard>, ["~> 0.6.0"])
|
66
66
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
@@ -3,18 +3,46 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
|
3
3
|
describe "SequelHiveAdapter" do
|
4
4
|
|
5
5
|
before(:all) do
|
6
|
-
TEST_HOST = "
|
6
|
+
TEST_HOST = "172.16.87.131" # CDH3 Demo VM
|
7
7
|
@testdb = Sequel.connect("hive://#{TEST_HOST}")
|
8
|
-
@testtable =
|
8
|
+
@testtable = :test_test_test
|
9
|
+
@sampletable = :pokes
|
10
|
+
@sampledb = @testdb[ @sampletable ]
|
9
11
|
end
|
10
12
|
|
11
13
|
it "should load" do
|
12
14
|
@testdb.should be_an_instance_of Sequel::Hive::Database
|
13
15
|
end
|
14
16
|
|
15
|
-
|
17
|
+
pending "should create and drop tables" do
|
18
|
+
@testdb.create_table @testtable do
|
19
|
+
column :name, :string
|
20
|
+
column :num, :integer
|
21
|
+
end
|
16
22
|
end
|
17
23
|
|
18
|
-
it "should
|
24
|
+
it "should list tables as symbols" do
|
25
|
+
res = @testdb.tables
|
26
|
+
res.each do |r|
|
27
|
+
r.should be_a Symbol
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should list table schemas" do
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should list table columns" do
|
35
|
+
@sampledb.columns.should == [:foo, :bar]
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should return the value for one column" do
|
39
|
+
@sampledb.select(:foo).first.should == {:foo => 5}
|
40
|
+
end
|
41
|
+
|
42
|
+
|
43
|
+
pending "should convert string columns to strings" do
|
44
|
+
end
|
45
|
+
|
46
|
+
pending "should handle group and count" do
|
19
47
|
end
|
20
48
|
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: sequel-hive-adapter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.
|
5
|
+
version: 0.3.0
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- hrp
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2011-
|
13
|
+
date: 2011-08-20 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: sequel
|
@@ -28,9 +28,9 @@ dependencies:
|
|
28
28
|
requirement: &id002 !ruby/object:Gem::Requirement
|
29
29
|
none: false
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ~>
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 0.2.2
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: *id002
|
@@ -135,7 +135,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
135
135
|
requirements:
|
136
136
|
- - ">="
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
hash: -
|
138
|
+
hash: -3353089072672527350
|
139
139
|
segments:
|
140
140
|
- 0
|
141
141
|
version: "0"
|
@@ -148,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
148
148
|
requirements: []
|
149
149
|
|
150
150
|
rubyforge_project:
|
151
|
-
rubygems_version: 1.
|
151
|
+
rubygems_version: 1.8.8
|
152
152
|
signing_key:
|
153
153
|
specification_version: 3
|
154
154
|
summary: Hive adapter for Sequel.
|