hdfs_jruby 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -13
- data/Gemfile +3 -0
- data/README.md +7 -7
- data/hdfs_jruby.gemspec +1 -1
- data/lib/hdfs_jruby/version.rb +1 -1
- data/lib/hdfs_jruby.rb +13 -1
- data/rspec/01.rb +107 -0
- data/rspec/test_data/a/a/test.txt +1 -0
- data/rspec/test_data/a/b/test.txt +1 -0
- data/rspec/test_data/a/c/test.txt +1 -0
- data/rspec/test_data/b/test.txt +1 -0
- data/rspec/test_data/c/test.txt +1 -0
- data/test/test_ls.rb +1 -1
- metadata +16 -10
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
OTI4NjYwNWJmZGEzN2YxNjI4MmVmOWM2YzNiNDkyMzg4OWI3NDFjOQ==
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 69687639027bf661cd93890f5c8fb6fd9e120213
|
4
|
+
data.tar.gz: 25deefd6f6d5c6a2ef817794207442889ac98ee6
|
7
5
|
SHA512:
|
8
|
-
metadata.gz:
|
9
|
-
|
10
|
-
MThkZDZjZmIzNDIwOWRhMTMyZTE4YTdmMjY1MTEzMjdiNTk0YTllMjI1NTQ1
|
11
|
-
MzU5MTY4NGRkM2Q4MmNhNDI0NTk0NjFhY2UwMzMxNmM0OGM3MmU=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
MWYyY2RiOGU2OWFmZjk3YTFmNjU5MmE3NTFhNDk4ZDE3ZjJhZDZmNWZhNjgx
|
14
|
-
ZjVhNDE0ZjVkMTY5YTQ2MzFkOTI2MTViNDU4MGMzMWY5ZWNiZDc4Y2JhYTdm
|
15
|
-
ZTNkYTExZGE5MTE3NGM5OTRmMmExN2VmNTdmMzljNzM5NDZkYzc=
|
6
|
+
metadata.gz: a6ef1a85d42d0c492c1584a975786476ef0c1bc5338bc61460e0f9ff349f46a189123d885cfdf11ef9139edb59a7fd0c80cf5dc6e9bbb3ae76d692feefc6a93c
|
7
|
+
data.tar.gz: df0f2243afaaaa31fa5211860ce1c65fe0c40b549813412ee347c6de7a51a365b16895260f288e52f0861551fdb081e2bbdcaa956587c0571005017ac2defb65
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -18,13 +18,13 @@ Or install it yourself as:
|
|
18
18
|
|
19
19
|
## Usage
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
21
|
+
require 'hdfs_jruby'
|
22
|
+
|
23
|
+
...
|
24
|
+
|
25
|
+
Hdfs.ls(path) do | stat |
|
26
|
+
p stat
|
27
|
+
end
|
28
28
|
|
29
29
|
|
30
30
|
## Contributing
|
data/hdfs_jruby.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.email = ["gm.ikeda@gmail.com"]
|
11
11
|
spec.summary = %q{ jruby hdfs api}
|
12
12
|
spec.description = %q{}
|
13
|
-
spec.homepage = ""
|
13
|
+
spec.homepage = "https://github.com/shinjiikeda/hdfs_jruby"
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
16
16
|
spec.files = `git ls-files -z`.split("\x0")
|
data/lib/hdfs_jruby/version.rb
CHANGED
data/lib/hdfs_jruby.rb
CHANGED
@@ -33,16 +33,26 @@ module Hdfs
|
|
33
33
|
class FsPermission < org.apache.hadoop.fs.permission.FsPermission
|
34
34
|
end
|
35
35
|
|
36
|
-
@conf = Hdfs::Configuration.new
|
36
|
+
@conf = Hdfs::Configuration.new
|
37
37
|
@fs = Hdfs::FileSystem.get(@conf)
|
38
38
|
|
39
|
+
def connectAsUser(user)
|
40
|
+
uri = Hdfs::FileSystem.getDefaultUri(@conf)
|
41
|
+
@fs.close if ! @fs.nil?
|
42
|
+
@fs = Hdfs::FileSystem.get(uri, @conf, user)
|
43
|
+
end
|
44
|
+
|
39
45
|
def ls(path)
|
40
46
|
p = _path(path)
|
41
47
|
list = @fs.globStatus(p)
|
48
|
+
return [] if list.nil?
|
49
|
+
|
42
50
|
ret_list = []
|
43
51
|
list.each do |stat|
|
44
52
|
if stat.isDir
|
45
53
|
sub_list = @fs.listStatus(stat.getPath)
|
54
|
+
next if sub_list.nil?
|
55
|
+
|
46
56
|
sub_list.each do | s |
|
47
57
|
if block_given?
|
48
58
|
yield _conv(s)
|
@@ -157,6 +167,7 @@ module Hdfs
|
|
157
167
|
module_function :set_owner
|
158
168
|
module_function :list
|
159
169
|
module_function :ls
|
170
|
+
module_function :connectAsUser
|
160
171
|
|
161
172
|
private
|
162
173
|
def _path(path)
|
@@ -180,4 +191,5 @@ module Hdfs
|
|
180
191
|
|
181
192
|
module_function :_path
|
182
193
|
module_function :_conv
|
194
|
+
|
183
195
|
end
|
data/rspec/01.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'rspec'
|
5
|
+
|
6
|
+
require 'hdfs_jruby'
|
7
|
+
require 'hdfs_jruby/file'
|
8
|
+
|
9
|
+
HDFS_TMP_DIR="./test_rspec.#{$$}"
|
10
|
+
puts "hdfs tmpdir: #{HDFS_TMP_DIR}"
|
11
|
+
|
12
|
+
describe "test1" do
|
13
|
+
|
14
|
+
before(:all) do
|
15
|
+
Hdfs.mkdir(HDFS_TMP_DIR)
|
16
|
+
end
|
17
|
+
|
18
|
+
it "put test_dir" do
|
19
|
+
Hdfs.put("./rspec/test_data", HDFS_TMP_DIR)
|
20
|
+
end
|
21
|
+
|
22
|
+
it "ls #{HDFS_TMP_DIR}/test_data use block" do
|
23
|
+
cnt = 0
|
24
|
+
Hdfs.ls("#{HDFS_TMP_DIR}/test_data").each do | stat |
|
25
|
+
#p stat
|
26
|
+
cnt+=1
|
27
|
+
end
|
28
|
+
expect(cnt).to eq 3
|
29
|
+
end
|
30
|
+
it "ls #{HDFS_TMP_DIR}/test_data" do
|
31
|
+
r = Hdfs.ls("#{HDFS_TMP_DIR}/test_data")
|
32
|
+
expect(r.size).to eq 3
|
33
|
+
end
|
34
|
+
|
35
|
+
it "exists #{HDFS_TMP_DIR}/test_data/a/a" do
|
36
|
+
r = Hdfs.exists?("#{HDFS_TMP_DIR}/test_data/a/a")
|
37
|
+
expect(r).to eq true
|
38
|
+
end
|
39
|
+
it "directoy? #{HDFS_TMP_DIR}/test_data/a/a" do
|
40
|
+
r = Hdfs.directory?("#{HDFS_TMP_DIR}/test_data/a/a")
|
41
|
+
expect(r).to eq true
|
42
|
+
end
|
43
|
+
|
44
|
+
it "file? #{HDFS_TMP_DIR}/test_data/a/a/test.txt" do
|
45
|
+
r = Hdfs.file?("#{HDFS_TMP_DIR}/test_data/a/a/test.txt")
|
46
|
+
expect(r).to eq true
|
47
|
+
end
|
48
|
+
|
49
|
+
it "size #{HDFS_TMP_DIR}/test_data/a/a/test.txt" do
|
50
|
+
size = Hdfs.size("#{HDFS_TMP_DIR}/test_data/a/a/test.txt")
|
51
|
+
expect(size).to eq 4
|
52
|
+
end
|
53
|
+
|
54
|
+
it "create #{HDFS_TMP_DIR}/test_data/d/a/test.txt" do
|
55
|
+
Hdfs::File.open("#{HDFS_TMP_DIR}/test_data/d/a/test.txt", "w") do |io|
|
56
|
+
io.puts("d/a")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
it "read #{HDFS_TMP_DIR}/test_data/d/a/test.txt" do
|
61
|
+
content = nil
|
62
|
+
Hdfs::File.open("#{HDFS_TMP_DIR}/test_data/d/a/test.txt", "r") do |io|
|
63
|
+
content = io.read
|
64
|
+
end
|
65
|
+
expect(content).to eq "d/a\n"
|
66
|
+
end
|
67
|
+
|
68
|
+
it "append #{HDFS_TMP_DIR}/test_data/append_test/test.txt" do
|
69
|
+
Hdfs::File.open("#{HDFS_TMP_DIR}/test_data/append_test/test.txt", "a") do |io|
|
70
|
+
io.puts("1")
|
71
|
+
end
|
72
|
+
Hdfs::File.open("#{HDFS_TMP_DIR}/test_data/append_test/test.txt", "a") do |io|
|
73
|
+
io.puts("2")
|
74
|
+
end
|
75
|
+
content = nil
|
76
|
+
Hdfs::File.open("#{HDFS_TMP_DIR}/test_data/append_test/test.txt", "r") do |io|
|
77
|
+
content = io.read
|
78
|
+
end
|
79
|
+
expect(content).to eq "1\n2\n"
|
80
|
+
end
|
81
|
+
|
82
|
+
it "delete not empty directory" do
|
83
|
+
begin
|
84
|
+
Hdfs.delete("#{HDFS_TMP_DIR}/test_data")
|
85
|
+
r = false
|
86
|
+
rescue Java::OrgApacheHadoopIpc::RemoteException => e
|
87
|
+
if e.to_s =~ /is non empty/
|
88
|
+
r = true
|
89
|
+
else
|
90
|
+
r = false
|
91
|
+
end
|
92
|
+
rescue
|
93
|
+
r = false
|
94
|
+
end
|
95
|
+
expect(r).to eq true
|
96
|
+
end
|
97
|
+
|
98
|
+
it "delete directory" do
|
99
|
+
Hdfs.delete("#{HDFS_TMP_DIR}/test_data", true)
|
100
|
+
end
|
101
|
+
|
102
|
+
after(:all) do
|
103
|
+
Hdfs.delete(HDFS_TMP_DIR, true)
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
|
@@ -0,0 +1 @@
|
|
1
|
+
a/a
|
@@ -0,0 +1 @@
|
|
1
|
+
a/b
|
@@ -0,0 +1 @@
|
|
1
|
+
a/c
|
@@ -0,0 +1 @@
|
|
1
|
+
b
|
@@ -0,0 +1 @@
|
|
1
|
+
c
|
data/test/test_ls.rb
CHANGED
metadata
CHANGED
@@ -1,41 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hdfs_jruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- shinji ikeda
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-09-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.6'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.6'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
description: ''
|
@@ -45,7 +45,7 @@ executables: []
|
|
45
45
|
extensions: []
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
|
-
- .gitignore
|
48
|
+
- ".gitignore"
|
49
49
|
- Gemfile
|
50
50
|
- LICENSE.txt
|
51
51
|
- README.md
|
@@ -54,6 +54,12 @@ files:
|
|
54
54
|
- lib/hdfs_jruby.rb
|
55
55
|
- lib/hdfs_jruby/file.rb
|
56
56
|
- lib/hdfs_jruby/version.rb
|
57
|
+
- rspec/01.rb
|
58
|
+
- rspec/test_data/a/a/test.txt
|
59
|
+
- rspec/test_data/a/b/test.txt
|
60
|
+
- rspec/test_data/a/c/test.txt
|
61
|
+
- rspec/test_data/b/test.txt
|
62
|
+
- rspec/test_data/c/test.txt
|
57
63
|
- test/test_append.rb
|
58
64
|
- test/test_get.rb
|
59
65
|
- test/test_ls.rb
|
@@ -64,7 +70,7 @@ files:
|
|
64
70
|
- test/test_read2.rb
|
65
71
|
- test/test_utils.rb
|
66
72
|
- test/test_write.rb
|
67
|
-
homepage:
|
73
|
+
homepage: https://github.com/shinjiikeda/hdfs_jruby
|
68
74
|
licenses:
|
69
75
|
- MIT
|
70
76
|
metadata: {}
|
@@ -74,12 +80,12 @@ require_paths:
|
|
74
80
|
- lib
|
75
81
|
required_ruby_version: !ruby/object:Gem::Requirement
|
76
82
|
requirements:
|
77
|
-
- -
|
83
|
+
- - ">="
|
78
84
|
- !ruby/object:Gem::Version
|
79
85
|
version: '0'
|
80
86
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
81
87
|
requirements:
|
82
|
-
- -
|
88
|
+
- - ">="
|
83
89
|
- !ruby/object:Gem::Version
|
84
90
|
version: '0'
|
85
91
|
requirements: []
|