mobilize-hdfs 1.3 → 1.21
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/mobilize-hdfs/handlers/hdfs.rb +4 -8
- data/lib/mobilize-hdfs/version.rb +1 -1
- data/mobilize-hdfs.gemspec +1 -1
- metadata +10 -4
@@ -96,7 +96,7 @@ module Mobilize
|
|
96
96
|
end
|
97
97
|
|
98
98
|
# converts a source path or target path to a dst in the context of handler and stage
|
99
|
-
def Hdfs.path_to_dst(path,stage_path
|
99
|
+
def Hdfs.path_to_dst(path,stage_path)
|
100
100
|
has_handler = true if path.index("://")
|
101
101
|
s = Stage.where(:path=>stage_path).first
|
102
102
|
params = s.params
|
@@ -114,7 +114,7 @@ module Mobilize
|
|
114
114
|
return Dataset.find_or_create_by_url(hdfs_url)
|
115
115
|
end
|
116
116
|
#otherwise, use ssh convention
|
117
|
-
return Ssh.path_to_dst(path,stage_path
|
117
|
+
return Ssh.path_to_dst(path,stage_path)
|
118
118
|
end
|
119
119
|
|
120
120
|
def Hdfs.url_by_path(path,user_name,is_target=false)
|
@@ -156,19 +156,15 @@ module Mobilize
|
|
156
156
|
end
|
157
157
|
|
158
158
|
def Hdfs.write_by_stage_path(stage_path)
|
159
|
-
gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
|
160
|
-
#return blank response if there are no slots available
|
161
|
-
return nil unless gdrive_slot
|
162
159
|
s = Stage.where(:path=>stage_path).first
|
163
|
-
source = s.sources
|
164
|
-
Gdrive.unslot_worker_by_path(stage_path)
|
160
|
+
source = s.sources.first
|
165
161
|
target = s.target
|
166
162
|
cluster = target.url.split("://").last.split("/").first
|
167
163
|
user_name = Hdfs.user_name_by_stage_path(stage_path,cluster)
|
168
164
|
stdout = if source.handler == 'hdfs'
|
169
165
|
Hdfs.copy(source.url,target.url,user_name)
|
170
166
|
elsif ["gsheet","gfile","ssh"].include?(source.handler)
|
171
|
-
in_string = source.read(user_name
|
167
|
+
in_string = source.read(user_name)
|
172
168
|
Dataset.write_by_url(target.url, in_string, user_name)
|
173
169
|
end
|
174
170
|
return {'out_str'=>stdout, 'signal' => 0}
|
data/mobilize-hdfs.gemspec
CHANGED
@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
17
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
18
|
gem.require_paths = ["lib"]
|
19
|
-
gem.add_runtime_dependency "mobilize-ssh","1.
|
19
|
+
gem.add_runtime_dependency "mobilize-ssh","1.21"
|
20
20
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-hdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.21'
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-03-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: mobilize-ssh
|
@@ -18,7 +18,7 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - '='
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: '1.
|
21
|
+
version: '1.21'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - '='
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: '1.
|
29
|
+
version: '1.21'
|
30
30
|
description: Adds hdfs read, write, and copy support to mobilize-ssh
|
31
31
|
email:
|
32
32
|
- cpaesleme@dena.com
|
@@ -62,12 +62,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
62
62
|
- - ! '>='
|
63
63
|
- !ruby/object:Gem::Version
|
64
64
|
version: '0'
|
65
|
+
segments:
|
66
|
+
- 0
|
67
|
+
hash: 4610890029758728306
|
65
68
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
66
69
|
none: false
|
67
70
|
requirements:
|
68
71
|
- - ! '>='
|
69
72
|
- !ruby/object:Gem::Version
|
70
73
|
version: '0'
|
74
|
+
segments:
|
75
|
+
- 0
|
76
|
+
hash: 4610890029758728306
|
71
77
|
requirements: []
|
72
78
|
rubyforge_project:
|
73
79
|
rubygems_version: 1.8.25
|