mobilize-hdfs 1.25 → 1.26
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/mobilize-hdfs/handlers/hdfs.rb +8 -4
- data/lib/mobilize-hdfs/version.rb +1 -1
- data/mobilize-hdfs.gemspec +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: af936dec46c54a0c77b1fc46ce1fff4750cd2438
|
4
|
+
data.tar.gz: f97d3403887b35068afb5e108ea95bb120986b7c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a2dc030b2d8a3756037e4241a70a1d488dd530262ee6935d0202dc028781578dab3feeaec253139690a6b1bb76c8a2dda59cfc49ec70d44f9a4e3cc727a8296a
|
7
|
+
data.tar.gz: 6ddb394475be051b9d664381b352b07ab51d5e9f968ba4f279651f1385ad2fe7cbf938d8d06c1918e16273b8b7201c4d5ffa933e9fcc0c705c8119412a26436a
|
@@ -96,7 +96,7 @@ module Mobilize
|
|
96
96
|
end
|
97
97
|
|
98
98
|
# converts a source path or target path to a dst in the context of handler and stage
|
99
|
-
def Hdfs.path_to_dst(path,stage_path)
|
99
|
+
def Hdfs.path_to_dst(path,stage_path,gdrive_slot)
|
100
100
|
has_handler = true if path.index("://")
|
101
101
|
s = Stage.where(:path=>stage_path).first
|
102
102
|
params = s.params
|
@@ -114,7 +114,7 @@ module Mobilize
|
|
114
114
|
return Dataset.find_or_create_by_url(hdfs_url)
|
115
115
|
end
|
116
116
|
#otherwise, use ssh convention
|
117
|
-
return Ssh.path_to_dst(path,stage_path)
|
117
|
+
return Ssh.path_to_dst(path,stage_path,gdrive_slot)
|
118
118
|
end
|
119
119
|
|
120
120
|
def Hdfs.url_by_path(path,user_name,is_target=false)
|
@@ -156,15 +156,19 @@ module Mobilize
|
|
156
156
|
end
|
157
157
|
|
158
158
|
def Hdfs.write_by_stage_path(stage_path)
|
159
|
+
gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
|
160
|
+
#return blank response if there are no slots available
|
161
|
+
return nil unless gdrive_slot
|
159
162
|
s = Stage.where(:path=>stage_path).first
|
160
|
-
source = s.sources.first
|
163
|
+
source = s.sources(gdrive_slot).first
|
164
|
+
Gdrive.unslot_worker_by_path(stage_path)
|
161
165
|
target = s.target
|
162
166
|
cluster = target.url.split("://").last.split("/").first
|
163
167
|
user_name = Hdfs.user_name_by_stage_path(stage_path,cluster)
|
164
168
|
stdout = if source.handler == 'hdfs'
|
165
169
|
Hdfs.copy(source.url,target.url,user_name)
|
166
170
|
elsif ["gsheet","gfile","ssh"].include?(source.handler)
|
167
|
-
in_string = source.read(user_name)
|
171
|
+
in_string = source.read(user_name,gdrive_slot)
|
168
172
|
Dataset.write_by_url(target.url, in_string, user_name)
|
169
173
|
end
|
170
174
|
return {'out_str'=>stdout, 'signal' => 0}
|
data/mobilize-hdfs.gemspec
CHANGED
@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
17
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
18
|
gem.require_paths = ["lib"]
|
19
|
-
gem.add_runtime_dependency "mobilize-ssh","1.
|
19
|
+
gem.add_runtime_dependency "mobilize-ssh","1.26"
|
20
20
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-hdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.26'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Cassio Paes-Leme
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-03-
|
11
|
+
date: 2013-03-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mobilize-ssh
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - '='
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.26'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.26'
|
27
27
|
description: Adds hdfs read, write, and copy support to mobilize-ssh
|
28
28
|
email:
|
29
29
|
- cpaesleme@dena.com
|