mobilize-hdfs 1.2 → 1.3

data/lib/mobilize-hdfs/handlers/hdfs.rb

@@ -96,7 +96,7 @@ module Mobilize
     end
 
     # converts a source path or target path to a dst in the context of handler and stage
-    def Hdfs.path_to_dst(path,stage_path)
+    def Hdfs.path_to_dst(path,stage_path,gdrive_slot)
       has_handler = true if path.index("://")
       s = Stage.where(:path=>stage_path).first
       params = s.params
@@ -114,7 +114,7 @@ module Mobilize
         return Dataset.find_or_create_by_url(hdfs_url)
       end
       #otherwise, use ssh convention
-      return Ssh.path_to_dst(path,stage_path)
+      return Ssh.path_to_dst(path,stage_path,gdrive_slot)
     end
 
     def Hdfs.url_by_path(path,user_name,is_target=false)
@@ -156,15 +156,19 @@ module Mobilize
     end
 
     def Hdfs.write_by_stage_path(stage_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
+      #return blank response if there are no slots available
+      return nil unless gdrive_slot
       s = Stage.where(:path=>stage_path).first
-      source = s.sources.first
+      source = s.sources(gdrive_slot).first
+      Gdrive.unslot_worker_by_path(stage_path)
       target = s.target
       cluster = target.url.split("://").last.split("/").first
       user_name = Hdfs.user_name_by_stage_path(stage_path,cluster)
       stdout = if source.handler == 'hdfs'
                    Hdfs.copy(source.url,target.url,user_name)
                  elsif ["gsheet","gfile","ssh"].include?(source.handler)
-                   in_string = source.read(user_name)
+                   in_string = source.read(user_name,gdrive_slot)
                    Dataset.write_by_url(target.url, in_string, user_name)
                  end
       return {'out_str'=>stdout, 'signal' => 0}

data/lib/mobilize-hdfs/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hdfs
-    VERSION = "1.2"
+    VERSION = "1.3"
   end
 end

data/mobilize-hdfs.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-ssh","1.2"
+  gem.add_runtime_dependency "mobilize-ssh","1.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hdfs
 version: !ruby/object:Gem::Version
-  version: '1.2'
+  version: '1.3'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-21 00:00:00.000000000 Z
+date: 2013-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-ssh
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
 description: Adds hdfs read, write, and copy support to mobilize-ssh
 email:
 - cpaesleme@dena.com
@@ -70,7 +70,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Adds hdfs read, write, and copy support to mobilize-ssh
@@ -79,4 +79,3 @@ test_files:
 - test/mobilize-hdfs_test.rb
 - test/redis-test.conf
 - test/test_helper.rb
-has_rdoc: