mobilize-hdfs 1.25 → 1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bfe10046bf2ec3042d461b8ce909cc68216e335a
-  data.tar.gz: 5b830763fa2ed7279ca11f4a0cb13c13f9052605
+  metadata.gz: af936dec46c54a0c77b1fc46ce1fff4750cd2438
+  data.tar.gz: f97d3403887b35068afb5e108ea95bb120986b7c
 SHA512:
-  metadata.gz: fc5bab8d9fa1045cd2f5a14589a883fe27cda6bda133156d7f322148169d7e24036358bc591fb0479cd45a25061feb2c378c1fa35185614ff22d1a757cd8c297
-  data.tar.gz: 1fcddaca9b2dc4beb458bf5e33c30e883c025770ccf4d67f5ce982300f5ba6f5c6fee86d075dbeee1c6f8e08b572f789bf03a52e6470379cb975dd483224a154
+  metadata.gz: a2dc030b2d8a3756037e4241a70a1d488dd530262ee6935d0202dc028781578dab3feeaec253139690a6b1bb76c8a2dda59cfc49ec70d44f9a4e3cc727a8296a
+  data.tar.gz: 6ddb394475be051b9d664381b352b07ab51d5e9f968ba4f279651f1385ad2fe7cbf938d8d06c1918e16273b8b7201c4d5ffa933e9fcc0c705c8119412a26436a

data/lib/mobilize-hdfs/handlers/hdfs.rb

@@ -96,7 +96,7 @@ module Mobilize
     end
 
     # converts a source path or target path to a dst in the context of handler and stage
-    def Hdfs.path_to_dst(path,stage_path)
+    def Hdfs.path_to_dst(path,stage_path,gdrive_slot)
       has_handler = true if path.index("://")
       s = Stage.where(:path=>stage_path).first
       params = s.params
@@ -114,7 +114,7 @@ module Mobilize
         return Dataset.find_or_create_by_url(hdfs_url)
       end
       #otherwise, use ssh convention
-      return Ssh.path_to_dst(path,stage_path)
+      return Ssh.path_to_dst(path,stage_path,gdrive_slot)
     end
 
     def Hdfs.url_by_path(path,user_name,is_target=false)
@@ -156,15 +156,19 @@ module Mobilize
     end
 
     def Hdfs.write_by_stage_path(stage_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
+      #return blank response if there are no slots available
+      return nil unless gdrive_slot
       s = Stage.where(:path=>stage_path).first
-      source = s.sources.first
+      source = s.sources(gdrive_slot).first
+      Gdrive.unslot_worker_by_path(stage_path)
       target = s.target
       cluster = target.url.split("://").last.split("/").first
       user_name = Hdfs.user_name_by_stage_path(stage_path,cluster)
       stdout = if source.handler == 'hdfs'
                    Hdfs.copy(source.url,target.url,user_name)
                  elsif ["gsheet","gfile","ssh"].include?(source.handler)
-                   in_string = source.read(user_name)
+                   in_string = source.read(user_name,gdrive_slot)
                    Dataset.write_by_url(target.url, in_string, user_name)
                  end
       return {'out_str'=>stdout, 'signal' => 0}

data/lib/mobilize-hdfs/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hdfs
-    VERSION = "1.25"
+    VERSION = "1.26"
   end
 end

data/mobilize-hdfs.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-ssh","1.25"
+  gem.add_runtime_dependency "mobilize-ssh","1.26"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hdfs
 version: !ruby/object:Gem::Version
-  version: '1.25'
+  version: '1.26'
 platform: ruby
 authors:
 - Cassio Paes-Leme
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-24 00:00:00.000000000 Z
+date: 2013-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-ssh
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.25'
+        version: '1.26'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.25'
+        version: '1.26'
 description: Adds hdfs read, write, and copy support to mobilize-ssh
 email:
 - cpaesleme@dena.com