mobilize-hive 1.25 → 1.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a728ff9978514fe5f0fcf230117bb601938eaba9
-  data.tar.gz: 556d3e8c3e22de09e5f251cadf693d1bfd2122b0
+  metadata.gz: a107187c623de567063792546016ea6fd59f92d3
+  data.tar.gz: 8a1aeaca87cdaa2161666a743d66af278a94eabb
 SHA512:
-  metadata.gz: 18ea8474ab5376cecee6fc1fe33b688b16eb88ab5da14cd09ef52892dbb1333a6991ff2581702f681b60915943a82e8572400a2adb59b11b96a47fa6e93353b3
-  data.tar.gz: a0e1ecfcee8f0eb7296b71c2700d9ed2d09688d52252b306c8c03b21c82d8d3ba6b09a849c62a55e68518f29a14d9f3da9095a73db5f3a34a627894b7916f8f6
+  metadata.gz: ab2b0d4b7ad35f3c5d213f534dff88f4e5fac0c7e43dd152b3325695e46ba970ac0a9616031520091b854aaac5a4fb66df9baf6da503be618b5483682d0091c7
+  data.tar.gz: 8a8ad939b734d328062f7dcf2f0b3702486804c20c1d127c1fbedc5d012694dcff996712321cfb8b0e49fcc48cc924938074b19d74d242ecf6d5792ba4a5dc7b

data/lib/mobilize-hive/handlers/hive.rb

@@ -52,7 +52,7 @@ module Mobilize
     end
 
     # converts a source path or target path to a dst in the context of handler and stage
-    def Hive.path_to_dst(path,stage_path)
+    def Hive.path_to_dst(path,stage_path,gdrive_slot)
       has_handler = true if path.index("://")
       s = Stage.where(:path=>stage_path).first
       params = s.params
@@ -78,7 +78,7 @@ module Mobilize
         return Dataset.find_or_create_by_url(hive_url)
       end
       #otherwise, use hdfs convention
-      return Ssh.path_to_dst(path,stage_path)
+      return Ssh.path_to_dst(path,stage_path,gdrive_slot)
     end
 
     def Hive.url_by_path(path,user_name,is_target=false)
@@ -163,6 +163,9 @@ module Mobilize
     end
 
     def Hive.run_by_stage_path(stage_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
+      #return blank response if there are no slots available
+      return nil unless gdrive_slot
       s = Stage.where(:path=>stage_path).first
       params = s.params
       cluster = params['cluster'] || Hive.clusters.keys.first
@@ -181,10 +184,12 @@ module Mobilize
       if params['hql']
         hql = params['hql']
       else
-        source = s.sources.first
-        hql = source.read(user_name)
+        source = s.sources(gdrive_slot).first
+        hql = source.read(user_name,gdrive_slot)
       end
 
+      Gdrive.unslot_worker_by_path(stage_path)
+
       #check for select at end
       hql_array = hql.split(";").map{|hc| hc.strip}.reject{|hc| hc.length==0}
       if hql_array.last.downcase.starts_with?("select")
@@ -231,7 +236,7 @@ module Mobilize
       file_name = schema_path.split("/").last
       out_url = "gridfs://#{schema_path}/#{file_name}"
       Dataset.write_by_url(out_url,out_tsv,user_name)
-      schema_tsv = Dataset.find_by_url(out_url).read(user_name)
+      schema_tsv = Dataset.find_by_url(out_url).read(user_name,gdrive_slot)
       schema_hash = {}
       schema_tsv.tsv_to_hash_array.each do |ha|
         schema_hash[ha['name']] = ha['datatype']
@@ -488,28 +493,26 @@ module Mobilize
     end
 
     def Hive.write_by_stage_path(stage_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
+      #return blank response if there are no slots available
+      return nil unless gdrive_slot
       s = Stage.where(:path=>stage_path).first
       params = s.params
-      source = s.sources.first
+      source = s.sources(gdrive_slot).first
       target = s.target
       cluster, db, table = target.url.split("://").last.split("/")
-      #update stage with the node so we can use it
-      user_name = Hdfs.user_name_by_stage_path(stage_path,cluster)
-      job_name = s.path.sub("Runner_","")
-
       #slot Hive worker if available
       slot_id = Hive.slot_worker_by_cluster_and_path(cluster,stage_path)
       return false unless slot_id
+      #update stage with the node so we can use it
+      user_name = Hdfs.user_name_by_stage_path(stage_path,cluster)
+      job_name = s.path.sub("Runner_","")
 
       schema_hash = if params['schema']
-                      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
-                      #return blank response if there are no slots available
-                      return nil unless gdrive_slot
                       Hive.schema_hash(params['schema'],user_name,gdrive_slot)
                     else
                       {}
                     end
-      Gdrive.unslot_worker_by_path(stage_path)
       #drop target before create/insert?
       drop = params['drop']
 
@@ -524,14 +527,15 @@ module Mobilize
           source_hql = "select * from #{source_path};"
         elsif ['gsheet','gridfs','hdfs'].include?(source.handler)
           if source.path.ie{|sdp| sdp.index(/\.[A-Za-z]ql$/) or sdp.ends_with?(".ql")}
-            source_hql = source.read(user_name)
+            source_hql = source.read(user_name,gdrive_slot)
           else
             #tsv from sheet
-            source_tsv = source.read(user_name)
+            source_tsv = source.read(user_name,gdrive_slot)
           end
         end
       end
 
+      Gdrive.unslot_worker_by_path(stage_path)
       part_array = if params['partitions']
                     params['partitions'].to_a.map{|p| p.gsub(".","/").split("/")}.flatten
                   elsif params['target']

data/lib/mobilize-hive/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hive
-    VERSION = "1.25"
+    VERSION = "1.26"
   end
 end

data/mobilize-hive.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-hdfs","1.25"
+  gem.add_runtime_dependency "mobilize-hdfs","1.26"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hive
 version: !ruby/object:Gem::Version
-  version: '1.25'
+  version: '1.26'
 platform: ruby
 authors:
 - Cassio Paes-Leme
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-24 00:00:00.000000000 Z
+date: 2013-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-hdfs
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.25'
+        version: '1.26'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.25'
+        version: '1.26'
 description: Adds hive read, write, and run support to mobilize-hdfs
 email:
 - cpaesleme@dena.com