mobilize-hive 1.297 → 1.298

data/lib/mobilize-hive/handlers/hive.rb

@@ -248,6 +248,7 @@ module Mobilize
     def Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, job_name, drop=false, schema_hash=nil)
       table_path = [db,table].join(".")
       table_stats = Hive.table_stats(cluster, db, table, user_name)
+      url = "hive://" + [cluster,db,table,part_array.compact.join("/")].join("/")
 
       source_hql_array = source_hql.split(";")
       last_select_i = source_hql_array.rindex{|hql| hql.downcase.strip.starts_with?("select")}
@@ -359,6 +360,12 @@ module Mobilize
           part_perm_tsv = response['stdout']
           #having gotten the permutations, ensure they are dropped
           part_hash_array = part_perm_tsv.tsv_to_hash_array
+          #make sure there is data
+          if part_hash_array.first.nil? or part_hash_array.first.values.include?(nil)
+            #blank result set, return url
+            return url
+          end
+
           part_drop_hql = part_hash_array.map do |h|
             part_drop_stmt = h.map do |name,value|
                                part_defs[name[1..-2]]=="string" ? "#{name}='#{value}'" : "#{name}=#{value}"
@@ -380,7 +387,6 @@ module Mobilize
         error_msg = "Incompatible partition specs"
         raise error_msg
       end
-      url = "hive://" + [cluster,db,table,part_array.compact.join("/")].join("/")
       return url
     end
 
@@ -389,6 +395,11 @@ module Mobilize
     #also schema with column datatype overrides
     def Hive.tsv_to_table(cluster, db, table, part_array, source_tsv, user_name, drop=false, schema_hash=nil)
       return nil if source_tsv.strip.length==0
+      if source_tsv.index("\r\n")
+        source_tsv = source_tsv.gsub("\r\n","\n")
+      elsif source_tsv.index("\r")
+        source_tsv = source_tsv.gsub("\r","\n")
+      end
       source_headers = source_tsv.tsv_header_array
 
       table_path = [db,table].join(".")
@@ -541,11 +552,11 @@ module Mobilize
           #source table
           cluster,source_path = source.path.split("/").ie{|sp| [sp.first, sp[1..-1].join(".")]}
           source_hql = "select * from #{source_path};"
-        elsif ['gsheet','gridfs','hdfs'].include?(source.handler)
+        elsif ['gsheet','gfile','gridfs','hdfs'].include?(source.handler)
           if source.path.ie{|sdp| sdp.index(/\.[A-Za-z]ql$/) or sdp.ends_with?(".ql")}
             source_hql = source.read(user_name,gdrive_slot)
           else
-            #tsv from sheet
+            #tsv from sheet or file
             source_tsv = source.read(user_name,gdrive_slot)
           end
         end

data/lib/mobilize-hive/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hive
-    VERSION = "1.297"
+    VERSION = "1.298"
   end
 end

data/mobilize-hive.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-hdfs","1.297"
+  gem.add_runtime_dependency "mobilize-hdfs","1.298"
 end

data/test/hive_job_rows.yml

@@ -26,7 +26,7 @@
   stage4: gsheet.write source:"hive://mobilize/hive_test_3", target:"hive_test_3.out"
 - name: hive_test_4
   active: true
-  trigger: after hive_test_2
+  trigger: after hive_test_3
   status: ""
   stage1: hive.write source:"hive_test_4_stage_1.in", target:"mobilize/hive_test_1", partitions:"act_date"
   stage2: hive.write source:"hive_test_4_stage_2.in", target:"mobilize/hive_test_1", partitions:"act_date"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hive
 version: !ruby/object:Gem::Version
-  version: '1.297'
+  version: '1.298'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-06 00:00:00.000000000 Z
+date: 2013-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-hdfs
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.297'
+        version: '1.298'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.297'
+        version: '1.298'
 description: Adds hive read, write, and run support to mobilize-hdfs
 email:
 - cpaesleme@dena.com
@@ -66,7 +66,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -281949259338771917
+      hash: 1394133607903248824
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -75,7 +75,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -281949259338771917
+      hash: 1394133607903248824
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25