mobilize-hive 1.291 → 1.292

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1eb5a243ff499f31c06f0c1e09abba3fafee0b31
-  data.tar.gz: d44471ad7d8fcc72ac8562ebfb529d31d839d195
+  metadata.gz: c6d52c5de86351ccf5ab3c4da09b7341f27e5163
+  data.tar.gz: 3e560d834babc1377d8e3c6c80b799e9c4655acf
 SHA512:
-  metadata.gz: a27bde80d634f949cbf7f82b0ceaad4e087e3fe1bee6fe4631aa8205b68a7897e9d0f8e8ec49700a37cc3bd54e266df06a30e2187e13b6f9a1357d1d270af54c
-  data.tar.gz: 3763262ee0ac27778cc2abb6342b20d9fb1a1c6c546ddd87f4ff73969a8054314480387c230aa232e89fad7e2df16f597f2fa50344430343a7cdde9aa03d0d79
+  metadata.gz: d910e35cefda69f9640105949454b04b6d08df9069e2e9fdb61b9875806c921fb187701977afee7efcc3a5b84dadbbd91abf4d6bf9b4817bcf4f3c1945effbfb
+  data.tar.gz: 9a41441ce0ddf76b81fe63350a2ece3d1cc5ab0daccb1bc09f6db394dcb618078c4d01c1893d3a16090656dfdd36ec5fa8332aa5a699d096abcd85b726a1efa2

data/lib/mobilize-hive/handlers/hive.rb

@@ -262,7 +262,8 @@ module Mobilize
       temp_set_hql = "set mapred.job.name=#{job_name} (temp table);"
       temp_drop_hql = "drop table if exists #{temp_table_path};"
       temp_create_hql = "#{temp_set_hql}#{prior_hql}#{temp_drop_hql}create table #{temp_table_path} as #{last_select_hql}"
-      Hive.run(cluster,temp_create_hql,user_name)
+      response = Hive.run(cluster,temp_create_hql,user_name)
+      raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
 
       source_table_stats = Hive.table_stats(cluster,temp_db,temp_table_name,user_name)
       source_fields = source_table_stats['field_defs']
@@ -300,7 +301,9 @@ module Mobilize
                            target_insert_hql,
                            temp_drop_hql].join
 
-        Hive.run(cluster, target_full_hql, user_name)
+        response = Hive.run(cluster, target_full_hql, user_name)
+
+        raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
 
       elsif part_array.length > 0 and
         table_stats.ie{|tts| tts.nil? || drop || tts['partitions'].to_a.map{|p| p['name']} == part_array}
@@ -350,7 +353,9 @@ module Mobilize
           part_set_hql = "set hive.cli.print.header=true;set mapred.job.name=#{job_name} (permutations);"
           part_select_hql = "select distinct #{target_part_stmt} from #{temp_table_path};"
           part_perm_hql = part_set_hql + part_select_hql
-          part_perm_tsv = Hive.run(cluster, part_perm_hql, user_name)['stdout']
+          response = Hive.run(cluster, part_perm_hql, user_name)
+          raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
+          part_perm_tsv = response['stdout']
           #having gotten the permutations, ensure they are dropped
           part_hash_array = part_perm_tsv.tsv_to_hash_array
           part_drop_hql = part_hash_array.map do |h|
@@ -368,7 +373,8 @@ module Mobilize
 
         target_full_hql = [target_set_hql, target_create_hql, target_insert_hql, temp_drop_hql].join
 
-        Hive.run(cluster, target_full_hql, user_name)
+        response = Hive.run(cluster, target_full_hql, user_name)
+        raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
       else
         error_msg = "Incompatible partition specs"
         raise error_msg
@@ -414,7 +420,8 @@ module Mobilize
 
         target_full_hql = [target_drop_hql,target_create_hql,target_insert_hql].join(";")
 
-        Hive.run(cluster, target_full_hql, user_name, file_hash)
+        response = Hive.run(cluster, target_full_hql, user_name, file_hash)
+        raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
 
       elsif part_array.length > 0 and
         table_stats.ie{|tts| tts.nil? || drop || tts['partitions'].to_a.map{|p| p['name']} == part_array}
@@ -441,7 +448,8 @@ module Mobilize
                             "partitioned by #{partition_defs}"
 
         #create target table early if not here
-        Hive.run(cluster, target_create_hql, user_name)
+        response = Hive.run(cluster, target_create_hql, user_name)
+        raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
 
         table_stats = Hive.table_stats(cluster, db, table, user_name)
 
@@ -480,7 +488,8 @@ module Mobilize
         #run actual partition adds all at once
         if target_part_hql.length>0
           puts "Adding partitions to #{cluster}/#{db}/#{table} for #{user_name} at #{Time.now.utc}"
-          Hive.run(cluster, target_part_hql, user_name)
+          response = Hive.run(cluster, target_part_hql, user_name)
+          raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
         end
       else
         error_msg = "Incompatible partition specs: " +
@@ -488,6 +497,7 @@ module Mobilize
                     "user_params:#{part_array.to_s}"
         raise error_msg
       end
+
       url = "hive://" + [cluster,db,table,part_array.compact.join("/")].join("/")
       return url
     end
@@ -580,11 +590,8 @@ module Mobilize
       select_hql = "select * from #{source_path};"
       hql = [set_hql,select_hql].join
       response = Hive.run(cluster, hql,user_name)
-      if response['exit_code']==0
-        return response['stdout']
-      else
-        raise "Unable to read hive://#{dst_path} with error: #{response['stderr']}"
-      end
+      raise "Unable to read hive://#{dst_path} with error: #{response['stderr']}" if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
+      return response['stdout']
     end
 
     def Hive.write_by_dataset_path(dst_path,source_tsv,user_name,*args)

data/lib/mobilize-hive/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hive
-    VERSION = "1.291"
+    VERSION = "1.292"
   end
 end

data/mobilize-hive.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-hdfs","1.291"
+  gem.add_runtime_dependency "mobilize-hdfs","1.292"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hive
 version: !ruby/object:Gem::Version
-  version: '1.291'
+  version: '1.292'
 platform: ruby
 authors:
 - Cassio Paes-Leme
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-27 00:00:00.000000000 Z
+date: 2013-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-hdfs
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.291'
+        version: '1.292'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.291'
+        version: '1.292'
 description: Adds hive read, write, and run support to mobilize-hdfs
 email:
 - cpaesleme@dena.com