mobilize-hive 1.22 → 1.23

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4c7e46212ccfe926790779712c8c2d5141d8343a
+  data.tar.gz: 099d652b673589e567e98d542cae516b4cfb1c3a
+SHA512:
+  metadata.gz: c7567c9d84ae911df3f586101314c32079abc512ce1b44b8d435a65de696b821531f1af3d63e5f39b4dca3a6e020d8f666837b58af8920d0b8a1110636635188
+  data.tar.gz: 8fe69c1b8a3f96eab10aa03e4d15a81437c337ef23d16d91668bd508bdf422d7b7e9c7a0d51b067fe5d445ea77aad034387f8fc807d784a9c46188d1ff040ad0

data/lib/mobilize-hive/handlers/hive.rb

@@ -108,7 +108,7 @@ module Mobilize
     def Hive.table_stats(cluster,db,table,user_name)
       describe_sql = "use #{db};describe extended #{table};"
       describe_response = Hive.run(cluster, describe_sql,user_name)
-      return describe_response if describe_response['stdout'].length==0
+      return nil if describe_response['stdout'].length==0
       describe_output = describe_response['stdout']
       describe_output.split("location:").last.split(",").first
       #get location, fields, partitions
@@ -232,24 +232,9 @@ module Mobilize
       schema_hash
     end
 
-    def Hive.path_params(cluster, path, user_name)
-      db, table, partitions = path.gsub(".","/").split("/").ie{|sp| [sp.first, sp.second, sp[2..-1]]}
-      #get existing table stats if any
-      curr_stats = begin
-                     Hive.table_stats(cluster, db, table, user_name)
-                   rescue
-                     nil
-                   end
-      {"db"=>db,
-       "table"=>table,
-       "partitions"=>partitions,
-       "curr_stats"=>curr_stats}
-    end
-
     def Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, job_name, drop=false, schema_hash=nil)
       table_path = [db,table].join(".")
-      target_params = Hive.path_params(cluster, table_path, user_name)
-      table_stats = target_params['curr_stats']
+      table_stats = Hive.table_stats(cluster, db, table, user_name)
 
       source_hql_array = source_hql.split(";")
       last_select_i = source_hql_array.rindex{|hql| hql.downcase.strip.starts_with?("select")}
@@ -267,9 +252,7 @@ module Mobilize
       temp_create_hql = "#{temp_set_hql}#{prior_hql}#{temp_drop_hql}create table #{temp_table_path} as #{last_select_hql}"
       Hive.run(cluster,temp_create_hql,user_name)
 
-      source_params = Hive.path_params(cluster, temp_table_path, user_name)
-      source_table_path = ['db','table'].map{|k| source_params[k]}.join(".")
-      source_table_stats = source_params['curr_stats']
+      source_table_stats = Hive.table_stats(cluster,temp_db,temp_table_name,user_name)
       source_fields = source_table_stats['field_defs']
 
       if part_array.length == 0 and
@@ -297,7 +280,7 @@ module Mobilize
 
         target_create_hql = "create table if not exists #{table_path} #{field_def_stmt};"
 
-        target_insert_hql = "insert overwrite table #{table_path} select #{target_field_stmt} from #{source_table_path};"
+        target_insert_hql = "insert overwrite table #{table_path} select #{target_field_stmt} from #{temp_table_path};"
 
         target_full_hql = [target_name_hql,
                            target_drop_hql,
@@ -353,7 +336,7 @@ module Mobilize
         else
           #get all the permutations of possible partititons
           part_set_hql = "set hive.cli.print.header=true;set mapred.job.name=#{job_name} (permutations);"
-          part_select_hql = "select distinct #{target_part_stmt} from #{source_table_path};"
+          part_select_hql = "select distinct #{target_part_stmt} from #{temp_table_path};"
           part_perm_hql = part_set_hql + part_select_hql
           part_perm_tsv = Hive.run(cluster, part_perm_hql, user_name)['stdout']
           #having gotten the permutations, ensure they are dropped
@@ -369,7 +352,7 @@ module Mobilize
 
         target_insert_hql = "insert overwrite table #{table_path} " +
                             "partition (#{target_part_stmt}) " +
-                            "select #{target_field_stmt},#{target_part_stmt} from #{source_table_path};"
+                            "select #{target_field_stmt},#{target_part_stmt} from #{temp_table_path};"
 
         target_full_hql = [target_set_hql, target_create_hql, target_insert_hql, temp_drop_hql].join
 
@@ -389,8 +372,7 @@ module Mobilize
       source_headers = source_tsv.tsv_header_array
 
       table_path = [db,table].join(".")
-      target_params = Hive.path_params(cluster, table_path, user_name)
-      table_stats = target_params['curr_stats']
+      table_stats = Hive.table_stats(cluster, db, table, user_name)
 
       schema_hash ||= {}
 
@@ -566,7 +548,7 @@ module Mobilize
                        end
                  {'stdout'=>url,'exit_code'=>0}
                rescue => exc
-                 {'stderr'=>exc.to_s, 'exit_code'=>500}
+                 {'stderr'=>"#{exc.to_s}\n#{exc.backtrace.join("\n")}", 'exit_code'=>500}
                end
 
       #unslot worker and write result

data/lib/mobilize-hive/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hive
-    VERSION = "1.22"
+    VERSION = "1.23"
   end
 end

data/mobilize-hive.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-hdfs","1.21"
+  gem.add_runtime_dependency "mobilize-hdfs","1.22"
 end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hive
 version: !ruby/object:Gem::Version
-  version: '1.22'
-  prerelease: 
+  version: '1.23'
 platform: ruby
 authors:
 - Cassio Paes-Leme
@@ -14,19 +13,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-hdfs
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.21'
+        version: '1.22'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.21'
+        version: '1.22'
 description: Adds hive read, write, and run support to mobilize-hdfs
 email:
 - cpaesleme@dena.com
@@ -54,33 +51,26 @@ files:
 - test/test_helper.rb
 homepage: http://github.com/dena/mobilize-hive
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -4582611704376704444
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -4582611704376704444
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Adds hive read, write, and run support to mobilize-hdfs
 test_files:
 - test/hive_job_rows.yml