mobilize-hive 1.0.07 → 1.0.08

data/README.md

@@ -102,6 +102,8 @@ in hadoop.yml. Each cluster has:
   * max_slots - defines the total number of simultaneous slots to be
     used for hive jobs on this cluster
   * output_db - defines the db which should be used to hold stage outputs.
+    * This db must have open permissions (777) so any user on the system can
+write to it -- the tables inside will be owned by the users themselves.
   * exec_path - defines the path to the hive executable
 
 Sample hive.yml:
@@ -227,3 +229,4 @@ Cassio Paes-Leme :: cpaesleme@dena.com :: @cpaesleme
 
 [mobilize-base]: https://github.com/dena/mobilize-base
 [mobilize-ssh]: https://github.com/dena/mobilize-ssh
+[mobilize-hdfs]: https://github.com/dena/mobilize-hdfs

data/lib/mobilize-hive/handlers/hive.rb

@@ -84,6 +84,8 @@ module Mobilize
 
     #run a generic hive command, with the option of passing a file hash to be locally available
     def Hive.run(hql,cluster,user,file_hash=nil)
+      # no TempStatsStore
+      hql = "set hive.stats.autogather=false;#{hql}"
       filename = hql.to_md5
       file_hash||= {}
       file_hash[filename] = hql
@@ -101,7 +103,6 @@ module Mobilize
       user = params['user']
       cluster = params['cluster'] || Hive.clusters.keys.first
       node = Hadoop.gateway_node(cluster)
-      node_user = Ssh.host(node)['user']
       if user and !Ssh.sudoers(node).include?(u.name)
         raise "#{u.name} does not have su permissions for #{node}"
       elsif user.nil? and Ssh.su_all_users(node)
@@ -140,17 +141,12 @@ module Mobilize
                             "create table #{output_path} as #{select_hql};"].join(";")
         full_hql = [prior_hql, output_table_hql].compact.join(";")
         Hive.run(full_hql, cluster, user)
-        #make sure node user owns the stage result directory
-        output_table_stats = Hive.table_stats(output_db,output_table,cluster,node_user)
-        output_table_location = output_table_stats['location']
-        chown_command = "#{Hadoop.exec_path(cluster)} fs -chown -R #{node_user} '#{output_table_location}'"
-        Ssh.run(node,chown_command,node_user)
         #already populated, make sure dataset exists
         Dataset.find_or_create_by_url(out_url)
       else
         out_string = Hive.run(hql, cluster, user)
         out_string = "result\n#{out_string}"
-        Dataset.write_by_url(out_url,out_string,node_user)
+        Dataset.write_by_url(out_url,out_string,user)
       end
       #unslot worker
       Hive.unslot_worker_by_path(stage_path)
@@ -442,7 +438,6 @@ module Mobilize
       return false unless slot_id
 
       node = Hadoop.gateway_node(cluster)
-      node_user = Ssh.host(node)['user']
       if user and !Ssh.sudoers(node).include?(u.name)
         raise "#{u.name} does not have su permissions for #{node}"
       elsif user.nil? and Ssh.su_all_users(node)
@@ -498,7 +493,7 @@ module Mobilize
       out_string = "result\n#{out_string}"
       output_db,output_table = [Hive.output_db(cluster),stage_path.gridsafe]
       out_url = "hive://#{cluster}/#{output_db}/#{output_table}"
-      Dataset.write_by_url(out_url,out_string,node_user)
+      Dataset.write_by_url(out_url,out_string,user)
       out_url
     end
 

data/lib/mobilize-hive/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hive
-    VERSION = "1.0.07"
+    VERSION = "1.0.08"
   end
 end

data/mobilize-hive.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-hdfs","1.0.07"
+  gem.add_runtime_dependency "mobilize-hdfs","1.0.08"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hive
 version: !ruby/object:Gem::Version
-  version: 1.0.07
+  version: 1.0.08
   prerelease: 
 platform: ruby
 authors:
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.07
+        version: 1.0.08
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.07
+        version: 1.0.08
 description: Adds hive read, write, and run support to mobilize-hdfs
 email:
 - cpaesleme@ngmoco.com