mobilize-hive 1.299 → 1.351

data/lib/mobilize-hive/handlers/hive.rb

@@ -101,7 +101,7 @@ module Mobilize
       file_hash[filename] = hql
       #add in default params
       params ||= {}
-      params.merge(Ssh.default_params)
+      params = params.merge(Hive.default_params)
       #replace any params in the file_hash and command
       params.each do |k,v|
         file_hash.each do |name,data|
@@ -303,7 +303,7 @@ module Mobilize
 
         target_set_hql = ["set mapred.job.name=#{job_name};",
                           "set hive.exec.dynamic.partition.mode=nonstrict;",
-                          "set hive.exec.max.dynamic.partitions.pernode=1000;",
+                          "set hive.exec.max.dynamic.partitions.pernode=10000;",
                           "set hive.exec.dynamic.partition=true;",
                           "set hive.exec.max.created.files = 200000;",
                           "set hive.max.created.files = 200000;"].join
@@ -516,7 +516,7 @@ module Mobilize
           #source table
           cluster,source_path = source.path.split("/").ie{|sp| [sp.first, sp[1..-1].join(".")]}
           source_hql = "select * from #{source_path};"
-        elsif ['gsheet','gfile','gridfs','hdfs'].include?(source.handler)
+        else
           if source.path.ie{|sdp| sdp.index(/\.[A-Za-z]ql$/) or sdp.ends_with?(".ql")}
             source_hql = source.read(user_name,gdrive_slot)
           else
@@ -542,7 +542,8 @@ module Mobilize
 
       result = begin
                  url = if source_hql
-                         Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, job_name, drop, schema_hash)
+                         #include any params (or nil) at the end
+                         Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, job_name, drop, schema_hash,params['params'])
                        elsif source_tsv
                          Hive.tsv_to_table(cluster, db, table, part_array, source_tsv, user_name, drop, schema_hash)
                        elsif source

data/lib/mobilize-hive/helpers/hive_helper.rb

@@ -50,6 +50,14 @@ module Mobilize
     def self.databases(cluster,user_name)
       self.run(cluster,"show databases",user_name)['stdout'].split("\n")
     end
+
+    def self.default_params
+      time = Time.now.utc
+      {
+       '$utc_date'=>time.strftime("%Y-%m-%d"),
+       '$utc_time'=>time.strftime("%H:%M"),
+      }
+    end
   end
 end
 

data/lib/mobilize-hive/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hive
-    VERSION = "1.299"
+    VERSION = "1.351"
   end
 end

data/mobilize-hive.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-hdfs","1.298"
+  gem.add_runtime_dependency "mobilize-hdfs","1.351"
 end

data/test/hive_job_rows.yml

@@ -30,3 +30,5 @@
   status: ""
   stage1: hive.write source:"hive_test_4_stage_1.in", target:"mobilize/hive_test_1", partitions:"act_date"
   stage2: hive.write source:"hive_test_4_stage_2.in", target:"mobilize/hive_test_1", partitions:"act_date"
+  stage3: hive.run hql:"select '$utc_date $utc_time' as `date_time`,product,category,value from mobilize.hive_test_1;"
+  stage4: gsheet.write source:stage3, target:"hive_test_4.out"

data/test/mobilize-hive_test.rb

@@ -63,6 +63,8 @@ describe "Mobilize" do
     [hive_2_target_sheet].each{|s| s.delete if s}
     hive_3_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_3.out",gdrive_slot)
     [hive_3_target_sheet].each{|s| s.delete if s}
+    hive_4_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_4.out",gdrive_slot)
+    [hive_4_target_sheet].each{|s| s.delete if s}
 
     puts "job row added, force enqueued requestor, wait for stages"
     r.enqueue!
@@ -73,11 +75,13 @@ describe "Mobilize" do
     hive_1_stage_3_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_1_stage_3.out",gdrive_slot)
     hive_2_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_2.out",gdrive_slot)
     hive_3_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_3.out",gdrive_slot)
+    hive_4_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_4.out",gdrive_slot)
 
     assert hive_1_stage_2_target_sheet.read(u.name).length == 219
     assert hive_1_stage_3_target_sheet.read(u.name).length > 3
     assert hive_2_target_sheet.read(u.name).length == 599
     assert hive_3_target_sheet.read(u.name).length == 347
+    assert hive_4_target_sheet.read(u.name).length == 432
   end
 
   def wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hive
 version: !ruby/object:Gem::Version
-  version: '1.299'
+  version: '1.351'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-17 00:00:00.000000000 Z
+date: 2013-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-hdfs
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.298'
+        version: '1.351'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.298'
+        version: '1.351'
 description: Adds hive read, write, and run support to mobilize-hdfs
 email:
 - cpaesleme@dena.com
@@ -67,7 +67,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3388772007190329704
+      hash: -249273093332881580
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -76,7 +76,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3388772007190329704
+      hash: -249273093332881580
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25