mobilize-hive 1.376 → 1.377

data/lib/mobilize-hive/handlers/hive.rb

@@ -211,7 +211,7 @@ module Mobilize
       schema_hash
     end
 
-    def Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, stage_path, drop=false, schema_hash=nil, run_params=nil)
+    def Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, stage_path, drop=false, schema_hash=nil, run_params=nil,compress=false)
       job_name = stage_path.sub("Runner_","")
       table_path = [db,table].join(".")
       table_stats = Hive.table_stats(cluster, db, table, user_name)
@@ -261,6 +261,12 @@ module Mobilize
         #always drop when no partititons
         target_name_hql = "set mapred.job.name=#{job_name};"
 
+        if compress
+          target_name_hql = target_name_hql+["set hive.exec.compress.output=true;",
+                          "set mapred.output.compression.codec=org.apache.hadoop.io.compress.SnappyCodec;",
+                          "set mapred.output.compression.type=BLOCK;"].join
+        end
+
         target_drop_hql = "drop table if exists #{table_path};"
 
         target_create_hql = "create table if not exists #{table_path} #{field_def_stmt};"
@@ -273,6 +279,10 @@ module Mobilize
                            target_insert_hql,
                            temp_drop_hql].join
 
+
+        puts "FULL HQL QUERY: " + target_full_hql
+
+        
         response = Hive.run(cluster, target_full_hql, user_name, run_params)
 
         raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
@@ -313,6 +323,11 @@ module Mobilize
                           "set hive.exec.dynamic.partition=true;",
                           "set hive.exec.max.created.files = 200000;",
                           "set hive.max.created.files = 200000;"].join
+        if compress
+          target_set_hql = target_set_hql+["set hive.exec.compress.output=true;",
+                          "set mapred.output.compression.codec=org.apache.hadoop.io.compress.SnappyCodec;",
+                          "set mapred.output.compression.type=BLOCK;"].join
+        end
 
         if drop or table_stats.nil?
           target_drop_hql = "drop table if exists #{table_path};"
@@ -352,6 +367,8 @@ module Mobilize
 
         target_full_hql = [target_set_hql, target_create_hql, target_insert_hql, temp_drop_hql].join
 
+        puts "FULL HQL QUERY: " + target_full_hql
+
         response = Hive.run(cluster, target_full_hql, user_name, run_params)
         raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
       else
@@ -426,6 +443,7 @@ module Mobilize
                     end
       #drop target before create/insert?
       drop = params['drop']
+      compress = params['compress']
 
       #determine source
       source_tsv,source_hql = [nil]*2
@@ -464,7 +482,7 @@ module Mobilize
                  url = if source_hql
                          #include any params (or nil) at the end
                          run_params = params['params']
-                         Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, stage_path,drop, schema_hash,run_params)
+                         Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, stage_path,drop, schema_hash,run_params,compress)
                        elsif source_tsv
                          #first write tsv to temp table
                          temp_table_path = "#{Hive.output_db(cluster)}.temptsv_#{job_name.downcase.alphanunderscore}"
@@ -472,7 +490,7 @@ module Mobilize
                          if has_data
                            #then do the regular insert, with source hql being select * from temp table
                            source_hql = "select * from #{temp_table_path}"
-                           Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, stage_path, drop, schema_hash)
+                           Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, stage_path, drop, schema_hash,nil,compress)
                          else
                            nil
                          end

data/lib/mobilize-hive/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hive
-    VERSION = "1.376"
+    VERSION = "1.377"
   end
 end

data/mobilize-hive.gemspec

@@ -17,5 +17,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-hdfs","1.376"
+  gem.add_runtime_dependency "mobilize-hdfs","1.377"
 end

data/test/fixtures/hive5.in.yml

@@ -0,0 +1,9 @@
+---
+- act_date: 2013-01-01
+  product: product1
+  category: category1
+  value: 7
+- act_date: 2013-01-02
+  product: product1
+  category: category1
+  value: 8

data/test/fixtures/integration_expected.yml

@@ -67,3 +67,23 @@
   state: working
   count: 1
   confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/hive5/stage1"
+  state: working
+  count: 1
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/hive5/stage2"
+  state: working
+  count: 1
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/hive5/stage3"
+  state: working
+  count: 1
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/hive5/stage4"
+  state: working
+  count: 1
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/hive5/stage5"
+  state: working
+  count: 1
+  confirmed_ats: []

data/test/fixtures/integration_jobs.yml

@@ -13,7 +13,7 @@
   active: true
   trigger: after hive1
   status: ""
-  stage1: hive.write save_logs:true, retries:3, source:"hdfs://user/mobilize/test/hdfs1.out", target:"mobilize.hive2", drop:true
+  stage1: hive.write save_logs:true, retries:3, source:"hdfs://user/mobilize/test/hdfs1.out", target:"mobilize.hive2", drop:true, compress:true
   stage2: hive.run save_logs:true, retries:3, hql:"select * from mobilize.hive2;"
   stage3: gsheet.write source:"stage2", target:"hive2.out"
 - name: hive3
@@ -22,7 +22,7 @@
   status: ""
   stage1: hive.run save_logs:true, retries:3, hql:"select '@date' as `date`,product,category,value from mobilize.hive1;", params:{'date':'2013-01-01'}
   stage2: hive.write save_logs:true, retries:3, source:"stage1",target:"mobilize/hive3", partitions:"date/product", drop:true
-  stage3: hive.write save_logs:true, retries:3, hql:"select * from mobilize.hive3;",target:"mobilize/hive3", partitions:"date/product", drop:false
+  stage3: hive.write save_logs:true, retries:3, hql:"select * from mobilize.hive3;",target:"mobilize/hive3", partitions:"date/product", drop:false, compress:true
   stage4: gsheet.write source:"hive://mobilize/hive3", target:"hive3.out"
 - name: hive4
   active: true
@@ -32,3 +32,12 @@
   stage2: hive.write save_logs:true, retries:3, source:"hive4_stage2.in", target:"mobilize/hive1", partitions:"act_date"
   stage3: hive.run save_logs:true, retries:3, hql:"select '@date $utc_time' as `date_time`,product,category,value from mobilize.hive1;", params:{'date':'$utc_date'}
   stage4: gsheet.write source:stage3, target:"hive4.out"
+- name: hive5
+  active: true
+  trigger: after hive4
+  status: ""
+  stage1: hive.write save_logs:true, retries:3, source:"hive5.in", target:"mobilize/hive5_in", drop:true
+  stage2: hive.write save_logs:true, retries:3, hql:"select act_date,product,category,value from mobilize.hive5_in where act_date='@date'", target:"mobilize/hive5", partitions:"act_date", params:{'date':'2013-01-01'}, drop:true
+  stage3: hive.write save_logs:true, retries:3, hql:"select act_date,product,category,value from mobilize.hive5_in where act_date='@date'", target:"mobilize/hive5", partitions:"act_date", params:{'date':'2013-01-02'}, compress:true
+  stage4: hive.run save_logs:true, retries:3, hql:"select act_date,product,category,value from mobilize.hive5;"
+  stage5: gsheet.write source:stage4, target:"hive5.out"

data/test/integration/mobilize-hive_test.rb

@@ -20,7 +20,7 @@ describe "Mobilize" do
     assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
 
     puts "add test data"
-    ["hive1.in","hive4_stage1.in","hive4_stage2.in","hive1.schema","hive1.sql"].each do |fixture_name|
+    ["hive1.in","hive4_stage1.in","hive4_stage2.in","hive1.schema","hive1.sql", "hive5.in"].each do |fixture_name|
       target_url = "gsheet://#{r.title}/#{fixture_name}"
       TestHelper.write_fixture(fixture_name, target_url, 'replace')
     end
@@ -47,5 +47,6 @@ describe "Mobilize" do
     assert TestHelper.check_output("gsheet://#{r.title}/hive2.out", 'min_length' => 599) == true
     assert TestHelper.check_output("gsheet://#{r.title}/hive3.out", 'min_length' => 347) == true
     assert TestHelper.check_output("gsheet://#{r.title}/hive4.out", 'min_length' => 432) == true
+    assert TestHelper.check_output("gsheet://#{r.title}/hive5.out", 'min_length' => 500) == true
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hive
 version: !ruby/object:Gem::Version
-  version: '1.376'
+  version: '1.377'
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-21 00:00:00.000000000 Z
+date: 2013-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-hdfs
@@ -19,7 +19,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.376'
+        version: '1.377'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,7 +27,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.376'
+        version: '1.377'
 description: Adds hive read, write, and run support to mobilize-hdfs
 email:
 - cpaesleme@dena.com
@@ -54,6 +54,7 @@ files:
 - test/fixtures/hive1.sql
 - test/fixtures/hive4_stage1.in
 - test/fixtures/hive4_stage2.in.yml
+- test/fixtures/hive5.in.yml
 - test/fixtures/integration_expected.yml
 - test/fixtures/integration_jobs.yml
 - test/integration/mobilize-hive_test.rb
@@ -91,6 +92,7 @@ test_files:
 - test/fixtures/hive1.sql
 - test/fixtures/hive4_stage1.in
 - test/fixtures/hive4_stage2.in.yml
+- test/fixtures/hive5.in.yml
 - test/fixtures/integration_expected.yml
 - test/fixtures/integration_jobs.yml
 - test/integration/mobilize-hive_test.rb