mobilize-hive 1.292 → 1.294

data/lib/mobilize-hive/handlers/hive.rb

@@ -306,7 +306,7 @@ module Mobilize
         raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
 
       elsif part_array.length > 0 and
-        table_stats.ie{|tts| tts.nil? || drop || tts['partitions'].to_a.map{|p| p['name']} == part_array}
+        table_stats.ie{|tts| tts.nil? || drop || tts['partitions'].to_a.map{|p| p['name']}.sort == part_array.sort}
         #partitions and no target table or same partitions in both target table and user params
 
         target_headers = source_fields.map{|f| f['name']}.reject{|h| part_array.include?(h)}
@@ -387,6 +387,7 @@ module Mobilize
     #Accepts options to drop existing target if any
     #also schema with column datatype overrides
     def Hive.tsv_to_table(cluster, db, table, part_array, source_tsv, user_name, drop=false, schema_hash=nil)
+      return nil if source_tsv.strip.length==0
       source_headers = source_tsv.tsv_header_array
 
       table_path = [db,table].join(".")
@@ -394,6 +395,8 @@ module Mobilize
 
       schema_hash ||= {}
 
+      url = "hive://" + [cluster,db,table,part_array.compact.join("/")].join("/")
+
       if part_array.length == 0 and
         table_stats.ie{|tts| tts.nil? || drop || tts['partitions'].nil?}
         #no partitions in either user params or the target table
@@ -424,7 +427,7 @@ module Mobilize
         raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
 
       elsif part_array.length > 0 and
-        table_stats.ie{|tts| tts.nil? || drop || tts['partitions'].to_a.map{|p| p['name']} == part_array}
+        table_stats.ie{|tts| tts.nil? || drop || tts['partitions'].to_a.map{|p| p['name']}.sort == part_array.sort}
         #partitions and no target table
         #or same partitions in both target table and user params
         #or drop and start fresh
@@ -451,11 +454,14 @@ module Mobilize
         response = Hive.run(cluster, target_create_hql, user_name)
         raise response['stderr'] if response['stderr'].to_s.ie{|s| s.index("FAILED") or s.index("KILLED")}
 
+        #return url (operation complete) if there's no data
+        source_hash_array = source_tsv.tsv_to_hash_array
+        return url if source_hash_array.length==1 and source_hash_array.first.values.compact.length==0
+
         table_stats = Hive.table_stats(cluster, db, table, user_name)
 
         #create data hash from source hash array
         data_hash = {}
-        source_hash_array = source_tsv.tsv_to_hash_array
         source_hash_array.each do |ha|
           tpmk = part_array.map{|pn| "#{pn}=#{ha[pn]}"}.join("/")
           tpmv = ha.reject{|k,v| part_array.include?(k)}.values.join("\001")
@@ -498,7 +504,6 @@ module Mobilize
         raise error_msg
       end
 
-      url = "hive://" + [cluster,db,table,part_array.compact.join("/")].join("/")
       return url
     end
 
@@ -564,6 +569,8 @@ module Mobilize
                          Hive.hql_to_table(cluster, db, table, part_array, source_hql, user_name, job_name, drop, schema_hash)
                        elsif source_tsv
                          Hive.tsv_to_table(cluster, db, table, part_array, source_tsv, user_name, drop, schema_hash)
+                       elsif source
+                         #null sheet
                        else
                          raise "Unable to determine source tsv or source hql"
                        end

data/lib/mobilize-hive/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hive
-    VERSION = "1.292"
+    VERSION = "1.294"
   end
 end

data/mobilize-hive.gemspec

@@ -16,5 +16,5 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-hdfs","1.292"
+  gem.add_runtime_dependency "mobilize-hdfs","1.294"
 end

data/test/hive_job_rows.yml

@@ -24,3 +24,9 @@
   stage2: hive.write source:"stage1",target:"mobilize/hive_test_3", partitions:"date/product", drop:true
   stage3: hive.write hql:"select * from mobilize.hive_test_3;",target:"mobilize/hive_test_3", partitions:"date/product", drop:false
   stage4: gsheet.write source:"hive://mobilize/hive_test_3", target:"hive_test_3.out"
+- name: hive_test_4
+  active: true
+  trigger: after hive_test_2
+  status: ""
+  stage1: hive.write source:"hive_test_4_stage_1.in", target:"mobilize/hive_test_1", partitions:"act_date"
+  stage2: hive.write source:"hive_test_4_stage_2.in", target:"mobilize/hive_test_1", partitions:"act_date"

data/test/mobilize-hive_test.rb

@@ -25,6 +25,18 @@ describe "Mobilize" do
     hive_1_in_tsv = YAML.load_file("#{Mobilize::Base.root}/test/hive_test_1_in.yml").hash_array_to_tsv
     hive_1_in_sheet.write(hive_1_in_tsv,Mobilize::Gdrive.owner_name)
 
+    #create blank sheet
+    hive_4_stage_1_in_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_4_stage_1.in",gdrive_slot)
+    [hive_4_stage_1_in_sheet].each {|s| s.delete if s}
+    hive_4_stage_1_in_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_4_stage_1.in",gdrive_slot)
+
+    #create sheet w just headers
+    hive_4_stage_2_in_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_4_stage_2.in",gdrive_slot)
+    [hive_4_stage_2_in_sheet].each {|s| s.delete if s}
+    hive_4_stage_2_in_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_4_stage_2.in",gdrive_slot)
+    hive_4_stage_2_in_sheet_header = hive_1_in_tsv.tsv_header_array.join("\t")
+    hive_4_stage_2_in_sheet.write(hive_4_stage_2_in_sheet_header,Mobilize::Gdrive.owner_name)
+
     hive_1_schema_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_1.schema",gdrive_slot)
     [hive_1_schema_sheet].each {|s| s.delete if s}
     hive_1_schema_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_1.schema",gdrive_slot)
@@ -54,7 +66,7 @@ describe "Mobilize" do
 
     puts "job row added, force enqueued requestor, wait for stages"
     r.enqueue!
-    wait_for_stages(1200)
+    wait_for_stages(2100)
 
     puts "jobtracker posted data to test sheet"
     hive_1_stage_2_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/hive_test_1_stage_2.out",gdrive_slot)

metadata

@@ -1,29 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hive
 version: !ruby/object:Gem::Version
-  version: '1.292'
+  version: '1.294'
+  prerelease: 
 platform: ruby
 authors:
 - Cassio Paes-Leme
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-28 00:00:00.000000000 Z
+date: 2013-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-hdfs
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.292'
+        version: '1.294'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.292'
+        version: '1.294'
 description: Adds hive read, write, and run support to mobilize-hdfs
 email:
 - cpaesleme@dena.com
@@ -51,26 +54,33 @@ files:
 - test/test_helper.rb
 homepage: http://github.com/dena/mobilize-hive
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2996483111251873179
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2996483111251873179
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.25
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Adds hive read, write, and run support to mobilize-hdfs
 test_files:
 - test/hive_job_rows.yml

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: c6d52c5de86351ccf5ab3c4da09b7341f27e5163
-  data.tar.gz: 3e560d834babc1377d8e3c6c80b799e9c4655acf
-SHA512:
-  metadata.gz: d910e35cefda69f9640105949454b04b6d08df9069e2e9fdb61b9875806c921fb187701977afee7efcc3a5b84dadbbd91abf4d6bf9b4817bcf4f3c1945effbfb
-  data.tar.gz: 9a41441ce0ddf76b81fe63350a2ece3d1cc5ab0daccb1bc09f6db394dcb618078c4d01c1893d3a16090656dfdd36ec5fa8332aa5a699d096abcd85b726a1efa2