patriot-hadoop 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 21e992800beb77008e49114125a7509e42820106
-  data.tar.gz: fc571743a6fdb51e62e1b02f1ee2337e330b0f30
+  metadata.gz: 3a75278502a8dd0d24ffca6600f006728ca4b97e
+  data.tar.gz: be5d6721503438d426a33b466bd9ccd7e6385c03
 SHA512:
-  metadata.gz: 993f0ad53ac7b18c264fa11d52bd0e0ab2c7fa15b9a9b805d99bfd05985e7c49d80dc5459058ec75bfc2339ca79edf9e55180a081c2785868e800b78391f4d23
-  data.tar.gz: 537a5e6a57c39727a57bb8f909006fde86d6f3ad420eb78818b430e6a1ac0adc0e4d850ed4b5ee3c3b132bc1f6d468e00bc0fee212e71117cdd720ac80ae76d9
+  metadata.gz: 60b66f64532ad327acfa9a5804b1085788f4f07e2ac87b0cb8d2c210553287c8296d117add0ada7e60fbd4859f13a56109a27b3678e16f9667b8c48acaa037f7
+  data.tar.gz: d5ba491a269f4d55789c090fdb8eb88cd5c57a9b13e6cd6ff118c3c90b9781ede959db5720519733860d7c96fd2daa943933651c1423a924717a0d94574c3cb1

data/lib/patriot_hadoop/command/hive.rb

@@ -4,7 +4,7 @@ module PatriotHadoop
       declare_command_name :hive
       include PatriotHadoop::Ext::Hive
 
-      command_attr :hive_ql, :output_prefix, :exec_user, :props, :name_suffix
+      command_attr :hive_ql, :output_prefix, :compression, :exec_user, :props, :name_suffix
 
       def job_id
         job_id = "#{command_name}"
@@ -28,8 +28,8 @@ module PatriotHadoop
 
         tmpfile = output_prefix + '.hql'
         _create_hivequery_tmpfile(@hive_ql, tmpfile, opt)
+        output_file = _create_output_filename(output_prefix, @compression)
 
-        output_file = output_prefix + '.tsv'
         execute_hivequery(tmpfile, output_file, @exec_user)
 
         if File.zero?(output_file)
@@ -41,6 +41,19 @@ module PatriotHadoop
       end
 
 
+      # true / 'gzip' / 'bzip2' are available.
+      def _create_output_filename(output_prefix, compression)
+        output_file = output_prefix + '.tsv'
+        case compression
+        when true, 'gzip'
+          output_file += '.gz'
+        when 'bzip2'
+          output_file += '.bz2'
+        end
+        return output_file
+      end
+
+
       def _create_hivequery_tmpfile(hive_ql, tmpfile, opt={})
         hive_ql = _add_udfs(hive_ql, opt[:udf]) if opt.has_key?(:udf)
         hive_ql = "#{_set_hive_property_prefix(opt[:props])}#{hive_ql}" if opt.has_key?(:props)

data/lib/patriot_hadoop/ext/hive.rb

@@ -16,7 +16,7 @@ module PatriotHadoop
 
 
       def execute_hivequery(hql_file, output_file=nil, user=nil)
-        command = "hive -f \"#{hql_file}\""
+        command = "hive -f \"#{hql_file}\"#{_compress_option(File.extname(output_file))}"
         unless user.nil?
           if user !~ /^[a-z_][a-z0-9_]{0,30}$/
             raise HiveException, "Invalid username" 
@@ -27,8 +27,17 @@ module PatriotHadoop
       end
 
 
+      def _compress_option(extension)
+        return case extension
+          when '.gz'  then ' | gzip --stdout --force'
+          when '.bz2' then ' | bzip2 --stdout --force'
+          else ''
+        end
+      end
+
+
       def _execute_hivequery_internal(command, output_file)
-        so = execute_command(command) do |status, so, se|
+        res = execute_command(command) do |status, so, se|
           err_size = File.stat(se).size
           err_msg  = ""
           max_err_size = HIVE_MAX_ERROR_MSG_SIZE
@@ -41,7 +50,7 @@ module PatriotHadoop
           end
           raise HiveException, "#{command}\n#{err_msg}"
         end
-        File.rename(so, output_file) unless output_file.nil?
+        File.rename(res, output_file) unless output_file.nil?
       end
 
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: patriot-hadoop
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Hitoshi Tsuda
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-19 00:00:00.000000000 Z
+date: 2018-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: patriot-workflow-scheduler
@@ -38,9 +38,9 @@ files:
 - lib/patriot_hadoop/ext.rb
 - lib/patriot_hadoop.rb
 - init.rb
-homepage: https://github.com/CyberAgent/patriot-workflow-scheduler
+homepage: https://github.com/CyberAgent/patriot-workflow-scheduler/tree/master/plugins/patriot-hadoop
 licenses:
-- Apache License, Version 2.0
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -58,9 +58,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: patriot-hadoop
-rubygems_version: 2.0.14
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: Hadoop plugin for Patriot Workflow Scheduler
 test_files: []
-has_rdoc: