patriot-hadoop 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 21e992800beb77008e49114125a7509e42820106
+  data.tar.gz: fc571743a6fdb51e62e1b02f1ee2337e330b0f30
+SHA512:
+  metadata.gz: 993f0ad53ac7b18c264fa11d52bd0e0ab2c7fa15b9a9b805d99bfd05985e7c49d80dc5459058ec75bfc2339ca79edf9e55180a081c2785868e800b78391f4d23
+  data.tar.gz: 537a5e6a57c39727a57bb8f909006fde86d6f3ad420eb78818b430e6a1ac0adc0e4d850ed4b5ee3c3b132bc1f6d468e00bc0fee212e71117cdd720ac80ae76d9

data/init.rb

@@ -0,0 +1,3 @@
+require 'patriot'
+require 'patriot_hadoop'
+

data/lib/patriot_hadoop.rb

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+require 'patriot_hadoop/ext'
+require 'patriot_hadoop/command'
+

data/lib/patriot_hadoop/command.rb

@@ -0,0 +1 @@
+require "patriot_hadoop/command/hive"

data/lib/patriot_hadoop/command/hive.rb

@@ -0,0 +1,74 @@
+module PatriotHadoop
+  module Command
+    class HiveCommand < Patriot::Command::Base
+      declare_command_name :hive
+      include PatriotHadoop::Ext::Hive
+
+      command_attr :hive_ql, :output_prefix, :exec_user, :props, :name_suffix
+
+      def job_id
+        job_id = "#{command_name}"
+        job_id = "#{job_id}_#{@name_suffix}" unless @name_suffix.nil?
+        return job_id
+      end
+
+
+      def execute
+        @logger.info "start hive"
+
+        opt = {}
+        opt[:udf] = @udf unless @udf.nil?
+        opt[:props] = @props unless @props.nil?
+
+        output_prefix = @output_prefix.nil? ? File.join('/tmp', job_id()) : @output_prefix
+        output_directory = File.dirname(output_prefix)
+        if not Dir.exist?(output_directory)
+          FileUtils.mkdir_p(output_directory)
+        end
+
+        tmpfile = output_prefix + '.hql'
+        _create_hivequery_tmpfile(@hive_ql, tmpfile, opt)
+
+        output_file = output_prefix + '.tsv'
+        execute_hivequery(tmpfile, output_file, @exec_user)
+
+        if File.zero?(output_file)
+          @logger.warn "#{@hive_ql} generated empty result"
+          return
+        end
+
+        @logger.info "end hive"
+      end
+
+
+      def _create_hivequery_tmpfile(hive_ql, tmpfile, opt={})
+        hive_ql = _add_udfs(hive_ql, opt[:udf]) if opt.has_key?(:udf)
+        hive_ql = "#{_set_hive_property_prefix(opt[:props])}#{hive_ql}" if opt.has_key?(:props)
+        File.write(tmpfile, hive_ql)
+      end
+
+
+      def _set_hive_property_prefix(props={})
+        return "" if props.nil?
+        return props.map{|k,v| "set #{k}=#{v};"}.join
+      end
+
+
+      def _add_udfs(hive_ql, udfs)
+        return hive_ql if udfs.nil?
+        register = ""
+        udfs = [udfs] unless udfs.is_a?(Array)
+        udfs.each do |udf|
+          register += "add jar #{udf['jar']};"
+          functions = udf['functions']
+          functions = [functions] unless functions.is_a?(Array)
+          functions.each do |f|
+            register += "create temporary function #{f['name']} as \"#{f['class']}\";"
+          end
+        end
+        return "#{register}#{hive_ql}"
+      end
+
+    end
+  end
+end

data/lib/patriot_hadoop/ext.rb

@@ -0,0 +1 @@
+require "patriot_hadoop/ext/hive"

data/lib/patriot_hadoop/ext/hive.rb

@@ -0,0 +1,49 @@
+module PatriotHadoop
+  module Ext
+    module Hive
+
+      HIVE_MAX_ERROR_MSG_SIZE = 512
+
+      include Patriot::Util::Logger
+      include Patriot::Util::DBClient
+      include Patriot::Util::System
+
+      def self.included(cls)
+        cls.send(:include, Patriot::Util::System)
+      end
+
+      class HiveException < Exception; end
+
+
+      def execute_hivequery(hql_file, output_file=nil, user=nil)
+        command = "hive -f \"#{hql_file}\""
+        unless user.nil?
+          if user !~ /^[a-z_][a-z0-9_]{0,30}$/
+            raise HiveException, "Invalid username" 
+          end
+          command = "sudo -u #{user} #{command}"
+        end
+        return _execute_hivequery_internal(command, output_file)
+      end
+
+
+      def _execute_hivequery_internal(command, output_file)
+        so = execute_command(command) do |status, so, se|
+          err_size = File.stat(se).size
+          err_msg  = ""
+          max_err_size = HIVE_MAX_ERROR_MSG_SIZE
+          File.open(se) do |f|
+            if err_size > max_err_size
+              f.seek(-1 * max_err_size, IO::SEEK_END)
+              err_msg = "\n(#{err_size - max_err_size} bytes are truncated)"
+            end
+            err_msg = "#{f.read}#{err_msg}"
+          end
+          raise HiveException, "#{command}\n#{err_msg}"
+        end
+        File.rename(so, output_file) unless output_file.nil?
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: patriot-hadoop
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Hitoshi Tsuda
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: patriot-workflow-scheduler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+description: a plugin for Patriot Workflow Scheduler, which deal with Hadoop-related
+  softwares.
+email:
+- tsuda_hitoshi@cyberagent.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/patriot_hadoop/command/hive.rb
+- lib/patriot_hadoop/command.rb
+- lib/patriot_hadoop/ext/hive.rb
+- lib/patriot_hadoop/ext.rb
+- lib/patriot_hadoop.rb
+- init.rb
+homepage: https://github.com/CyberAgent/patriot-workflow-scheduler
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: patriot-hadoop
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Hadoop plugin for Patriot Workflow Scheduler
+test_files: []
+has_rdoc: