mandy 0.4.994 → 0.4.995

data/VERSION

@@ -1 +1 @@
-0.4.994
+0.4.995

data/bin/mandy-hadoop

@@ -5,6 +5,7 @@ require 'optparse'
 require 'ostruct'
 require 'cgi'
 
+errors = false
 options = OpenStruct.new
 
 OptionParser.new do |opts|
@@ -53,24 +54,26 @@ inputs  = ARGV[1].split(",")
 input = inputs.map {|path| "-input \"#{path}\""}.join(" ")
 
 output_folder = ARGV[2]
-config = options.config || 'cluster.xml'
-puts "Packaging Gems for distribution..."
+config = absolute_path(options.config || 'cluster.xml')
+puts "Packaging code for distribution..."
 payload = Mandy::Packer.pack(file, options.payload || ARGV[0], gemfile(options.gemfile))
 cmdenv = options.cmdenv
 
 at_exit do
+  puts
   puts "Cleaning up..."
   Mandy::Packer.cleanup!(payload)
-  puts "All done!"
+  puts errors ? "Completed with errors!" : "Completed Successfully!"
 end
 
 puts "Loading Mandy scripts..."
 require absolute_path(file)
 
 output = nil
+puts
 begin
   Mandy::Job.jobs.each_with_index do |job, i|
-    puts "Running Job [#{i+1}] #{job.name}..."
+    puts "Submitting Job: [#{i+1}] #{job.name}..."
     
     jobconf = job.settings.map { |key, value| %(-D #{key}='#{value}') }.join(' ')
     output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
@@ -88,15 +91,28 @@ begin
                     #{ cmdenv.nil? ? '' : "-cmdenv #{cmdenv}" }\
                     -output "#{output}" 2>&1)
 
-    result = `#{command}`
-    raise(Mandy::HadoopJobFailure.new(job, result)) unless $?.to_i==0
-
+    result = []
+    IO.popen(command, 'r') do |subprocess|
+      while line = subprocess.gets
+        if line.include?('Running job:')
+          job_id = line.split(' ').last.strip
+          puts "Job ID: #{job_id}"
+          puts "Kill Command: mandy-kill #{job_id} -c #{config}"         
+        end
+        puts "Tracking URL: #{line.split(' ').last.strip}" if line.include?('Tracking URL:')
+        result << line
+      end
+    end
+    
+    raise(Mandy::HadoopJobFailure.new(job, result.join("\n"))) unless $?.to_i==0
+    puts
     # puts "#{command}"
     input = "-input #{output}"
   end
   # print out the output location so caller can know where to get the results from
   puts output
 rescue Mandy::HadoopJobFailure => e
+  errors = true
   STDERR.puts e.to_s
   exit(1)
 end

data/bin/mandy-kill

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'ostruct'
+
+exec('mandy-kill -h') unless ARGV.size >= 1
+
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: mandy-kill job [options]"
+
+  opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config|
+    options.config = config
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+job = ARGV[0]
+config = options.config || 'cluster.xml'
+
+`$HADOOP_HOME/bin/hadoop job -conf #{config} -kill #{job}`

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.4.994
+  version: 0.4.995
 platform: ruby
 authors: 
 - Andy Kent
@@ -27,6 +27,7 @@ description: Mandy is Ruby Map/Reduce Framework built onto of the Hadoop Distrib
 email: andy.kent@me.com
 executables: 
 - mandy
+- mandy-kill
 - mandy-hadoop
 - mandy-local
 - mandy-map
@@ -48,6 +49,7 @@ files:
 - bin/mandy
 - bin/mandy-hadoop
 - bin/mandy-local
+- bin/mandy-kill
 - bin/mandy-map
 - bin/mandy-put
 - bin/mandy-get