mandy 0.4.98 → 0.4.99

data/Rakefile

@@ -17,7 +17,7 @@ namespace :gem do
   end
 
   task :install => :build do
-    `sudo gem install pkg/mandy-*.gem`
+    `sudo gem install pkg/mandy-*.gem --no-rdoc --no-ri`
   end
   
   task :push => :build do

data/VERSION

@@ -1 +1 @@
-0.4.98
+0.4.99

data/bin/mandy-hadoop

@@ -56,7 +56,7 @@ puts "Packaging Gems for distribution..."
 payload = Mandy::Packer.pack(file, options.payload || ARGV[0], gemfile(options.gemfile))
 cmdenv = options.cmdenv
 
-at_exit do 
+at_exit do
   puts "Cleaning up..."
   Mandy::Packer.cleanup!(payload)
   puts "All done!"
@@ -66,32 +66,35 @@ puts "Loading Mandy scripts..."
 require absolute_path(file)
 
 output = nil
-
-puts "Sending jobs to Hadoop..."
-
-Mandy::Job.jobs.each_with_index do |job, i|
-  
-  jobconf = job.settings.map { |key, value| %(-D #{key}='#{value}') }.join(' ')
-  output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
+begin
+  Mandy::Job.jobs.each_with_index do |job, i|
+    puts "Running Job [#{i+1}] #{job.name}..."
+    
+    jobconf = job.settings.map { |key, value| %(-D #{key}='#{value}') }.join(' ')
+    output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
   
-  bootstrap_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'bootstrap.rb'))
-  inputreader = job.input_format == :xml ? "StreamXmlRecordReader,begin=<#{job.input_format_options[:xml_tag]} ,end=</#{job.input_format_options[:xml_tag]}>" : nil
+    bootstrap_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'bootstrap.rb'))
+    inputreader = job.input_format == :xml ? "StreamXmlRecordReader,begin=<#{job.input_format_options[:xml_tag]} ,end=</#{job.input_format_options[:xml_tag]}>" : nil
   
-  command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar #{jobconf}\
-                  -files "#{payload}","#{bootstrap_file}" \
-                  -conf '#{config}' \
-                  -input "#{input}"  \
-                  #{ inputreader.nil? ? '' : "-inputreader \"#{inputreader}\"" } \
-                  -mapper "ruby bootstrap.rb #{File.basename(payload)} map #{filename} '#{job.name}'"  \
-                  -reducer "ruby bootstrap.rb #{File.basename(payload)} reduce #{filename} '#{job.name}'"  \
-                  #{ cmdenv.nil? ? '' : "-cmdenv #{cmdenv}" }\
-                  -output "#{output}")
+    command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar #{jobconf}\
+                    -files "#{payload}","#{bootstrap_file}" \
+                    -conf '#{config}' \
+                    -input "#{input}"  \
+                    #{ inputreader.nil? ? '' : "-inputreader \"#{inputreader}\"" } \
+                    -mapper "ruby bootstrap.rb #{File.basename(payload)} map #{filename} '#{job.name}'"  \
+                    -reducer "ruby bootstrap.rb #{File.basename(payload)} reduce #{filename} '#{job.name}'"  \
+                    #{ cmdenv.nil? ? '' : "-cmdenv #{cmdenv}" }\
+                    -output "#{output}" 2>&1)
 
-  `#{command}`
+    result = `#{command}`
+    raise(Mandy::HadoopJobFailure.new(job, result)) unless $?.to_i==0
 
-  # puts "#{command}"
-  input = output
-end
-
-# print out the output location so caller can know where to get the results from
-puts output
+    # puts "#{command}"
+    input = output
+  end
+  # print out the output location so caller can know where to get the results from
+  puts output
+rescue Mandy::HadoopJobFailure => e
+  STDERR.puts e.to_s
+  exit(1)
+end

data/lib/errors.rb

@@ -0,0 +1,31 @@
+module Mandy
+  class HadoopJobFailure < StandardError
+    attr_reader :output
+    
+    def initialize(job, output)
+      @job, @output = job, output
+    end
+    
+    def job_name
+      @job.name
+    end
+    
+    def tracking_url
+      line = @output.split("\n").find {|line| line =~ /Tracking URL/ }
+      return nil unless line
+      line.split('Tracking URL: ').last
+    end
+    
+    def hadoop_error
+      @output.split("\n").find {|line| line =~ /ERROR/ }
+    end
+    
+    def to_s
+      output = []
+      output << %(Hadoop ERROR: #{hadoop_error || 'Unkown Error'})
+      output << %(Mandy Job Name: #{job_name})
+      output << %(Tracking URL: #{tracking_url}) if tracking_url
+      output*"\n"
+    end
+  end
+end

data/lib/mandy.rb

@@ -26,6 +26,7 @@ require "cgi"
   stores/in_memory
   test_runner
   ruby-hbase
+  errors
 ).each {|file| require File.join(File.dirname(__FILE__), file) }
 
 module Mandy

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.4.98
+  version: 0.4.99
 platform: ruby
 authors: 
 - Andy Kent
@@ -23,7 +23,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "0"
     version: 
-description: Map/Reduce
+description: Mandy is Ruby Map/Reduce Framework built onto of the Hadoop Distributed computing platform.
 email: andy.kent@me.com
 executables: 
 - mandy
@@ -64,6 +64,7 @@ files:
 - Rakefile
 - bootstrap.rb
 - geminstaller.yml
+- lib/errors.rb
 - lib/mandy.rb
 - lib/support/tuple.rb
 - lib/support/formatting.rb
@@ -118,6 +119,6 @@ rubyforge_project:
 rubygems_version: 1.3.5
 signing_key: 
 specification_version: 2
-summary: Map/Reduce
+summary: Map/Reduce Framework
 test_files: []