mandy 0.5.11 → 0.5.13

data/VERSION

@@ -1 +1 @@
-0.5.11
+0.5.13

data/bin/mandy-hadoop

@@ -19,6 +19,14 @@ OptionParser.new do |opts|
   opts.on("-i", "--input-format class", "Set the hadoop input format type") do |format_class|
     options.input_format = format_class
   end
+  
+  opts.on("-o", "--output-format class", "Set the hadoop output format type") do |format_class|
+    options.output_format = format_class
+  end
+  
+  opts.on("-l", "--libjars jar files", "comma-separated jar files to include in the classpath") do |libjars|
+    options.libjars = libjars
+  end
 
   opts.on("-p", "--payload PAYLOAD", "Add a working directory to be sent to the cluster.") do |payload|
     options.payload = payload
@@ -73,11 +81,12 @@ input = inputs.map {|path| "-input \"#{path}\""}.join(" ")
 output_folder = ARGV[2]
 config = absolute_path(options.config || 'cluster.xml')
 puts "Packaging code for distribution..."
-payload = Mandy::Packer.pack(file, options.payload || ARGV[0], gemfile(options.gemfile))  
+payload = Mandy::Packer.pack(file, options.payload || ARGV[0], gemfile(options.gemfile))
 cmdenv = options.cmdenv
 set_env(cmdenv)
 
 inputformat = !options.input_format.nil? ? options.input_format : "TextInputFormat"
+outputformat = !options.output_format.nil? ? options.output_format : "org.apache.hadoop.mapred.TextOutputFormat"
 
 at_exit do
   puts
@@ -100,18 +109,21 @@ begin
   
     bootstrap_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'bootstrap.rb'))
     inputreader = job.input_format == :xml ? "StreamXmlRecordReader,begin=<#{job.input_format_options[:xml_tag]} ,end=</#{job.input_format_options[:xml_tag]}>" : nil
-  
-    command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop*streaming*.jar #{jobconf}\
+    
+    command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop*streaming*.jar \
+                    #{options.libjars.nil? ? '' : "-libjars \"#{options.libjars}\""} \
+                    #{jobconf}\
                     -files "#{payload}","#{bootstrap_file}" \
                     -conf '#{config}' \
                     -inputformat '#{inputformat}' \
+                    -outputformat '#{outputformat}' \
                     #{input} \
                     #{ inputreader.nil? ? '' : "-inputreader \"#{inputreader}\"" } \
                     -mapper "ruby bootstrap.rb #{File.basename(payload)} map #{filename} '#{job.name}'"  \
                     -reducer "ruby bootstrap.rb #{File.basename(payload)} reduce #{filename} '#{job.name}'"  \
                     #{ cmdenv.nil? ? '' : "-cmdenv #{cmdenv}" }\
                     -output "#{output}" 2>&1)
-                    
+    
     result = []
     IO.popen(command, 'r') do |subprocess|
       while line = subprocess.gets

data/lib/mandy/errors.rb

@@ -17,13 +17,13 @@ module Mandy
     end
     
     def hadoop_error
-      @output.split("\n").find {|line| line =~ /ERROR/ }
+      @output
     end
     
     def to_s
       output = []
-      output << %(Hadoop ERROR: #{hadoop_error || 'Unkown Error'})
       output << %(Mandy Job Name: #{job_name})
+      output << %(Hadoop ERROR: #{hadoop_error})    
       output << %(Tracking URL: #{tracking_url}) if tracking_url
       output*"\n"
     end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 5
-  - 11
-  version: 0.5.11
+  - 13
+  version: 0.5.13
 platform: ruby
 authors: 
 - Andy Kent