mandy 0.4.2 → 0.4.3

data/bin/mandy-hadoop

@@ -75,14 +75,15 @@ Mandy::Job.jobs.each_with_index do |job, i|
   output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
   
   bootstrap_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'bootstrap.rb'))
+  inputreader = job.input_format == :xml ? "StreamXmlRecordReader,begin=<#{job.input_format_options[:xml_tag]} ,end=</#{job.input_format_options[:xml_tag]}>" : nil
   
   command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar #{jobconf}\
+                  -files "#{payload}","#{bootstrap_file}" \
                   -conf '#{config}' \
                   -input "#{input}"  \
+                  #{ inputreader.nil? ? '' : "-inputreader \"#{inputreader}\"" } \
                   -mapper "ruby bootstrap.rb #{File.basename(payload)} map #{filename} '#{job.name}'"  \
                   -reducer "ruby bootstrap.rb #{File.basename(payload)} reduce #{filename} '#{job.name}'"  \
-                  -file "#{payload}" \
-                  -file "#{bootstrap_file}" \
                   #{ cmdenv.nil? ? '' : "-cmdenv #{cmdenv}" }\
                   -output "#{output}")
 

data/lib/job.rb

@@ -12,7 +12,8 @@ module Mandy
     
     attr_reader :settings
     attr_reader :name
-    
+    attr_reader :input_format_options
+
     def initialize(name, &blk)
       @name = name
       @settings = {}
@@ -25,11 +26,14 @@ module Mandy
       modules.each {|m| @modules << m}
     end
     alias_method :serialize, :mixin
-    
-    def input_format(format)
+
+    def input_format(format=nil, options={})
+      return @input_format if format.nil?
+      
       @input_format = format
+      @input_format_options = options
     end
-
+    
     def output_format(format)
       @output_format = format
     end

data/lib/wrappers/mandy_wrapper.rb

@@ -27,7 +27,7 @@ module Mandy
       def put_files_on_hdfs(hdfs_path, input_files)
         input_files = [input_files] unless input_files.is_a?(Array)
         input_files.each do |input_file|
-          input_file_path = File.expand_path(input_file.path)
+          input_file_path = input_file.is_a?(File) ? File.expand_path(input_file.path) : input_file
           base_filename = input_file_path.split("/").last
           dest_file = ["input/#{hdfs_path}", base_filename].join("/")
           run_command "mandy-put #{input_file_path} #{dest_file}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors: 
 - Andy Kent