mandy 0.3.12 → 0.3.13

data/bin/mandy

@@ -11,6 +11,7 @@ puts ''
   'mandy-install' => 'Installs the Mandy Rubygem on several hosts via ssh.',
   'mandy-local' => 'Run a Map/Reduce task locally without requiring hadoop',
   'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
+  'mandy-run' => 'Run an entire Map/Reduce workflow with one command',
   'mandy-rm' => 'remove a file or directory from HDFS',
   'mandy-put' => 'upload a file into HDFS',
   'mandy-map' => 'Run a map task reading on STDIN and writing to STDOUT',

data/lib/mandy.rb

@@ -29,12 +29,12 @@ require "cgi"
 
 module Mandy
   class << self
+    attr_accessor :local_input
     def stores
       @stores||={}
     end
   end
   
-  
   def job(name, &blk)
     job = Mandy::Job.new(name)
     job.instance_eval(&blk) unless blk.nil?
@@ -42,4 +42,26 @@ module Mandy
     job
   end
   module_function :job
+end
+
+at_exit do
+  raise $! if $!
+  caller = Kernel.caller.first.split(':').first
+  next if caller =~ /bin\/(rake|mandy)/
+  input = Mandy.local_input || ENV['MANDY_INPUT']
+  unless input
+    print "Input file: "
+    input = gets.chomp
+  end
+  file  = caller
+  output_folder = FileUtils.mkdir_p("/tmp/mandy-local")
+  out = nil
+  Mandy::Job.jobs.each_with_index do |job, i|
+    out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
+    puts "Running #{job.name}..."
+    `cat #{input} | mandy-map #{file} "#{job.name}" | sort | mandy-reduce #{file} "#{job.name}" > #{out}`
+    input = out
+  end
+
+  puts File.read(out)
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors: 
 - Andy Kent