mandy 0.3.11 → 0.3.12

data/bin/mandy-run

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "mandy"
+require 'optparse'
+require 'ostruct'
+require 'json'
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: mandy-run script input [options]"
+
+  opts.on("-p", "--payload PAYLOAD", "Add a working directory to be sent to the cluster.") do |payload|
+    options.payload = payload
+  end
+
+  opts.on("-o", "--output OUTPUT", "Specify output path for your results.") do |payload|
+    options.payload = payload
+  end
+  
+  opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config|
+    options.config = config
+  end
+
+  opts.on("-j", '--json "{\"key\":\"1 value\"}"', "Pass JSON encoded parameters to jobs") do |config|
+    options.json = config
+  end
+
+  opts.on("-l", "--local", "Run script using mandy-local.") do
+    options.local = true
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+file   = ARGV[0]
+input  = File.new(ARGV[1])
+
+params = {}
+params[:output_file] = options.output if options.output
+params[:lib] = options.payload if options.payload
+params[:parameters] = JSON.parse(options.json) if options.json
+
+if options.local
+  require 'wrappers/mandy_local_wrapper'
+else
+  require 'wrappers/mandy_wrapper'
+  set_mandy_config options.config || 'cluster.xml'
+end
+
+output_file = run_mandy(file, input, params)
+
+puts "Results stored in: #{output_file}"

data/lib/wrappers/mandy_local_wrapper.rb

@@ -3,12 +3,11 @@ module Mandy
     module Wrapper
       SESSION_ID = Process.pid
   
-      def set_mandy_config(file_path)
-        @@config_path = file_path
-      end
-  
       def run_mandy(script, input_files, options = {})
         begin
+          #doing this will load all the mandy jobs in memory which will be useful later on
+          require script
+
           input_file = concat_input_files(input_files)
           output_file_path = run_mandy_local(script, input_file, options)
           return output_file_path unless block_given?
@@ -37,8 +36,9 @@ module Mandy
             FileUtils.cp(script, options[:lib])
             script = File.join(options[:lib], File.basename(script))
           end
-
-          output_file = `#{param_args} mandy-local #{script} #{input} #{generate_output_path}`
+          
+          output_path = options[:output_file] || generate_output_path
+          output_file = `#{param_args} mandy-local #{script} #{input} #{output_path}`
           output_file = output_file.split("\n").last
           output_file
         ensure
@@ -46,9 +46,10 @@ module Mandy
         end
     
         def generate_output_path
-          output_dir = "/tmp/mandy_test_output"
+          output_dir = "/tmp/mandy_local_output"
           FileUtils.mkdir_p(output_dir)
-          "#{output_dir}/#{SESSION_ID}"
+          file_name = Mandy::Job.jobs.last.name.downcase.gsub(/\W/, '-')
+          "#{output_dir}/#{file_name}_#{DateTime.now.strftime('%Y%m%d%H%M%S')}"
         end
     end
   end

data/lib/wrappers/mandy_wrapper.rb

@@ -8,6 +8,9 @@ module Mandy
   
     def run_mandy(script, input_files, options = {})
       begin
+        #doing this will load all the mandy jobs in memory which will be useful later on
+        require script
+        
         hdfs_path = "#{self.class.to_s.split('::').join('-').downcase}/#{SESSION_ID}"
         put_files_on_hdfs(hdfs_path, input_files)
         run_mandy_hadoop(hdfs_path, script, options)
@@ -59,7 +62,8 @@ module Mandy
       def generate_output_path
         output_dir = "/tmp/mandy_output"
         FileUtils.mkdir_p(output_dir)
-        "#{output_dir}/#{SESSION_ID}"
+        file_name = Mandy::Job.jobs.last.name.downcase.gsub(/\W/, '-')
+        "#{output_dir}/#{file_name}_#{DateTime.now.strftime('%Y%m%d%H%M%S')}"
       end
       
       def get_hdfs_output(hdfs_output_path)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.3.11
+  version: 0.3.12
 platform: ruby
 authors: 
 - Andy Kent
@@ -35,6 +35,7 @@ executables:
 - mandy-reduce
 - mandy-rm
 - mandy-install
+- mandy-run
 extensions: []
 
 extra_rdoc_files: []
@@ -46,6 +47,7 @@ files:
 - bin/mandy-get
 - bin/mandy-put
 - bin/mandy-reduce
+- bin/mandy-run
 - readme.md
 - Rakefile
 - bootstrap.rb