mandy 0.4.83 → 0.4.86

data/Gemfile

@@ -1,2 +1,3 @@
 source "http://gemcutter.org"
-gem 'mandy'
+gem 'mandy'
+gem 'json'

data/bin/mandy

@@ -35,7 +35,6 @@ puts '------------------------'
   'mandy-install' => 'Installs the Mandy Rubygem on several hosts via ssh.',
   'mandy-local' => 'Run a Map/Reduce task locally without requiring hadoop',
   'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
-  'mandy-run' => 'Run an entire Map/Reduce workflow with one command',
   'mandy-rm' => 'remove a file or directory from HDFS',
   'mandy-put' => 'upload a file into HDFS',
   'mandy-map' => 'Run a map task reading on STDIN and writing to STDOUT',

data/lib/mandy.rb

@@ -11,6 +11,7 @@ require "cgi"
   packer
   support/tuple 
   support/array_serializer 
+  support/hdfs_location 
   mappers/base_mapper 
   mappers/transpose_mapper 
   mappers/pass_through_mapper 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.4.83
+  version: 0.4.86
 platform: ruby
 authors: 
 - Andy Kent
@@ -49,7 +49,6 @@ executables:
 - mandy-mkdir
 - mandy-exists
 - mandy-install
-- mandy-run
 extensions: []
 
 extra_rdoc_files: []
@@ -61,7 +60,6 @@ files:
 - bin/mandy-get
 - bin/mandy-put
 - bin/mandy-reduce
-- bin/mandy-run
 - readme.md
 - Rakefile
 - bootstrap.rb
@@ -93,8 +91,6 @@ files:
 - lib/ruby-hbase/version.rb
 - lib/ruby-hbase/xml_decoder.rb
 - lib/test_runner.rb
-- lib/wrappers/mandy_wrapper.rb
-- lib/wrappers/mandy_local_wrapper.rb
 has_rdoc: true
 homepage: http://github.com/trafficbroker/mandy
 licenses: []

data/bin/mandy-run

@@ -1,58 +0,0 @@
-#!/usr/bin/env ruby
-require "rubygems"
-require "mandy"
-require 'optparse'
-require 'ostruct'
-require 'json'
-
-options = OpenStruct.new
-
-OptionParser.new do |opts|
-  opts.banner = "USAGE: mandy-run script input [options]"
-
-  opts.on("-p", "--payload PAYLOAD", "Add a working directory to be sent to the cluster.") do |payload|
-    options.payload = payload
-  end
-
-  opts.on("-o", "--output OUTPUT", "Specify output path for your results.") do |payload|
-    options.payload = payload
-  end
-  
-  opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config|
-    options.config = config
-  end
-
-  opts.on("-j", '--json "{\"key\":\"1 value\"}"', "Pass JSON encoded parameters to jobs") do |config|
-    options.json = config
-  end
-
-  opts.on("-l", "--local", "Run script using mandy-local.") do
-    options.local = true
-  end
-  
-  opts.on_tail("-h", "--help", "Show this message") do
-    puts opts
-    exit
-  end
-end.parse!
-
-exec('mandy-local -h') unless ARGV.size >= 2
-
-file   = ARGV[0]
-input  = File.new(ARGV[1])
-
-params = {}
-params[:output_file] = options.output if options.output
-params[:lib] = options.payload if options.payload
-params[:parameters] = JSON.parse(options.json) if options.json
-
-if options.local
-  require 'wrappers/mandy_local_wrapper'
-else
-  require 'wrappers/mandy_wrapper'
-  set_mandy_config options.config || 'cluster.xml'
-end
-
-output_file = run_mandy(file, input, params)
-
-puts "Results stored in: #{output_file}"

data/lib/wrappers/mandy_local_wrapper.rb

@@ -1,58 +0,0 @@
-module Mandy
-  module Local
-    module Wrapper
-      SESSION_ID = Process.pid
-  
-      def run_mandy(script, input_files, options = {})
-        begin
-          #doing this will load all the mandy jobs in memory which will be useful later on
-          require script
-
-          input_file = concat_input_files(input_files)
-          output_file_path = run_mandy_local(script, input_file, options)
-          return output_file_path unless block_given?
-          #if a block is given then yield the output file path and then delete this file before returning
-          yield output_file_path
-        ensure 
-          File.delete(input_file) if File.exists?(input_file)
-          File.delete(output_file_path) if output_file_path && File.exists?(output_file_path) if block_given?
-        end
-      end
-  
-      private
-        def concat_input_files(inputs)
-          inputs = [inputs] unless inputs.is_a?(Array)
-          base_dir = File.dirname(inputs.first.path)
-          input_file = "#{base_dir}/#{SESSION_ID}.csv"
-          `cat #{inputs.collect{|f| f.path}.join(' ')} > #{input_file}`
-          input_file
-        end
-    
-        def run_mandy_local(script, input, options)
-          mandy_job_params = options.include?(:parameters) ? options[:parameters] : {}
-          param_args = "export json='#{mandy_job_params.to_json}' &&"
-          
-          if options.include?(:lib)
-            FileUtils.cp(script, options[:lib])
-            script = File.join(options[:lib], File.basename(script))
-          end
-          
-          output_path = options[:output_file] || generate_output_path
-          output_file = `#{param_args} mandy-local #{script} #{input} #{output_path}`
-          output_file = output_file.split("\n").last
-          output_file
-        ensure
-          File.delete(script) if options.include?(:lib)
-        end
-    
-        def generate_output_path
-          output_dir = "/tmp/mandy_local_output"
-          FileUtils.mkdir_p(output_dir)
-          file_name = Mandy::Job.jobs.last.name.downcase.gsub(/\W/, '-')
-          "#{output_dir}/#{file_name}_#{DateTime.now.strftime('%Y%m%d%H%M%S')}"
-        end
-    end
-  end
-end
-
-Object.send(:include, Mandy::Local::Wrapper)

data/lib/wrappers/mandy_wrapper.rb

@@ -1,94 +0,0 @@
-module Mandy
-  module Wrapper
-    SESSION_ID = Process.pid
-  
-    def set_mandy_config(file_path)
-      @@config_path = file_path
-    end
-  
-    def run_mandy(script, inputs, options = {})
-      begin
-        #doing this will load all the mandy jobs in memory which will be useful later on
-        require script
-        inputs = [inputs] unless inputs.is_a?(Array)
-        
-        hdfs_input = inputs.all? {|i| i.is_a?(File)} ? process_files(inputs) : process_hdfs_locations(inputs)
-        
-        run_mandy_hadoop(hdfs_input, script, options)
-        
-        output_file_path = get_file_from_hdfs(hdfs_path, options)
-        return output_file_path unless block_given?
-        #if a block is given then yield the output file path and then delete this file before returning
-        yield output_file_path
-      ensure 
-        File.delete(output_file_path) if output_file_path && File.exists?(output_file_path) if block_given?
-      end
-    end
-  
-    private
-      def process_files(input_files)
-        hdfs_path = "#{self.class.to_s.split('::').join('-').downcase}/#{SESSION_ID}"
-        put_files_on_hdfs(hdfs_path, input_files)
-        hdfs_path
-      end
-      
-      def process_locations(input_locations)
-        return input_locations.first if input_locations.size == 1
-        
-        hdfs_path = "#{self.class.to_s.split('::').join('-').downcase}/#{SESSION_ID}"
-        input_locations.each_with_index do |location, index|
-          run_command "mandy-cp #{location} #{hdfs_path}/input#{index}"
-        end
-        hdfs_path
-      end
-    
-      def put_files_on_hdfs(hdfs_path, input_files)
-        input_files.each do |input_file|
-          input_file_path = input_file.is_a?(File) ? File.expand_path(input_file.path) : input_file
-          base_filename = input_file_path.split("/").last
-          dest_file = ["input/#{hdfs_path}", base_filename].join("/")
-          run_command "mandy-put #{input_file_path} #{dest_file}"
-        end
-      end
-    
-      def run_mandy_hadoop(hdfs_path, script, options)
-        mandy_job_params = options.include?(:parameters) ? options[:parameters] : {}
-        param_args = "-j '#{mandy_job_params.to_json}'"
-        param_args += " -p '#{options[:lib]}'" if options.include?(:lib)
-
-        hdfs_output_path = "output/#{hdfs_path}"
-        run_command "mandy-rm output/#{hdfs_path}"
-        run_command "mandy-hadoop #{script} input/#{hdfs_path} output/#{hdfs_path} #{param_args}"
-      end
-    
-      def get_file_from_hdfs(hdfs_path, options)
-        output_file_path = options[:output_file] || generate_output_path
-        hdfs_output_path = "output/#{hdfs_path}"
-        run_command "mandy-get #{get_hdfs_output(hdfs_output_path)} #{output_file_path}"
-        run_command "mandy-rm input/#{hdfs_path}"
-        run_command "mandy-rm output/#{hdfs_path}"
-        output_file_path
-      end
-  
-      def run_command(command)
-        command = "#{command} -c #{@@config_path}"
-        respond_to?(:logger) ? logger.info(command) : p(command)
-        @output = `#{command}`
-      end
-    
-      def generate_output_path
-        output_dir = "/tmp/mandy_output"
-        FileUtils.mkdir_p(output_dir)
-        file_name = Mandy::Job.jobs.last.name.downcase.gsub(/\W/, '-')
-        "#{output_dir}/#{file_name}_#{DateTime.now.strftime('%Y%m%d%H%M%S')}"
-      end
-      
-      def get_hdfs_output(hdfs_output_path)
-        @output.each_line do |line|
-          return line.chomp.strip if line.include?(hdfs_output_path)
-        end
-      end
-  end
-end
-
-Object.send(:include, Mandy::Wrapper)