mandy 0.4.8 → 0.4.10

data/bin/mandy-cp

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'ostruct'
+
+exec('mandy-cp -h') unless ARGV.size >= 2
+
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: mandy-cp source destination [options]"
+
+  opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config|
+    options.config = config
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+source = ARGV[0]
+destination = ARGV[1]
+config = options.config || 'cluster.xml'
+
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -cp #{source} #{destination}`

data/lib/support/hdfs_location.rb

@@ -0,0 +1,13 @@
+module Mandy
+  class HdfsLocation
+    attr_reader :path
+    
+    def initialize(path)
+      @path = path
+    end
+    
+    def to_s
+      @path
+    end
+  end
+end

data/lib/wrappers/mandy_wrapper.rb

@@ -6,14 +6,16 @@ module Mandy
       @@config_path = file_path
     end
   
-    def run_mandy(script, input_files, options = {})
+    def run_mandy(script, inputs, options = {})
       begin
         #doing this will load all the mandy jobs in memory which will be useful later on
         require script
+        inputs = [inputs] unless inputs.is_a?(Array)
+        
+        hdfs_input = inputs.all? {|i| i.is_a?(File)} ? process_files(inputs) : process_hdfs_locations(inputs)
+        
+        run_mandy_hadoop(hdfs_input, script, options)
         
-        hdfs_path = "#{self.class.to_s.split('::').join('-').downcase}/#{SESSION_ID}"
-        put_files_on_hdfs(hdfs_path, input_files)
-        run_mandy_hadoop(hdfs_path, script, options)
         output_file_path = get_file_from_hdfs(hdfs_path, options)
         return output_file_path unless block_given?
         #if a block is given then yield the output file path and then delete this file before returning
@@ -24,8 +26,23 @@ module Mandy
     end
   
     private
+      def process_files(input_files)
+        hdfs_path = "#{self.class.to_s.split('::').join('-').downcase}/#{SESSION_ID}"
+        put_files_on_hdfs(hdfs_path, input_files)
+        hdfs_path
+      end
+      
+      def process_locations(input_locations)
+        return input_locations.first if input_locations.size == 1
+        
+        hdfs_path = "#{self.class.to_s.split('::').join('-').downcase}/#{SESSION_ID}"
+        input_locations.each_with_index do |location, index|
+          run_command "mandy-cp #{location} #{hdfs_path}/input#{index}"
+        end
+        hdfs_path
+      end
+    
       def put_files_on_hdfs(hdfs_path, input_files)
-        input_files = [input_files] unless input_files.is_a?(Array)
         input_files.each do |input_file|
           input_file_path = input_file.is_a?(File) ? File.expand_path(input_file.path) : input_file
           base_filename = input_file_path.split("/").last

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.4.8
+  version: 0.4.10
 platform: ruby
 authors: 
 - Andy Kent
@@ -45,6 +45,7 @@ executables:
 - mandy-reduce
 - mandy-rm
 - mandy-mv
+- mandy-cp
 - mandy-mkdir
 - mandy-exists
 - mandy-install
@@ -69,6 +70,7 @@ files:
 - lib/support/tuple.rb
 - lib/support/formatting.rb
 - lib/support/array_serializer.rb
+- lib/support/hdfs_location.rb
 - lib/task.rb
 - lib/dsl.rb
 - lib/job.rb