trafficbroker-mandy 0.2 → 0.2.2

data/bin/mandy-hadoop

@@ -1,11 +1,13 @@
 #!/usr/bin/env ruby
+require "rubygems"
+require "mandy"
 
 def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end
 
 if ARGV.size==0
-  puts "USAGE: mandy-hadoop my_script.rb input_file_or_folder_on_hdfs output_folder_on_hdfs cluster-config.xml"
+  puts "USAGE: mandy-hadoop my_script.rb input_file_or_folder_on_hdfs output_folder_on_hdfs cluster-config.xml [payload]"
   exit
 end
 
@@ -14,6 +16,7 @@ filename = File.basename(file)
 input  = ARGV[1]
 output_folder = ARGV[2]
 config = ARGV[3]
+payload = ARGV[4] ? Mandy::Packer.pack(ARGV[4]) : ARGV[0]
 
 require absolute_path(file)
 
@@ -25,9 +28,9 @@ Mandy::Job.jobs.each_with_index do |job, i|
   command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar #{jobconf}\
                   -conf '#{config}' \
                   -input "#{input}"  \
-                  -mapper "mandy-map #{filename} '#{job.name}'"  \
-                  -reducer "mandy-reduce #{filename} '#{job.name}'"  \
-                  -file "#{file}" \
+                  -mapper "mandy-map #{filename} '#{job.name}' #{payload}"  \
+                  -reducer "mandy-reduce #{filename} '#{job.name}' #{payload}"  \
+                  -file "#{payload}" \
                   -output "#{output}")
 
   `#{command}`

data/bin/mandy-map

@@ -1,10 +1,17 @@
 #!/usr/bin/env ruby
+require "rubygems"
+require "mandy"
 
 if ARGV.size==0
-  puts "USAGE: mandy-map my_script.rb 'Job Name'"
+  puts "USAGE: mandy-map my_script.rb 'Job Name' [payload]"
   exit
 end
 
+if ARGV.size > 3
+  payload = ARGV[3]
+  Mandy::Packer.unpack(payload)
+end
+
 def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end

data/bin/mandy-reduce

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 if ARGV.size==0
-  puts "USAGE: mandy-reduce my_script.rb 'Job Name'"
+  puts "USAGE: mandy-reduce my_script.rb 'Job Name' [payload]"
   exit
 end
 

data/lib/mandy.rb

@@ -2,6 +2,7 @@
   task
   dsl 
   job 
+  packer
   support/tuple 
   support/array_serializer 
   mappers/base_mapper 

data/lib/packer.rb

@@ -0,0 +1,20 @@
+require "fileutils"
+
+module Mandy
+  class Packer
+    TMP_DIR = '/tmp/mandy'
+    
+    def self.pack(dir)
+      return dir if File.file?(dir)
+      FileUtils.mkdir_p(TMP_DIR)
+      tmp_path = "#{TMP_DIR}/packed-job-#{Time.now.to_i}.tar"
+      `tar -cf #{tmp_path} #{dir}`
+      tmp_path
+    end
+    
+    def self.unpack(file)
+      return false unless File.extname(payload) == '.tar'
+      `tar -xf --overwrite #{file}`
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: trafficbroker-mandy
 version: !ruby/object:Gem::Version 
-  version: "0.2"
+  version: 0.2.2
 platform: ruby
 authors: 
 - Andy Kent
@@ -46,6 +46,7 @@ files:
 - lib/job.rb
 - lib/mappers/base_mapper.rb
 - lib/mappers/pass_through_mapper.rb
+- lib/packer.rb
 - lib/reducers/base_reducer.rb
 - lib/reducers/pass_through_reducer.rb
 - lib/reducers/sum_reducer.rb