mandy 0.2.15 → 0.3.1

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://gemcutter.org"
+gem 'mandy'

data/bin/mandy-hadoop

@@ -26,6 +26,10 @@ OptionParser.new do |opts|
     options.cmdenv = "json=#{URI.encode(config)}"
   end
   
+  opts.on("-g", '--gemfile filepath', "Path to your jobs Gemfile (defaults to ./Gemfile)") do |config|
+    options.gemfile = config
+  end
+  
   opts.on_tail("-h", "--help", "Show this message") do
     puts opts
     exit
@@ -36,31 +40,47 @@ def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end
 
+def gemfile(file)
+  path = absolute_path(file || 'Gemfile')
+  File.exist?(path) ? path : File.expand_path(File.join(File.dirname(__FILE__), '..', 'Gemfile'))
+end
+
 file   = ARGV[0]
 filename = File.basename(file)
 input  = ARGV[1]
 output_folder = ARGV[2]
 config = options.config || 'cluster.xml'
-payload = options.payload ? Mandy::Packer.pack(options.payload) : ARGV[0]
+puts "Packaging Gems for distribution..."
+payload = Mandy::Packer.pack(options.payload || ARGV[0], gemfile(options.gemfile))
 cmdenv = options.cmdenv
 
-at_exit { Mandy::Packer.cleanup!(payload) }
+at_exit do 
+  puts "Cleaning up..."
+  Mandy::Packer.cleanup!(payload)
+  puts "All done!"
+end
 
+puts "Loading Mandy scripts..."
 require absolute_path(file)
 
 output = nil
 
+puts "Sending jobs to Hadoop..."
+
 Mandy::Job.jobs.each_with_index do |job, i|
   
   jobconf = job.settings.map { |key, value| %(-D #{key}='#{value}') }.join(' ')
   output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
   
+  bootstrap_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'bootstrap.rb'))
+  
   command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar #{jobconf}\
                   -conf '#{config}' \
                   -input "#{input}"  \
-                  -mapper "mandy-map #{filename} '#{job.name}' #{File.basename(payload)}"  \
-                  -reducer "mandy-reduce #{filename} '#{job.name}' #{File.basename(payload)}"  \
+                  -mapper "ruby bootstrap.rb #{File.basename(payload)} map #{filename} '#{job.name}'"  \
+                  -reducer "ruby bootstrap.rb #{File.basename(payload)} reduce #{filename} '#{job.name}'"  \
                   -file "#{payload}" \
+                  -file "#{bootstrap_file}" \
                   -cmdenv #{cmdenv} \
                   -output "#{output}")
 

data/bin/mandy-map

@@ -1,17 +1,13 @@
 #!/usr/bin/env ruby
-require "rubygems"
+environment = File.expand_path(File.join(File.dirname(__FILE__), '..', 'vendor', 'gems', 'environment.rb'))
+require File.exist?(environment) ? environment : "rubygems"
 require "mandy"
 
 if ARGV.size==0
-  puts "USAGE: mandy-map my_script.rb 'Job Name' [payload]"
+  puts "USAGE: mandy-map my_script.rb 'Job Name'"
   exit
 end
 
-if ARGV.size > 2
-  payload = ARGV[2]
-  Mandy::Packer.unpack(payload)
-end
-
 def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end

data/bin/mandy-reduce

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
-require "rubygems"
+environment = File.expand_path(File.join(File.dirname(__FILE__), '..', 'vendor', 'gems', 'environment.rb'))
+require File.exist?(environment) ? environment : "rubygems"
 require "mandy"
 
 if ARGV.size==0

data/bootstrap.rb

@@ -0,0 +1,29 @@
+require "fileutils"
+
+module Mandy
+  class Packer
+    TMP_DIR = '/tmp/mandy'
+    
+    def self.pack(dir)
+      return dir if File.file?(dir)
+      FileUtils.mkdir_p(TMP_DIR)
+      tmp_path = "#{TMP_DIR}/packed-job-#{Time.now.to_i}.tar"
+      Dir.chdir(dir) { `tar -cf #{tmp_path} *` }
+      tmp_path
+    end
+    
+    def self.unpack(file)
+      return false unless File.extname(file) == '.tar'
+      `tar -xf #{file}`
+    end
+    
+    def self.cleanup!(file)
+      return false unless File.extname(file) == '.tar'
+      `rm #{file}`
+    end
+  end
+end
+
+Mandy::Packer.unpack(ARGV[0])
+
+`bin/mandy-#{ARGV[1]} #{ARGV[2]} '#{ARGV[3]}'`

data/lib/mappers/base_mapper.rb

@@ -16,7 +16,8 @@ module Mandy
            key, value = line.split(KEY_VALUE_SEPERATOR, 2)
            key, value = nil, key if value.nil?
            value.chomp!
-           mapper(input_deserialize_key(key), input_deserialize_value(value))
+           args = [input_deserialize_key(key), input_deserialize_value(value)].compact
+           mapper(*args)
         end
       end
 

data/lib/mappers/pass_through_mapper.rb

@@ -1,9 +1,9 @@
 module Mandy
   module Mappers
     class PassThroughMapper < Base
-      def mapper(key,value)
+      def mapper(*params)
         # default map is simply a pass-through
-        emit(key, value)
+        params.size == 1 ? emit(params[0]) : emit(params[0], params[1])
       end
     end
   end

data/lib/packer.rb

@@ -4,12 +4,15 @@ module Mandy
   class Packer
     TMP_DIR = '/tmp/mandy'
     
-    def self.pack(dir)
-      return dir if File.file?(dir)
-      FileUtils.mkdir_p(TMP_DIR)
-      tmp_path = "#{TMP_DIR}/packed-job-#{Time.now.to_i}.tar"
-      Dir.chdir(dir) { `tar -cf #{tmp_path} *` }
-      tmp_path
+    def self.pack(dir, gemfile)
+      tmp_path = "#{TMP_DIR}/packed-job-#{Time.now.to_i}"
+      FileUtils.mkdir_p(tmp_path)
+      to_be_copied = File.file?(dir) ? dir : File.join(dir, '*')
+      FileUtils.cp_r(Dir.glob(to_be_copied), tmp_path)
+      FileUtils.cp_r(gemfile, tmp_path)
+      Dir.chdir(tmp_path) { `gem bundle` }
+      Dir.chdir(tmp_path) { `tar -cf bundle.tar *` }
+      File.join(tmp_path, 'bundle.tar')
     end
     
     def self.unpack(file)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.2.15
+  version: 0.3.1
 platform: ruby
 authors: 
 - Andy Kent
@@ -10,10 +10,19 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-09 00:00:00 +01:00
+date: 2009-10-02 00:00:00 +01:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
 description: Map/Reduce
 email: andy.kent@me.com
 executables: 
@@ -39,6 +48,8 @@ files:
 - bin/mandy-reduce
 - readme.md
 - Rakefile
+- bootstrap.rb
+- Gemfile
 - lib/mandy.rb
 - lib/support/tuple.rb
 - lib/support/formatting.rb
@@ -89,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.4
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 2
 summary: Map/Reduce