trafficbroker-mandy 0.1.3 → 0.1.5

data/bin/mandy-hadoop

@@ -4,6 +4,11 @@ def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end
 
+if ARGV.size==0
+  puts "USAGE: mandy-hadoop my_script.rb input_file_or_folder_on_hdfs output_folder_on_hdfs cluster-config.xml"
+  exit
+end
+
 file   = ARGV[0]
 filename = File.basename(file)
 input  = ARGV[1]

data/bin/mandy-install

@@ -1,5 +1,10 @@
 #!/usr/bin/env ruby
 
+if ARGV.size==0
+  puts "USAGE: mandy-install user@server1.com [user@server2.com ...]"
+  exit
+end
+
 ARGV.each do |server|
   puts "Installing on #{server}..."
   system %(ssh #{server} "sudo gem install trafficbroker-mandy --source http://gems.github.com")

data/bin/mandy-local

@@ -1,5 +1,10 @@
 #!/usr/bin/env ruby
 
+if ARGV.size==0
+  puts "USAGE: mandy-local my_script.rb local_input_file local_output_folder"
+  exit
+end
+
 require "fileutils"
 
 def absolute_path(path)
@@ -13,6 +18,9 @@ require file
 
 Mandy::Job.jobs.each_with_index do |job, i|
   out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
+  puts "Running #{job.name}..."
   `cat #{input} | mandy-map #{file} "#{job.name}" | sort | mandy-reduce #{file} "#{job.name}" > #{out}`
   input = out
-end
+end
+
+puts "All Done! [#{input}]"

data/bin/mandy-map

@@ -1,5 +1,10 @@
 #!/usr/bin/env ruby
 
+if ARGV.size==0
+  puts "USAGE: mandy-map my_script.rb 'Job Name'"
+  exit
+end
+
 def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end

data/bin/mandy-put

@@ -1,5 +1,10 @@
 #!/usr/bin/env ruby
 
+if ARGV.size==0
+  puts "USAGE: mandy-put local_file_or_folder hdfs_destination_location cluster-config.xml"
+  exit
+end
+
 def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end

data/bin/mandy-reduce

@@ -1,5 +1,10 @@
 #!/usr/bin/env ruby
 
+if ARGV.size==0
+  puts "USAGE: mandy-reduce my_script.rb 'Job Name'"
+  exit
+end
+
 def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end

data/bin/mandy-rm

@@ -1,6 +1,11 @@
 #!/usr/bin/env ruby
 
-config = ARGV[0]
+if ARGV.size==0
+  puts "USAGE: mandy-rm file_or_folder_on_hdfs cluster-config.xml"
+  exit
+end
+
 file = ARGV[1]
+config = ARGV[0]
 
 `$HADOOP_HOME/bin/hadoop fs -conf #{config} -rmr #{file}`

data/lib/job.rb

@@ -16,8 +16,8 @@ module Mandy
     def initialize(name, &blk)
       @name = name
       @settings = {}
-      @mapper_class = Mandy::Mapper
-      @reducer_class = Mandy::Reducer
+      @mapper_class = Mandy::Mappers::PassThroughMapper
+      @reducer_class = Mandy::Reducers::PassThroughReducer
       set('mapred.job.name', name)
       instance_eval(&blk) if blk
     end
@@ -26,12 +26,20 @@ module Mandy
       @settings[key.to_s] = value.to_s
     end
     
-    def map(&blk)
-      @mapper_class = Mandy::Mapper.compile(&blk)
+    def map_tasks(count)
+      set('mapred.map.tasks', count)
     end
     
-    def reduce(&blk)
-      @reducer_class = Mandy::Reducer.compile(&blk)
+    def reduce_tasks(count)
+      set('mapred.reduce.tasks', count)
+    end
+    
+    def map(klass=nil, &blk)
+      @mapper_class = klass || Mandy::Mappers::Base.compile(&blk)
+    end
+    
+    def reduce(klass=nil, &blk)
+      @reducer_class = klass || Mandy::Reducers::Base.compile(&blk)
     end
     
     def run_map(input=STDIN, output=STDOUT, &blk)

data/lib/mandy.rb

@@ -1 +1,12 @@
-%w(tuple array_serializer mapper reducer dsl job test_runner).each {|file| require File.join(File.dirname(__FILE__), file) }
+%w(
+  support/tuple 
+  support/array_serializer 
+  mappers/base_mapper 
+  mappers/pass_through_mapper 
+  reducers/base_reducer 
+  reducers/pass_through_reducer 
+  reducers/sum_reducer 
+  dsl 
+  job 
+  test_runner
+).each {|file| require File.join(File.dirname(__FILE__), file) }

data/lib/mappers/base_mapper.rb

@@ -0,0 +1,40 @@
+module Mandy
+  module Mappers
+    class Base
+      
+      KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
+    
+      def initialize(input=STDIN, output=STDOUT)
+        @input, @output = input, output
+      end
+    
+      def self.compile(&blk)
+        Class.new(Mandy::Mappers::Base) do 
+          self.class_eval do
+            define_method(:mapper, blk) if blk
+          end
+        end
+      end
+    
+      def execute
+        @input.each_line do |line|
+           key, value = line.split(KEY_VALUE_SEPERATOR)
+           key, value = nil, key if value.nil?
+           value.chomp!
+           mapper(key, value)
+        end
+      end
+    
+      def emit(key, value=nil)
+        key = 'nil' if key.nil?
+        @output.puts(value.nil? ? key.to_s : "#{key}\t#{value}")
+      end
+    
+      private
+    
+      def mapper(key,value)
+        #nil
+      end
+    end
+  end
+end

data/lib/mappers/pass_through_mapper.rb

@@ -0,0 +1,10 @@
+module Mandy
+  module Mappers
+    class PassThroughMapper < Base
+      def mapper(key,value)
+        # default map is simply a pass-through
+        emit(key, value)
+      end
+    end
+  end
+end

data/lib/reducers/base_reducer.rb

@@ -0,0 +1,45 @@
+module Mandy
+  module Reducers
+    class Base
+      KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
+    
+      def initialize(input=STDIN, output=STDOUT)
+        @input, @output = input, output
+      end
+    
+      def self.compile(&blk)
+        Class.new(Mandy::Reducers::Base) do 
+          self.class_eval do
+            define_method(:reducer, blk) if blk
+          end
+        end
+      end
+    
+      def execute
+        last_key, values = nil, []
+        @input.each_line do |line|
+           key, value = line.split(KEY_VALUE_SEPERATOR)
+           value.chomp!
+           last_key = key if last_key.nil?
+           if key != last_key
+             reducer(last_key, values)
+             last_key, values = key, []
+           end
+           values << value
+        end
+        reducer(last_key, values)
+      end
+    
+      def emit(key, value=nil)
+        key = 'nil' if key.nil?
+        @output.puts(value.nil? ? key.to_s : "#{key}\t#{value}")
+      end
+    
+      private
+    
+      def reducer(key,values)
+        #nil
+      end
+    end
+  end
+end

data/lib/reducers/pass_through_reducer.rb

@@ -0,0 +1,9 @@
+module Mandy
+  module Reducers
+    class PassThroughReducer < Base
+      def reducer(key,values)
+        values.each {|value| emit(key, value) }
+      end
+    end
+  end
+end

data/lib/reducers/sum_reducer.rb

@@ -0,0 +1,9 @@
+module Mandy
+  module Reducers
+    class SumReducer < Base
+      def reducer(key,values)
+        emit(key, values.inject(0) {|sum,count| sum+count.to_f})
+      end
+    end
+  end
+end

data/lib/test_runner.rb

@@ -1,7 +1,7 @@
 module Mandy
   class TestRunner
-    def initialize(job=Mandy::Job.jobs.first)
-      @job = job
+    def initialize(job=Mandy::Job.jobs.first.name)
+      @job = Mandy::Job.find_by_name(job)
     end
     
     def map(input, output_stream=StringIO.new(''), &blk)

data/readme.md

@@ -3,7 +3,7 @@ Mandy - Simplified Hadoop distribution for Ruby code
 
 Mandy hides the differences and complexities between running map/reduce tasks locally or distributed or in test environments.
 
-It provides a simple DSL to define new jobs for distribution. See examples/word_count.rb for a very simple demo.
+It provides a simple DSL to define new jobs for distribution. See examples/word_count.rb for a demo of some functionality.
 Run the word count example locally with...
 
-    bin/mandy local examples/word_count.rb examples/alice.txt examples/output.txt
+    mandy-local examples/word_count.rb examples/alice.txt examples/output

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: trafficbroker-mandy
 version: !ruby/object:Gem::Version 
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors: 
 - Andy Kent
@@ -37,13 +37,16 @@ files:
 - readme.md
 - Rakefile
 - lib/mandy.rb
-- lib/array_serializer.rb
+- lib/support/tuple.rb
+- lib/support/array_serializer.rb
 - lib/dsl.rb
 - lib/job.rb
-- lib/mapper.rb
-- lib/reducer.rb
+- lib/mappers/base_mapper.rb
+- lib/mappers/pass_through_mapper.rb
+- lib/reducers/base_reducer.rb
+- lib/reducers/pass_through_reducer.rb
+- lib/reducers/sum_reducer.rb
 - lib/test_runner.rb
-- lib/tuple.rb
 has_rdoc: false
 homepage: 
 post_install_message: 

data/lib/mapper.rb

@@ -1,40 +0,0 @@
-module Mandy
-  class Mapper
-    
-    KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
-    
-    def initialize(input=STDIN, output=STDOUT)
-      @input, @output = input, output
-      
-    end
-    
-    def self.compile(&blk)
-      Class.new(Mandy::Mapper) do 
-        self.class_eval do
-          define_method(:mapper, blk) if blk
-        end
-      end
-    end
-    
-    def execute
-      @input.each_line do |line|
-         key, value = line.split(KEY_VALUE_SEPERATOR)
-         key, value = nil, key if value.nil?
-         value.chomp!
-         mapper(key, value)
-      end
-    end
-    
-    def emit(key, value=nil)
-      key = 'nil' if key.nil?
-      @output.puts(value.nil? ? key.to_s : "#{key}\t#{value}")
-    end
-    
-    private
-    
-    def mapper(key,value)
-      # default map is simply a pass-through
-      emit(key, value)
-    end
-  end
-end

data/lib/reducer.rb

@@ -1,44 +0,0 @@
-module Mandy
-  class Reducer
-    
-    KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
-    
-    def initialize(input=STDIN, output=STDOUT)
-      @input, @output = input, output
-    end
-    
-    def self.compile(&blk)
-      Class.new(Mandy::Reducer) do 
-        self.class_eval do
-          define_method(:reducer, blk) if blk
-        end
-      end
-    end
-    
-    def execute
-      last_key, values = nil, []
-      @input.each_line do |line|
-         key, value = line.split(KEY_VALUE_SEPERATOR)
-         value.chomp!
-         last_key = key if last_key.nil?
-         if key != last_key
-           reducer(last_key, values)
-           last_key, values = key, []
-         end
-         values << value
-      end
-    end
-    
-    def emit(key, value=nil)
-      key = 'nil' if key.nil?
-      @output.puts(value.nil? ? key.to_s : "#{key}\t#{value}")
-    end
-    
-    private
-    
-    def reducer(key,value)
-      # default reducer is simply a pass-through
-      emit(key, value)
-    end
-  end
-end