mandy 0.2.14

data/Rakefile

@@ -0,0 +1,17 @@
+require 'rubygems'
+require "rake"
+require File.expand_path(File.join(File.dirname(__FILE__), 'lib', 'mandy'))
+require 'spec/rake/spectask'
+
+task :default => :spec
+
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_files = FileList['spec/lib/**/*_spec.rb']
+  t.spec_opts = %w{-f s -c -L mtime}
+end
+
+task :gem do
+  `sudo gem build mandy.gemspec`
+  `mkdir pkg; mv mandy-*.gem pkg/`
+  `sudo gem install pkg/mandy-*.gem`
+end

data/bin/mandy

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "mandy"
+
+puts "\nYou are running Mandy!"
+puts "Here are the commands at your disposal..."
+puts ''
+
+{
+  'mandy-install' => 'Installs the Mandy Rubygem on several hosts via ssh.',
+  'mandy-local' => 'Run a Map/Reduce task locally without requiring hadoop',
+  'mandy-hadoop' => 'Run a Map/Reduce task on hadoop using the provided cluster config',
+  'mandy-rm' => 'remove a file or directory from HDFS',
+  'mandy-put' => 'upload a file into HDFS',
+  'mandy-map' => 'Run a map task reading on STDIN and writing to STDOUT',
+  'mandy-reduce' => 'Run a reduce task reading on STDIN and writing to STDOUT'
+}.each do |command, description|
+  
+  puts "#{command.ljust(15)} #{description}"
+end

data/bin/mandy-get

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: mandy-get hdfs_file_location local_file_destination [options]"
+
+  opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config|
+    options.config = config
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+
+def absolute_path(path)
+  path =~ /^\// ? path : File.join(Dir.pwd, path)
+end
+
+remote_file = ARGV[0]
+local_file = ARGV[1]
+config = absolute_path(options.config || 'cluster.xml')
+
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -getmerge #{remote_file} #{local_file}`

data/bin/mandy-hadoop

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "mandy"
+require 'optparse'
+require 'ostruct'
+require 'uri'
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: mandy-hadoop script input output [options]"
+
+  opts.on("-p", "--payload PAYLOAD", "Add a working directory to be sent to the cluster.") do |payload|
+    options.payload = payload
+  end
+  
+  opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config|
+    options.config = config
+  end
+  
+  opts.on("-v", '--variables name=value', "Pass additional parameters to jobs") do |config|
+    options.cmdenv = config
+  end
+  
+  opts.on("-j", '--json {"key":"1 value"}', "Pass JSON encoded parameters to jobs") do |config|
+    options.cmdenv = "json=#{URI.encode(config)}"
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+def absolute_path(path)
+  path =~ /^\// ? path : File.join(Dir.pwd, path)
+end
+
+file   = ARGV[0]
+filename = File.basename(file)
+input  = ARGV[1]
+output_folder = ARGV[2]
+config = options.config || 'cluster.xml'
+payload = options.payload ? Mandy::Packer.pack(options.payload) : ARGV[0]
+cmdenv = options.cmdenv
+
+at_exit { Mandy::Packer.cleanup!(payload) }
+
+require absolute_path(file)
+
+output = nil
+
+Mandy::Job.jobs.each_with_index do |job, i|
+  
+  jobconf = job.settings.map { |key, value| %(-D #{key}='#{value}') }.join(' ')
+  output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
+  
+  command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar #{jobconf}\
+                  -conf '#{config}' \
+                  -input "#{input}"  \
+                  -mapper "mandy-map #{filename} '#{job.name}' #{File.basename(payload)}"  \
+                  -reducer "mandy-reduce #{filename} '#{job.name}' #{File.basename(payload)}"  \
+                  -file "#{payload}" \
+                  -cmdenv #{cmdenv} \
+                  -output "#{output}")
+
+  `#{command}`
+
+  # puts "#{command}"
+  input = output
+end
+
+# print out the output location so caller can know where to get the results from
+puts output

data/bin/mandy-install

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+if ARGV.size==0
+  puts "USAGE: mandy-install user@server1.com [user@server2.com ...]"
+  exit
+end
+
+ARGV.each do |server|
+  puts "Installing on #{server}..."
+  system %(ssh #{server} "sudo gem install trafficbroker-mandy --source http://gems.github.com")
+end

data/bin/mandy-local

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+if ARGV.size==0
+  puts "USAGE: mandy-local my_script.rb local_input_file local_output_folder"
+  exit
+end
+
+require "fileutils"
+
+def absolute_path(path)
+  path =~ /^\// ? path : File.join(Dir.pwd, path)
+end
+
+file   = absolute_path(ARGV[0])
+input  = absolute_path(ARGV[1])
+output_folder = FileUtils.mkdir_p(absolute_path(ARGV[2]))
+require file
+
+out = nil
+Mandy::Job.jobs.each_with_index do |job, i|
+  out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
+  puts "Running #{job.name}..."
+  `cat #{input} | mandy-map #{file} "#{job.name}" | sort | mandy-reduce #{file} "#{job.name}" > #{out}`
+  input = out
+end
+
+puts out

data/bin/mandy-map

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "mandy"
+
+if ARGV.size==0
+  puts "USAGE: mandy-map my_script.rb 'Job Name' [payload]"
+  exit
+end
+
+if ARGV.size > 2
+  payload = ARGV[2]
+  Mandy::Packer.unpack(payload)
+end
+
+def absolute_path(path)
+  path =~ /^\// ? path : File.join(Dir.pwd, path)
+end
+
+file   = absolute_path(ARGV[0])
+job_name = ARGV[1]
+
+require file
+
+Mandy::Job.find_by_name(job_name).run_map

data/bin/mandy-put

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: mandy-put local_file_or_folder hdfs_destination_location [options]"
+
+  opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config|
+    options.config = config
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+def absolute_path(path)
+  path =~ /^\// ? path : File.join(Dir.pwd, path)
+end
+
+source = absolute_path(ARGV[0])
+dest = ARGV[1]
+config = options.config || 'cluster.xml'
+
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -copyFromLocal #{source} #{dest}`

data/bin/mandy-reduce

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "mandy"
+
+if ARGV.size==0
+  puts "USAGE: mandy-reduce my_script.rb 'Job Name' [payload]"
+  exit
+end
+
+def absolute_path(path)
+  path =~ /^\// ? path : File.join(Dir.pwd, path)
+end
+
+if ARGV.size > 2
+  payload = ARGV[2]
+  Mandy::Packer.unpack(payload)
+end
+
+file   = absolute_path(ARGV[0])
+job_name = ARGV[1]
+
+require file
+
+Mandy::Job.find_by_name(job_name).run_reduce

data/bin/mandy-rm

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: mandy-rm file_or_folder_on_hdfs [options]"
+
+  opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config|
+    options.config = config
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+file = ARGV[0]
+config = options.config || 'cluster.xml'
+
+`$HADOOP_HOME/bin/hadoop fs -conf #{config} -rmr #{file}`

data/lib/dsl.rb

@@ -0,0 +1,10 @@
+module Mandy
+  module DSL
+    def job(name, &blk)
+      job = Mandy::Job.new(name)
+      job.instance_eval(&blk) unless blk.nil?
+      Mandy::Job.jobs << job
+      job
+    end
+  end
+end

data/lib/job.rb

@@ -0,0 +1,92 @@
+module Mandy
+  class Job
+    class << self
+      def jobs
+        @jobs ||= []
+      end
+      
+      def find_by_name(name)
+        jobs.find {|job| job.name == name }
+      end
+    end
+    
+    attr_reader :settings
+    attr_reader :name
+    
+    def initialize(name, &blk)
+      @name = name
+      @settings = {}
+      @modules = []
+      @mapper_class = Mandy::Mappers::PassThroughMapper
+      @reducer_class = Mandy::Reducers::PassThroughReducer
+      set('mapred.job.name', name)
+      instance_eval(&blk) if blk
+    end
+    
+    def mixin(*modules)
+      modules.each {|m| @modules << m}
+    end
+    alias_method :serialize, :mixin
+    
+    def input_format(format)
+      @input_format = format
+    end
+
+    def output_format(format)
+      @output_format = format
+    end
+    
+    def set(key, value)
+      @settings[key.to_s] = value.to_s
+    end
+    
+    def map_tasks(count)
+      set('mapred.map.tasks', count)
+    end
+    
+    def reduce_tasks(count)
+      set('mapred.reduce.tasks', count)
+    end
+    
+    def store(type, name, options={})
+      Mandy.stores[name] = case type
+      when :hbase
+        Stores::HBase.new(options)
+      else
+        raise "Unknown store type #{type}"
+      end
+    end
+    
+    def map(klass=nil, &blk)
+      @mapper_class = klass || Mandy::Mappers::Base.compile(&blk)
+      @modules.each {|m| @mapper_class.send(:include, m) }
+      @mapper_class
+    end
+    
+    def reduce(klass=nil, &blk)
+      @reducer_class = klass || Mandy::Reducers::Base.compile(&blk)
+      @modules.each {|m| @reducer_class.send(:include, m) }
+      @reducer_class
+    end
+    
+    def run_map(input=STDIN, output=STDOUT, &blk)
+      @mapper_class.send(:include, Mandy::IO::OutputFormatting) unless reducer_defined?
+      mapper = @mapper_class.new(input, output, @input_format, @output_format)
+      yield(mapper) if blk
+      mapper.execute
+    end
+    
+    def run_reduce(input=STDIN, output=STDOUT, &blk)
+      reducer = @reducer_class.new(input, output, @input_format, @output_format)
+      yield(reducer) if blk
+      reducer.execute
+    end
+    
+    private
+    
+    def reducer_defined?
+      @reducer_class != Mandy::Reducers::PassThroughReducer
+    end
+    
+  end
+end

data/lib/mandy.rb

@@ -0,0 +1,35 @@
+require "rubygems"
+require "json"
+require "uri"
+
+%w(
+  support/formatting
+  task
+  dsl 
+  job 
+  packer
+  support/tuple 
+  support/array_serializer 
+  mappers/base_mapper 
+  mappers/transpose_mapper 
+  mappers/pass_through_mapper 
+  reducers/base_reducer 
+  reducers/pass_through_reducer 
+  reducers/sum_reducer 
+  reducers/max_reducer 
+  reducers/min_reducer
+  reducers/transpose_reducer
+  serializers/json
+  stores/hbase
+  stores/in_memory
+  test_runner
+  ruby-hbase
+).each {|file| require File.join(File.dirname(__FILE__), file) }
+
+module Mandy
+  class << self
+    def stores
+      @stores||={}
+    end
+  end
+end

data/lib/mappers/base_mapper.rb

@@ -0,0 +1,30 @@
+module Mandy
+  module Mappers
+    class Base < Mandy::Task
+      include Mandy::IO::InputFormatting
+
+      def self.compile(&blk)
+        Class.new(Mandy::Mappers::Base) do 
+          self.class_eval do
+            define_method(:mapper, blk) if blk
+          end
+        end
+      end
+      
+      def execute
+        @input.each_line do |line|
+           key, value = line.split(KEY_VALUE_SEPERATOR, 2)
+           key, value = nil, key if value.nil?
+           value.chomp!
+           mapper(input_deserialize_key(key), input_deserialize_value(value))
+        end
+      end
+
+      private
+    
+      def mapper(key,value)
+        #nil
+      end
+    end
+  end
+end

data/lib/mappers/pass_through_mapper.rb

@@ -0,0 +1,10 @@
+module Mandy
+  module Mappers
+    class PassThroughMapper < Base
+      def mapper(key,value)
+        # default map is simply a pass-through
+        emit(key, value)
+      end
+    end
+  end
+end

data/lib/mappers/transpose_mapper.rb

@@ -0,0 +1,10 @@
+module Mandy
+  module Mappers
+    class TransposeMapper < Base
+      def mapper(key,value)
+        # default map is simply a pass-through
+        emit(value, key)
+      end
+    end
+  end
+end

data/lib/packer.rb

@@ -0,0 +1,25 @@
+require "fileutils"
+
+module Mandy
+  class Packer
+    TMP_DIR = '/tmp/mandy'
+    
+    def self.pack(dir)
+      return dir if File.file?(dir)
+      FileUtils.mkdir_p(TMP_DIR)
+      tmp_path = "#{TMP_DIR}/packed-job-#{Time.now.to_i}.tar"
+      Dir.chdir(dir) { `tar -cf #{tmp_path} *` }
+      tmp_path
+    end
+    
+    def self.unpack(file)
+      return false unless File.extname(file) == '.tar'
+      `tar -xf #{file}`
+    end
+    
+    def self.cleanup!(file)
+      return false unless File.extname(file) == '.tar'
+      `rm #{file}`
+    end
+  end
+end

data/lib/reducers/base_reducer.rb

@@ -0,0 +1,36 @@
+module Mandy
+  module Reducers
+    class Base < Mandy::Task
+      include Mandy::IO::OutputFormatting
+
+      def self.compile(&blk)
+        Class.new(Mandy::Reducers::Base) do 
+          self.class_eval do
+            define_method(:reducer, blk) if blk
+          end
+        end
+      end
+    
+      def execute
+        last_key, values = nil, []
+        @input.each_line do |line|
+           key, value = line.split(KEY_VALUE_SEPERATOR)
+           value.chomp!
+           last_key = key if last_key.nil?
+           if key != last_key
+             reducer(last_key, values)
+             last_key, values = key, []
+           end
+           values << value
+        end
+        reducer(deserialize_key(last_key), values.map {|v| deserialize_value(v) })
+      end
+    
+      private
+      
+      def reducer(key,values)
+        #nil
+      end
+    end
+  end
+end

data/lib/reducers/max_reducer.rb

@@ -0,0 +1,10 @@
+module Mandy
+  module Reducers
+    class MaxReducer < Base
+      def reducer(key,values)
+        values.map! {|value| value.to_f}
+        emit(key, values.max)
+      end
+    end
+  end
+end

data/lib/reducers/min_reducer.rb

@@ -0,0 +1,10 @@
+module Mandy
+  module Reducers
+    class MinReducer < Base
+      def reducer(key,values)
+        values.map! {|value| value.to_f}
+        emit(key, values.min)
+      end
+    end
+  end
+end

data/lib/reducers/pass_through_reducer.rb

@@ -0,0 +1,9 @@
+module Mandy
+  module Reducers
+    class PassThroughReducer < Base
+      def reducer(key,values)
+        values.each {|value| emit(key, value) }
+      end
+    end
+  end
+end

data/lib/reducers/sum_reducer.rb

@@ -0,0 +1,9 @@
+module Mandy
+  module Reducers
+    class SumReducer < Base
+      def reducer(key,values)
+        emit(key, values.inject(0) {|sum,count| sum+count.to_f})
+      end
+    end
+  end
+end

data/lib/reducers/transpose_reducer.rb

@@ -0,0 +1,9 @@
+module Mandy
+  module Reducers
+    class TransposeReducer < Base
+      def reducer(key,values)
+        values.each {|value| emit(value, key) }
+      end
+    end
+  end
+end

data/lib/ruby-hbase.rb

@@ -0,0 +1,10 @@
+$:.unshift File.dirname(__FILE__)
+
+require "rubygems"
+require "net/http"
+require "erb"
+require "xml/libxml"
+
+require "ruby-hbase/xml_decoder"
+require "ruby-hbase/hbase_table"
+require "ruby-hbase/scanner"

data/lib/ruby-hbase/hbase_table.rb

@@ -0,0 +1,166 @@
+module HBase
+  class RowNotFound < Exception
+    def initialize(msg=nil)
+      super
+    end
+  end
+  
+  class HTable    
+    include XmlDecoder
+    
+    def initialize(table_uri)
+      @table_uri = table_uri
+
+      @uri = URI.parse(table_uri)
+      
+      @host, @table_name = @uri.host, @uri.path.split("/").last
+    end
+
+    def name
+      @table_name
+    end
+
+    ######################
+    # Meta-type requests
+
+    def start_keys
+      raise NotImplementedError
+    end
+
+
+    def column_descriptors
+      column_families = []
+      
+      # get the xml for the column descriptors
+      response = Net::HTTP.get_response(@uri.host, "/api/#{@table_name}", @uri.port)
+      body = response.body
+      
+      # parse the xml into a document
+      doc = XML::Parser.string(body).parse
+      
+      doc.find("/table/columnfamilies/columnfamily").each do |node|
+        colfam = {}
+        colfam[:name] = node.find_first("name").content.strip.chop
+        column_families << colfam
+      end
+      column_families
+    end    
+
+    
+    #####################
+    # Standard CRUD ops
+    
+    DEFAULT_GET_OPTIONS = {:timestamp => nil, :columns => nil}
+    
+    def get(key, options = {})
+      opts = DEFAULT_GET_OPTIONS.merge(options)
+      
+      columns = Array(opts.delete(:columns)).compact
+      timestamp = opts.delete(:timestamp)
+      timestamp = (timestamp.to_f * 1000).to_i.to_s if timestamp
+      
+      Net::HTTP.start(@uri.host, @uri.port) do |session|
+        columns_query = columns.map{ |name| "column=#{name}" }.join("&")
+
+        ts_section = timestamp ? "/#{timestamp}" : ""
+
+        query_string = "?" + columns_query
+                      
+        query = "/api/#{@table_name}/row/#{url_encode(key)}#{ts_section}#{query_string}"
+        response = session.get(query, {"Accept" => "*/*"})
+
+        case response.code.to_i
+          when 200 #success!
+            body = response.body
+            parse_row_result(body).last
+          when 204 #no data - probably an incorrect colname
+            raise "Didn't get any data back - check your column names!"
+          when 404
+            raise RowNotFound, "Could not find row '#{key}'"
+          else
+            nil
+        end
+      end
+    end
+    
+    def put(key, keys_and_values, timestamp = nil)
+      Net::HTTP.start(@uri.host, @uri.port) do |session|
+        xml = "<columns>"
+        
+        ts_section = timestamp ? "/#{(timestamp.to_f * 1000).to_i}" : ""
+        
+        keys_and_values.each do |name, value|
+          xml << "<column><name>#{name}</name><value>#{[value.to_s].pack("m")}</value></column>"          
+        end
+        
+        xml << "</columns>"
+        
+        query = "/api/#{@table_name}/row/#{url_encode(key)}#{ts_section}"
+        response = session.post(query, xml, {"Content-type" => "text/xml"})
+        
+        case response.code.to_i
+          when 200
+            true
+          else
+            unexpected_response(response)
+        end
+      end
+    end
+        
+    def delete(row, columns = nil, timestamp = nil)
+      Net::HTTP.start(@uri.host, @uri.port) do |session|
+        columns_query = Array(columns).compact.map{ |name| "column=#{name}" }.join("&")
+        
+        response = session.delete("/api/#{@table_name}/row/#{row}?#{columns_query}")  
+        case response.code.to_i
+          when 202
+            return true
+          else
+            unexpected_response(response)
+        end
+
+      end
+    end
+    
+    #######################
+    # Scanning interface
+    
+    def get_scanner(start_row, end_row, timestamp = nil, columns = nil)
+      start_row_query = start_row ? "start_row=#{start_row}" : nil
+      end_row_query = end_row ? "end_row=#{end_row}" : nil      
+      timestamp_section = timestamp ? "/#{(timestamp.to_f * 1000).to_i}" : nil
+      columns_section = columns ? columns.map{ |col| "column=#{col}" }.join("&") : nil
+
+      query_string = [start_row_query, end_row_query, 
+                      timestamp_section, columns_section].compact.join("&")
+
+      path = ""
+
+      # open the scanner
+      Net::HTTP.start(@uri.host, @uri.port) do |session|
+        response = session.post("/api/#{@table_name}/scanner?#{query_string}", 
+          "", {"Accept" => "text/xml"}
+        )
+        
+        case response.code.to_i
+          when 201
+            # redirect - grab the path and send
+            Scanner.new(self, "http://#{@uri.host}:#{@uri.port}" + response["Location"])
+          else
+            unexpected_response(response)
+        end
+      end      
+    end
+    
+    
+    private
+      
+    def url_encode(str)
+      ERB::Util.url_encode(str)
+    end
+    
+    def unexpected_response(response)
+      raise "Unexpected response code #{response.code.to_i}:\n#{response.body}"
+    end
+  end
+end

data/lib/ruby-hbase/scanner.rb

@@ -0,0 +1,55 @@
+module HBase
+  class Scanner
+    include XmlDecoder
+    
+    def initialize(table, scanner_uri)
+      @table, @scanner_uri = table, scanner_uri
+    end
+    
+    def close
+      
+    end
+    
+    def next
+      
+    end
+    
+    def each
+      parsed_uri = URI.parse(@scanner_uri)
+      Net::HTTP.start(parsed_uri.host, parsed_uri.port) do |session|
+        while true
+          response = session.post(@scanner_uri, "")
+
+          case response.code.to_i
+            when 404
+              # over
+              break
+            when 200
+              # item
+              yield *parse_row_result(response.body)
+            else
+              # error
+              raise "Unexpected response code #{response.code}, body:\n#{response.body}"
+          end
+        end
+      end
+    end
+
+    private
+    
+    # def parse_row(xml)
+    #   doc = REXML::Document.new(xml)
+    # 
+    #   result = {}
+    # 
+    #   doc.root.each_element("/row/column") do |column|
+    #     name = column.get_elements("name")[0].text.strip
+    #     value = column.get_elements("value")[0].text.strip.unpack("m").first
+    #     result[name] = value
+    #   end
+    #   
+    #   [doc.root.get_elements("name")[0].text.strip, result]
+    # end
+    
+  end
+end

data/lib/ruby-hbase/version.rb

@@ -0,0 +1,9 @@
+module RubyHbase #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 4
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/ruby-hbase/xml_decoder.rb

@@ -0,0 +1,18 @@
+module HBase
+  module XmlDecoder
+    def parse_row_result(xml)
+      doc = XML::Parser.string(xml).parse
+
+      name_node = doc.root.find_first("/row/name")
+      name = name_node ? name_node.content.strip : nil
+      
+      values = {}
+      
+      doc.find("/row/columns/column").each do |node|
+        values[node.find_first("name").content.strip.unpack('m').first] = node.find_first("value").content.strip.unpack("m").first
+      end
+      
+      [name, values]
+    end
+  end
+end

data/lib/serializers/json.rb

@@ -0,0 +1,13 @@
+module Mandy
+  module Serializers
+    module Json
+      def serialize_value(value)
+        value.to_json
+      end
+      
+      def deserialize_value(value)
+        JSON.parse(value)
+      end
+    end
+  end
+end

data/lib/stores/hbase.rb

@@ -0,0 +1,24 @@
+module Mandy
+  module Stores
+    class HBase
+      attr_reader :options
+      
+      def initialize(options)
+        @options = options
+        @table = ::HBase::HTable.new(options[:url])
+      end
+      
+      def get(key)
+        @table.get(key)
+      end
+      
+      def put(key, values)
+        @table.put(key, values)
+      end
+      
+      def ==(other)
+        self.class == other.class && self.options == other.options
+      end
+    end
+  end
+end

data/lib/stores/in_memory.rb

@@ -0,0 +1,24 @@
+module Mandy
+  module Stores
+    class InMemory
+      attr_reader :options
+      
+      def initialize(options={})
+        @options = options
+        @table = {}
+      end
+      
+      def get(key)
+        @table[key.to_s]
+      end
+      
+      def put(key, values)
+        @table[key.to_s] = values
+      end
+      
+      def ==(other)
+        self.class == other.class && self.options == other.options
+      end
+    end
+  end
+end

data/lib/support/array_serializer.rb

@@ -0,0 +1,32 @@
+module Mandy
+  class ArraySerializer
+    
+    SEPERATOR = '|' unless defined?(SEPERATOR)
+    
+    attr_reader :items
+    
+    def initialize(items)
+      @items = items || []
+    end
+    
+    def to_s
+      @items.join(SEPERATOR)
+    end
+    
+    def ==(other)
+      (self.class == other.class && self.items == other.items) || (other.is_a?(Array) && self.items == other)
+    end
+    
+    def to_a
+      @items
+    end
+    
+    def self.from_s(str)
+      str.split(SEPERATOR)
+    end
+    
+    def self.tuples_from_s(str)
+      from_s(str).map {|s| Tuple.from_s(s) }
+    end
+  end
+end

data/lib/support/formatting.rb

@@ -0,0 +1,27 @@
+module Mandy
+  module IO
+    module InputFormatting
+      def input_deserialize_key(key)
+        return key if input_format && input_format == :plain
+        deserialize_key(key)
+      end
+      
+      def input_deserialize_value(value)
+        return value if input_format && input_format == :plain
+        deserialize_value(value)
+      end
+    end
+    
+    module OutputFormatting
+      def output_serialize_key(key)
+        return key if output_format && output_format == :plain
+        serialize_key(key)
+      end
+      
+      def output_serialize_value(value)
+        return value if output_format && output_format == :plain
+        serialize_value(value)
+      end
+    end
+  end
+end

data/lib/support/tuple.rb

@@ -0,0 +1,40 @@
+module Mandy
+  class Tuple
+    
+    SEPERATOR = ',' unless defined?(SEPERATOR)
+    
+    attr_accessor :name, :value
+    
+    def initialize(name, value, name_accessor = nil, value_accessor = nil)
+      @name, @value = name, value
+      alias_accessor(name_accessor, :name) unless name_accessor.nil?
+      alias_accessor(value_accessor, :value) unless value_accessor.nil?
+    end
+    
+    def to_s
+      %(#{@name}#{SEPERATOR}#{@value})
+    end
+    
+    def self.from_s(str)
+      parts = str.split(SEPERATOR)
+      raise "Can't create tuple from #{str.inspect}. Format should be 'A#{SEPERATOR}B'" unless parts.size==2
+      new(*parts)
+    end
+    
+    def inspect
+      %(<Tuple #{self.to_s}>)
+    end
+    
+    def ==(other)
+      return false unless self.class == other.class
+      self.name == other.name && self.value == other.value
+    end
+    
+    private
+    
+    def alias_accessor(new_accessor, old_accessor)
+      self.class.send(:alias_method, new_accessor, old_accessor)
+      self.class.send(:alias_method, :"#{new_accessor}=", :"#{old_accessor}=")
+    end
+  end
+end

data/lib/task.rb

@@ -0,0 +1,83 @@
+module Mandy
+  class Task
+    JSON_PAYLOAD_KEY = "json"
+    KEY_VALUE_SEPERATOR = "\t" unless defined?(KEY_VALUE_SEPERATOR)
+    NUMERIC_PADDING = 16
+    
+    attr_reader :input_format, :output_format
+    
+    def initialize(input=STDIN, output=STDOUT, input_format = nil, output_format = nil)
+      @input, @output = input, output
+      @input_format, @output_format = input_format, output_format
+    end
+
+    def emit(key, value=nil)
+      key = 'nil' if key.nil?
+      @output.puts(value.nil? ? key.to_s : "#{output_serialize_key(key)}\t#{output_serialize_value(value)}")
+    end
+    
+    def get(store, key)
+      Mandy.stores[store].get(key)
+    end
+
+    def put(store, key, values)
+      Mandy.stores[store].put(key, values)
+    end
+
+    private
+    def pad(key)
+      key_parts = key.to_s.split(".")
+      key_parts[0] = key_parts.first.rjust(NUMERIC_PADDING, '0')
+      key_parts.join('.')
+    end
+    
+    def update_status(message)
+      STDERR.puts("reporter:status:#{message}")
+    end
+    
+    def update_counter(group, counter, count)
+      STDERR.puts("reporter:counter:#{group},#{counter},#{count}")
+    end
+    
+    def parameter(name)
+      return find_json_param(name) if json_provided?
+      ENV[name.to_s]
+    end
+    
+    def find_json_param(name)
+      @json_args ||= JSON.parse(URI.decode(ENV[JSON_PAYLOAD_KEY]))
+      @json_args[name.to_s]
+    end
+    
+    def json_provided?
+      !ENV[JSON_PAYLOAD_KEY].nil?
+    end
+    
+    def deserialize_key(key)
+      key
+    end
+    
+    def deserialize_value(value)
+      value
+    end
+    
+    def serialize_key(key)
+      key = pad(key) if key.is_a?(Numeric) && key.to_s.length < NUMERIC_PADDING
+      key
+    end
+
+    def serialize_value(value)
+      value = ArraySerializer.new(value) if value.is_a?(Array)
+      value.to_s
+    end
+    
+    def output_serialize_key(key)
+      serialize_key(key)
+    end
+
+    def output_serialize_value(value)
+      serialize_value(value)
+    end
+    
+  end
+end

data/lib/test_runner.rb

@@ -0,0 +1,75 @@
+module Mandy
+  class TestRunner
+    attr_reader :job
+    
+    def initialize(job=Mandy::Job.jobs.first.name, opts={})
+      ENV[Mandy::Task::JSON_PAYLOAD_KEY] = opts[:parameters].to_json
+      @job = Mandy::Job.find_by_name(job)
+    end
+    
+    def map(input_stream, output_stream=StringIO.new(''), &blk)
+      input_stream = input_from_array(input_stream) if input_stream.is_a?(Array)
+      input_stream = StringIO.new(input_stream) if input_stream.is_a?(String)
+      @job.run_map(input_stream, output_stream, &blk)
+      output_stream.rewind
+      output_stream
+    end
+    
+    def reduce(input_stream, output_stream=StringIO.new(''), &blk)
+      input_stream = input_from_hash(input_stream) if input_stream.is_a?(Hash)
+      input_stream = StringIO.new(input_stream) if input_stream.is_a?(String)
+      @job.run_reduce(input_stream, output_stream, &blk)
+      output_stream.rewind
+      output_stream
+    end
+    
+    def self.end_to_end(verbose=false)
+      CompositeJobRunner.new(Mandy::Job.jobs,verbose)
+    end
+    
+    private
+    
+    def input_from_array(input)
+      input.join("\n")
+    end
+    
+    def input_from_hash(input)
+      output = []
+      input.each do |key, values|
+        output << "#{key}\t#{values}" and next unless values.is_a?(Array)
+        values.each { |value| output << "#{key}\t#{value}" }
+      end
+      input_from_array(output.sort)
+    end
+    
+    class CompositeJobRunner
+      def initialize(jobs, verbose=false)
+        @jobs = jobs
+        @verbose = verbose
+        @job_runners = @jobs.map { |job| Mandy::TestRunner.new(job.name) }
+      end
+      
+      def execute(input_stream, output_stream=StringIO.new(''))
+        map_temp = StringIO.new('')
+        reduce_temp = StringIO.new('')
+        @job_runners.each_with_index do |runner, index|
+          runner.map(input_stream, map_temp)
+          if @verbose
+            puts "#{runner.job.name} [MAP] #{map_temp.readlines.inspect}"
+            map_temp.rewind
+          end
+          reduce_input = StringIO.new(map_temp.readlines.sort.join(''))
+          runner.reduce(reduce_input, (index==@job_runners.size-1 ? output_stream : reduce_temp))
+          if @verbose
+            puts "#{runner.job.name} [RED] #{reduce_temp.readlines.inspect}"
+            reduce_temp.rewind
+          end
+          input_stream = reduce_temp
+          map_temp = StringIO.new('')
+          reduce_temp = StringIO.new('')
+        end
+        output_stream
+      end
+    end
+  end
+end

data/readme.md

@@ -0,0 +1,11 @@
+Mandy - Simplified Hadoop distribution for Ruby code
+====================================================
+
+Mandy hides the differences and complexities between running map/reduce tasks locally or distributed or in test environments.
+
+It provides a simple DSL to define new jobs for distribution. See examples/word_count.rb for a demo of some functionality.
+Run the word count example locally with...
+
+    mandy-local examples/word_count.rb examples/alice.txt examples/output
+
+Mandy is licensed under the MIT Licence, please see LICENCE for further information.

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification 
+name: mandy
+version: !ruby/object:Gem::Version 
+  version: 0.2.14
+platform: ruby
+authors: 
+- Andy Kent
+- Paul Ingles
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-07-09 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Map/Reduce
+email: andy.kent@me.com
+executables: 
+- mandy
+- mandy-hadoop
+- mandy-local
+- mandy-map
+- mandy-put
+- mandy-get
+- mandy-reduce
+- mandy-rm
+- mandy-install
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/mandy-hadoop
+- bin/mandy-local
+- bin/mandy-map
+- bin/mandy-get
+- bin/mandy-put
+- bin/mandy-reduce
+- readme.md
+- Rakefile
+- lib/mandy.rb
+- lib/support/tuple.rb
+- lib/support/formatting.rb
+- lib/support/array_serializer.rb
+- lib/task.rb
+- lib/dsl.rb
+- lib/job.rb
+- lib/mappers/base_mapper.rb
+- lib/mappers/transpose_mapper.rb
+- lib/mappers/pass_through_mapper.rb
+- lib/packer.rb
+- lib/reducers/base_reducer.rb
+- lib/reducers/transpose_reducer.rb
+- lib/reducers/pass_through_reducer.rb
+- lib/reducers/sum_reducer.rb
+- lib/reducers/max_reducer.rb
+- lib/reducers/min_reducer.rb
+- lib/serializers/json.rb
+- lib/stores/hbase.rb
+- lib/stores/in_memory.rb
+- lib/ruby-hbase.rb
+- lib/ruby-hbase/hbase_table.rb
+- lib/ruby-hbase/scanner.rb
+- lib/ruby-hbase/version.rb
+- lib/ruby-hbase/xml_decoder.rb
+- lib/test_runner.rb
+has_rdoc: true
+homepage: http://github.com/trafficbroker/mandy
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.4
+signing_key: 
+specification_version: 2
+summary: Map/Reduce
+test_files: []
+