mapredus 0.0.1

data/lib/mapredus/process.rb

@@ -0,0 +1,366 @@
+module MapRedus
+
+  # This is what keeps track of our map reduce processes
+  #  
+  # We use a redis key to identify the id of map reduce process
+  # the value of the redis object is a json object which contains: 
+  # 
+  #   {
+  #     mapper : mapclass,
+  #     reducer : reduceclass,
+  #     finalizer : finalizerclass,
+  #     partitioner : <not supported>,
+  #     combiner : <not supported>,
+  #     ordered : true_or_false   ## ensures ordering keys from the map output --> [ order, key, value ],
+  #     synchronous : true_or_false   ## runs the process synchronously or not (generally used for testing)
+  #     result_timeout : lenght of time a result is saved ## 3600 * 24
+  #     keyname : the location to the save the result of the process (cache location)
+  #     state : the current state of the process (shouldn't be set by the process and starts off as nil)
+  #   }
+  #
+  # The user has the ability in subclassing this class to create extra features if needed
+  # 
+  class Process
+    # Public: Keep track of information that may show up as the redis json value
+    #         This is so we know exactly what might show up in the json hash
+    READERS = [:pid]
+    ATTRS = [:inputter, :mapper, :reducer, :finalizer, :outputter, :ordered, :synchronous, :result_timeout, :keyname, :state]
+    READERS.each { |r| attr_reader r }
+    ATTRS.each { |a| attr_accessor a }
+
+    DEFAULT_TIME = 3600 * 24
+    def initialize(pid, json_info)
+      @pid = pid
+      read(json_info)
+    end
+
+    def read(json_info)
+      @inputter = Helper.class_get(json_helper(json_info, :inputter))
+      @mapper = Helper.class_get(json_helper(json_info, :mapper))
+      @reducer = Helper.class_get(json_helper(json_info, :reducer))
+      @finalizer = Helper.class_get(json_helper(json_info, :finalizer))
+      @ordered = json_helper(json_info, :ordered)
+      @synchronous = json_helper(json_info, :synchronous)
+      @result_timeout = json_helper(json_info, :result_timeout) || DEFAULT_TIME
+      @keyname = json_helper(json_info, :keyname)
+      @state = json_helper(json_info, :state) || NOT_STARTED
+      @outputter = json_helper(json_info, :outputter)
+      @outputter = @outputter ? Helper.class_get(@outputter) : MapRedus::Outputter
+    end
+
+    def json_helper(json_info, key)
+      json_info[key.to_s] || json_info[key.to_sym]
+    end
+
+    def to_s; to_json; end
+
+    def to_hash
+      (ATTRS + READERS).inject({}) do |h, attr|
+        h[attr] = send(attr)
+        h 
+      end
+    end
+
+    def to_json
+      Helper.encode(to_hash)
+    end
+
+    def save
+      FileSystem.sadd( ProcessInfo.processes, @pid ) 
+      FileSystem.save( ProcessInfo.pid(@pid), to_json )
+      self
+    end
+
+    def update(attrs = {})
+      attrs.each do |attr, val|
+        send("#{attr}=", val)
+      end
+      save
+    end
+
+    def reload
+      read(Helper.decode(FileSystem.get(ProcessInfo.pid(@pid))))
+      self
+    end
+
+    # This will not delete if the master is working
+    # It can't get ahold of the files to shred while the master is working
+    #
+    # if safe is set to false, this will delete all the redis stores associated
+    # with this process, but will not kill the process from the queue, if it is
+    # on the queue.  The process operations will fail to work when its data is deleted 
+    #
+    # Examples
+    #   delete(safe)
+    #   # => true or false
+    #
+    # Returns true as long as the master is not working.
+    def delete(safe = true)
+      return false if (safe && Master.working?(@pid))
+      FileSystem.keys("mapredus:process:#{@pid}*").each do |k|
+        FileSystem.del(k)
+      end        
+      FileSystem.srem(ProcessInfo.processes, @pid)
+      FileSystem.set(ProcessInfo.processes_count, 0) if( 0 == FileSystem.scard(ProcessInfo.processes) )
+      true
+    end
+
+    # Iterates through the key, values
+    # 
+    # Example
+    #   each_key_reduced_value(pid)
+    # 
+    # Returns nothing.
+    def each_key_reduced_value
+      map_keys.each do |key|
+        reduce_values(key).each do |value|
+          yield key, value
+        end
+      end
+    end
+
+    # Iterates through the key, values
+    # 
+    # Example
+    #   each_key_nonreduced_value(pid)
+    # 
+    # Returns nothing.
+    def each_key_nonreduced_value
+      map_keys.each do |key|
+        map_values(key).each do |value|
+          yield key, value
+        end
+      end
+    end
+
+    def run( data_object, synchronous = false )
+      update(:synchronous => synchronous)
+      Master.mapreduce( self, data_object )
+      true
+    end
+
+    # TODO:
+    # Should also have some notion of whether the process is completed or not
+    # since the master might not be working, but the process is not yet complete
+    # so it is still running
+    def running?
+      Master.working?(@pid)
+    end
+
+    # Change the process state
+    # if the process is not running and is not synchronous
+    #
+    # Examples
+    #   process.next_state(pid)
+    #
+    # returns the state that the process switched to (or stays the same)
+    def next_state
+      if((not running?) and (not @synchronous))
+        new_state = STATE_MACHINE[self.state]
+        update(:state => new_state)
+        method = "enslave_#{new_state}".to_sym
+        Master.send(method, self) if( Master.respond_to?(method) )
+        new_state
+      end
+    end
+
+    ### The following functions deal with keys/values produced during the
+    ### running of a process
+    
+    # Emissions, when we get map/reduce results back we emit these 
+    # to be stored in our file system (redis)
+    #
+    # key_value  - The key, value
+    #
+    # Examples
+    #   emit_intermediate(key, value)
+    #   # =>
+    #   emit_intermediate(rank, key, value)
+    #
+    # Returns the true on success.
+    def emit_intermediate(*key_value)
+      if( not @ordered )
+        key, value = key_value
+        FileSystem.sadd( ProcessInfo.keys(@pid), key )
+        hashed_key = Helper.hash(key)
+        FileSystem.rpush( ProcessInfo.map(@pid, hashed_key), value )
+      else
+        # if there's an order for the process then we should use a zset above
+        # ordered process's map emits [rank, key, value]
+        #
+        rank, key, value = key_value
+        FileSystem.zadd( ProcessInfo.keys(@pid), rank, key )
+        hashed_key = Helper.hash(key)
+        FileSystem.rpush( ProcessInfo.map(@pid, hashed_key), value )
+      end
+      raise "Key Collision: key:#{key}, #{key.class} => hashed key:#{hashed_key}" if key_collision?(hashed_key, key)
+      true
+    end
+
+    def emit(key, reduce_val)
+      hashed_key = Helper.hash(key)
+      FileSystem.rpush( ProcessInfo.reduce(@pid, hashed_key), reduce_val )
+    end
+
+    def key_collision?(hashed_key, key)
+      not ( FileSystem.setnx( ProcessInfo.hash_to_key(@pid, hashed_key), key ) ||
+            FileSystem.get( ProcessInfo.hash_to_key(@pid, hashed_key) ) == key.to_s )
+    end
+
+    # Saves the result to the specified keyname, using the specified outputter
+    #
+    # Example
+    #   (mapreduce:process:result:KEYNAME)
+    # OR
+    #   process:pid:result
+    #
+    # The client must ensure the the result will not be affected when to_s is applied
+    # since redis stores all values as strings
+    #
+    # Returns true on success.
+    def save_result(result)
+      res = @outputter.encode(result)
+      FileSystem.save(ProcessInfo.result(@pid), res)
+      FileSystem.save(ProcessInfo.result_cache(@keyname), res, @result_timeout) if @keyname
+      true
+    end
+
+    def get_saved_result
+      @outputter.decode(Process.get_saved_result(@keyname))
+    end
+
+    def delete_saved_result
+      Process.delete_saved_result(@keyname)
+    end
+
+    # Keys that the map operation produced
+    #
+    # Examples
+    #   map_keys
+    #   # =>
+    #
+    # Returns the Keys.
+    def map_keys
+      if( not @ordered )
+        FileSystem.smembers( ProcessInfo.keys(@pid) )
+      else
+        FileSystem.zrange( ProcessInfo.keys(@pid), 0, -1 )
+      end
+    end
+
+    def num_values(key)
+      hashed_key = Helper.hash(key)
+      FileSystem.llen( ProcessInfo.map(@pid, hashed_key) )
+    end
+
+    # values that the map operation produced, for a key
+    #
+    # Examples
+    #   map_values(key)
+    #   # =>
+    #
+    # Returns the values.
+    def map_values(key)
+      hashed_key = Helper.hash(key)
+      FileSystem.lrange( ProcessInfo.map(@pid, hashed_key), 0, -1 )
+    end
+
+
+    # values that the reduce operation produced, for a key
+    #
+    # Examples
+    #   reduce_values(key)
+    #   # =>
+    #
+    # Returns the values.
+    def reduce_values(key)
+      hashed_key = Helper.hash(key)
+      FileSystem.lrange( ProcessInfo.reduce(@pid, hashed_key), 0, -1 )
+    end
+
+    # Map and Reduce are strings naming the Mapper and Reducer
+    # classes we want to run our map reduce with.
+    # 
+    # For instance
+    #   Mapper = "Mapper"
+    #   Reducer = "Reducer"
+    # 
+    # Default finalizer
+    #   "MapRedus::Finalizer"
+    # 
+    # Returns the new process id.
+    def self.create( *args )
+      new_pid = get_available_pid
+      
+      spec = specification(*args)
+      return nil unless spec
+
+      Process.new(new_pid, spec).save
+    end
+    
+    def self.specification(*args)
+      raise ProcessSpecificationError
+    end
+
+    def self.info(pid)
+      FileSystem.keys(ProcessInfo.pid(pid) + "*")
+    end
+    
+    def self.open(pid)
+      spec = Helper.decode( FileSystem.get(ProcessInfo.pid(pid)) )
+      spec && Process.new( pid, spec )
+    end
+
+    # Find out what map reduce processes are out there
+    # 
+    # Examples
+    #   FileSystem::ps
+    #
+    # Returns a list of the map reduce process ids
+    def self.ps
+      FileSystem.smembers(ProcessInfo.processes)
+    end
+
+    # Find out what map reduce processes are out there
+    # 
+    # Examples
+    #   FileSystem::get_available_pid
+    #
+    # Returns an avilable pid.
+    def self.get_available_pid
+      FileSystem.incrby(ProcessInfo.processes_count, 1 + rand(20)) 
+    end
+
+    # Given a result keyname, delete the result
+    #
+    # Examples
+    #   Process.delete_saved_result(key)
+    def self.delete_saved_result(keyname)
+      FileSystem.del( ProcessInfo.result_cache(keyname) )
+    end
+    
+    # Remove redis keys associated with this process if the Master isn't working.
+    #
+    # potentially is very expensive.
+    #
+    # Example
+    #   Process::kill(pid)
+    #   # => true
+    #
+    # Returns true on success.
+    def self.kill(pid)
+      num_killed = Master.emancipate(pid)
+      proc = Process.open(pid)
+      proc.delete if proc
+      num_killed
+    end
+
+    def self.kill_all
+      ps.each do |pid|
+        kill(pid)
+      end
+      FileSystem.del(ProcessInfo.processes)
+      FileSystem.del(ProcessInfo.processes_count)
+    end
+  end
+end

data/lib/mapredus/reducer.rb

@@ -0,0 +1,39 @@
+module MapRedus
+  # Reduce is a function that takes in "all" the values for a single given key
+  # and outputs a list of values or a single value that usually "reduces"
+  # the initial given value set.
+  #
+  # The output of the reduce shall always be
+  #   reduce(values) = [ reduced value, reduced value, ... ]
+  # and it will often only be a single element array
+  #
+  # The input values and the output values of the reduce will always
+  # be a string. As described in the paper, it is up to the client
+  # to define how to deal with this restriction.
+  #
+  class Reducer < QueueProcess
+    #
+    # After a recoverable fail this describes how much time we shall wait before
+    # readding the reducer back on to the queue.
+    #
+    DEFAULT_WAIT = 10 # seconds
+    def self.wait; DEFAULT_WAIT; end
+    
+    def self.reduce(values); raise InvalidReducer; end
+    
+    # Doesn't handle redundant workers and fault tolerance
+    #
+    # TODO: Resque::AutoRetry might mess this up.
+    def self.perform(pid, key)
+      process = Process.open(pid)
+      reduce(process.map_values(key)) do |reduce_val|
+        process.emit( key, reduce_val )
+      end
+    rescue MapRedus::RecoverableFail
+      Master.enslave_later_reduce(process, key)
+    ensure
+      Master.free_slave(pid)
+      process.next_state
+    end
+  end
+end

data/lib/mapredus/support.rb

@@ -0,0 +1,56 @@
+module MapRedus
+  module Support
+    class MapRedusRunnerError < StandardError; end
+    class DuplicateProcessDefinitionError < MapRedusRunnerError ; end
+
+    class Runner
+      attr_reader :process
+      def initialize(class_name)
+        @class = class_name
+      end
+
+      def method_missing(method, *args, &block)
+        mr_process = "#{@class}_#{method.to_s}"
+        if self.respond_to?(mr_process)
+          self.send(mr_process, *args, &block)
+        else
+          super(method, *args, &block)
+        end
+      end
+    end
+
+    def mapreduce
+      @mapreduce_runner ||= Runner.new(self.class.to_s.gsub(/\W/,"_"))
+    end
+
+    module ClassMethods
+      def mapreduce_process( process_name, mapredus_process_class, result_store, opts = {})
+        runner_self = Runner
+        class_name = self.to_s.gsub(/\W/,"_")
+
+        global_process_name = "#{class_name}_#{process_name.to_s}"
+
+        if runner_self.methods.include?(global_process_name)
+          raise DuplicateProcessDefintionError
+        end
+        
+        keyname = "mapredus_key_#{global_process_name}"
+        RedisSupport.redis_key( keyname, result_store )
+
+        runner_self.send( :define_method, global_process_name ) do |data, *var|
+          @process = mapredus_process_class.create
+          @process.update(:keyname => RedisSupport::Keys.send( keyname, *var ))
+          @process.run(data)
+        end
+
+        runner_self.send( :define_method, "#{global_process_name}_result" ) do |*outputter_args|
+          @process.outputter.decode(@process.keyname, *outputter_args)
+        end
+      end
+    end
+
+    def self.included(model)
+      model.extend ClassMethods
+    end
+  end
+end

data/lib/mapredus.rb

@@ -0,0 +1,106 @@
+require 'redis'
+require 'redis_support'
+require 'resque'
+require 'resque_scheduler'
+
+module MapRedus
+  include RedisSupport
+  
+  class InvalidProcess < NotImplementedError
+    def initialize; super("MapRedus QueueProcess: need to have perform method defined");end
+  end
+  
+  class ProcessSpecificationError < InvalidProcess
+    def initialize; super("MapRedus Process: need to have the specification defined");end
+  end
+
+  class InvalidMapper < NotImplementedError
+    def initialize; super("MapRedus Mapper: need to have map method defined");end
+  end
+
+  class InvalidReducer < NotImplementedError
+    def initialize; super("MapRedus Reducer: need to have reduce method defined");end
+  end
+
+  class InvalidInputStream < NotImplementedError
+    def initialize; super("MapRedus InputStream: need to have scan method defined");end
+  end
+
+  class InvalidProcess < NotImplementedError
+    def initialize; super("MapRedus Process Creation Failed: Specifications were not specified");end
+  end
+
+  class RecoverableFail < StandardError
+    def initialize; super("MapRedus Operation Failed: but it is recoverable") ;end
+  end
+  
+  # All Queue Processes should have a function called perform
+  # ensuring that when the class is put on the resque queue it can perform its work
+  # 
+  # Caution: defines redis, which is also defined in RedisSupport
+  # 
+  class QueueProcess
+    def self.queue; :mapredus; end
+    def self.perform(*args); raise InvalidProcess; end
+  end
+
+  # TODO: When you send work to a worker using a mapper you define, 
+  # the worker won't have that class name defined, unless it was started up
+  # with the class loaded
+  #
+  def register_reducer(klass); end;
+  def register_mapper(klass); end;
+
+  class Helper
+    # resque helpers defines
+    #   redis
+    #   encode
+    #   decode
+    #   classify
+    #   constantize
+    #
+    # This is extended here because we want to use the encode and decode function
+    # when we interact with resque queues
+    extend Resque::Helpers
+
+    # Defines a hash by taking the absolute value of ruby's string
+    # hash to rid the dashes since redis keys should not contain any.
+    #
+    # key - The key to be hashed.
+    #
+    # Examples
+    #
+    #   Support::hash( key )
+    #   # => '8dd8hflf8dhod8doh9hef'
+    #
+    # Returns the hash.
+    def self.hash( key )
+      key.to_s.hash.abs.to_s(16)
+    end
+
+    # Returns the classname of the namespaced class.
+    #
+    # The full name of the class.
+    #
+    # Examples
+    #
+    #   Support::class_get( Super::Long::Namespace::ClassName )
+    #   # => 'ClassName'
+    #
+    # Returns the class name.
+    def self.class_get(string)
+      constantize(string)
+    end
+  end 
+end
+
+require 'mapredus/keys'
+require 'mapredus/process'
+require 'mapredus/filesystem'
+require 'mapredus/master'
+require 'mapredus/mapper'
+require 'mapredus/reducer'
+require 'mapredus/finalizer'
+require 'mapredus/support'
+require 'mapredus/outputter'
+require 'mapredus/inputter'

data/spec/helper.rb

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'spec'
+
+dir = File.dirname(__FILE__)
+$LOAD_PATH.unshift(File.join(dir, '..', 'lib'))
+$LOAD_PATH.unshift(dir)
+require 'mapredus'
+
+#
+# make sure we can run redis
+#
+if !system("which redis-server")
+  puts '', "** can't find `redis-server` in your path"
+  abort ''
+end
+
+#
+# start our own redis when the tests start,
+# kill it when they end (redis is run as a daemon)
+#
+puts "Starting redis for testing at localhost:9736..."
+`redis-server #{dir}/redis-test.conf`
+
+at_exit do
+  #
+  # hope that no other processes have redis-test in the name...
+  # TODO: fixme
+  #
+  pid = `ps -A -o pid,command | grep [r]edis-test`.split(" ")[0]
+  puts "Killing test redis server..."
+  `rm -f #{dir}/dump.rdb`
+  Process.kill("KILL", pid.to_i)
+end
+
+#
+# Set the redis server
+#
+MapRedus.redis = 'localhost:9736:0'
+Resque.redis = MapRedus.redis
+require 'resque/failure/redis'
+Resque::Failure.backend = Resque::Failure::Redis
+
+require 'helper_classes'
+
+def work_off
+  Resque::Worker.new("*").work(0)
+end