mapredus 0.0.5 → 0.0.6

data/lib/mapredus.rb

@@ -70,11 +70,11 @@ module MapRedus
     #
     # Examples
     #
-    #   Support::hash( key )
+    #   Support::key_hash( key )
     #   # => '8dd8hflf8dhod8doh9hef'
     #
     # Returns the hash.
-    def self.hash( key )
+    def self.key_hash( key )
       key.to_s.hash.abs.to_s(16)
     end
 

data/lib/mapredus/default_classes.rb

@@ -15,6 +15,8 @@ module MapRedus
     end
   end
 
+  ################################################################################
+
   class WordCounter < Mapper
     def self.map(map_data)
       map_data.split(/\W/).each do |word|
@@ -24,12 +26,51 @@ module MapRedus
     end
   end
 
+  ####################################REDUCERS####################################
+
   class Adder < Reducer
     def self.reduce(value_list)
       yield( value_list.reduce(0) { |r, v| r += v.to_i } )
     end
   end
 
+  # Emits the identity function on the map values.
+  #
+  # The identity reducer should never actually have to reduce as a
+  # special class in mapredus, the values should just be copied from
+  # one key to a new key directly in redis.
+  class Identity < Reducer
+    def self.reduce_perform(process, key)
+      FileSystem.copy( process.map_key(key), process.reduce_key(key) )
+    end
+
+    def self.reduce(value_list)
+      value_list.each do |v|
+        yield v 
+      end
+    end
+  end
+
+  # Emits the length of the mapped value list.
+  # 
+  # The counter reducer tells how many values were emitted by the
+  # mapper.  In situations where an adder could used but only has to
+  # sum up 1's, counter will be much faster.
+  #
+  # This works in MapRedus because all the values produced for one key
+  # is processed (reduced) by a single worker.
+  class Counter < Reducer
+    def self.reduce_perform(process, key)
+      process.emit(key, FileSystem.llen(process.map_key(key)))
+    end
+
+    def self.reduce(value_list)
+      yield value_list.size
+    end
+  end
+
+  ################################################################################
+
   class ToRedisHash < Finalizer
     def self.finalize(process)
       process.each_key_reduced_value do |key, value|

data/lib/mapredus/filesystem.rb

@@ -18,7 +18,32 @@ module MapRedus
     end
 
     def self.method_missing(method, *args, &block)
-      storage.send(method, *args)
+      if storage.respond_to?(method)
+        storage.send(method, *args)
+      else
+        super
+      end
+    end
+
+    # Copy the values from one key to a second key
+    # 
+    # NOTE TODO: currently only works for the redis list data
+    # structure but will be extended for arbitrary data types.
+    #
+    # NOTE: this does not account for the key being changed during the
+    # copy, so should not be used in situations where the first_key
+    # value can change during the running of copy.
+    #
+    # Examples
+    #   FileSystem.copy("key_one", "key_two")
+    #
+    # returns true on success false otherwise
+    def self.copy(first_key, second_key)
+      list_length = storage.llen(first_key)
+      list_length.times do |index|
+        storage.rpush(second_key, storage.lindex(first_key, index))
+      end
+      true
     end
     
     # Setup locks on results using RedisSupport lock functionality

data/lib/mapredus/process.rb

@@ -186,7 +186,7 @@ module MapRedus
       if( not @ordered )
         key, value = key_value
         FileSystem.sadd( ProcessInfo.keys(@pid), key )
-        hashed_key = Helper.hash(key)
+        hashed_key = Helper.key_hash(key)
         FileSystem.rpush( ProcessInfo.map(@pid, hashed_key), value )
       else
         # if there's an order for the process then we should use a zset above
@@ -194,7 +194,7 @@ module MapRedus
         #
         rank, key, value = key_value
         FileSystem.zadd( ProcessInfo.keys(@pid), rank, key )
-        hashed_key = Helper.hash(key)
+        hashed_key = Helper.key_hash(key)
         FileSystem.rpush( ProcessInfo.map(@pid, hashed_key), value )
       end
       raise "Key Collision: key:#{key}, #{key.class} => hashed key:#{hashed_key}" if key_collision?(hashed_key, key)
@@ -212,7 +212,7 @@ module MapRedus
     #
     # Returns "OK" on success.
     def emit(key, reduce_val)
-      hashed_key = Helper.hash(key)
+      hashed_key = Helper.key_hash(key)
       FileSystem.rpush( ProcessInfo.reduce(@pid, hashed_key), reduce_val )
     end
 
@@ -221,6 +221,21 @@ module MapRedus
             FileSystem.get( ProcessInfo.hash_to_key(@pid, hashed_key) ) == key.to_s )
     end
 
+    # Convenience methods to get the mapredus internal key string for a key
+    #
+    # Examples
+    #   reduce_key("document")
+    #   # => mapredus:process:PID:map_key:<Helper.key_hash("document")>:reduce
+    #   map_key("document")
+    #   # => mapredus:process:PID:map_key:<Helper.key_hash("document")>
+    #
+    # Returns the internal mapreduce string key for a given key.
+    [:reduce, :map].each do |internal_key|
+      define_method("#{internal_key}_key") do |key|
+        ProcessInfo.send(internal_key, @pid, Helper.key_hash(key))
+      end
+    end
+
     # Keys that the map operation produced
     #
     # Examples
@@ -252,12 +267,12 @@ module MapRedus
     #
     # Returns the values.
     def map_values(key)
-      hashed_key = Helper.hash(key)
+      hashed_key = Helper.key_hash(key)
       FileSystem.lrange( ProcessInfo.map(@pid, hashed_key), 0, -1 )
     end
 
     def num_values(key)
-      hashed_key = Helper.hash(key)
+      hashed_key = Helper.key_hash(key)
       FileSystem.llen( ProcessInfo.map(@pid, hashed_key) )
     end
 
@@ -269,7 +284,7 @@ module MapRedus
     #
     # Returns the values.
     def reduce_values(key)
-      hashed_key = Helper.hash(key)
+      hashed_key = Helper.key_hash(key)
       FileSystem.lrange( ProcessInfo.reduce(@pid, hashed_key), 0, -1 )
     end
 

data/lib/mapredus/reducer.rb

@@ -21,14 +21,24 @@ module MapRedus
     
     def self.reduce(values); raise InvalidReducer; end
     
+    #
+    # The overridable portion of a reducer perform.  In some default
+    # classes like Identity and Counter we do not call self.reduce but
+    # provide optimization for the reduction by overriding this
+    # method.
+    #
+    def self.reduce_perform(process, key)
+      reduce(process.map_values(key)) do |reduce_val|
+        process.emit( key, reduce_val )
+      end
+    end
+    
     # Doesn't handle redundant workers and fault tolerance
     #
     # TODO: Resque::AutoRetry might mess this up.
     def self.perform(pid, key)
       process = Process.open(pid)
-      reduce(process.map_values(key)) do |reduce_val|
-        process.emit( key, reduce_val )
-      end
+      reduce_perform(process, key)
     rescue MapRedus::RecoverableFail
       Master.enslave_later_reduce(process, key)
     ensure

data/spec/mapredus_spec.rb

@@ -402,3 +402,27 @@ describe "MapRedus Support" do
     end
   end
 end
+
+describe "MapRedus Default Classes" do
+  before(:each) do
+    MapRedus::FileSystem.flushall
+    @process = GetWordCount.create
+  end
+
+  it "testing that the identity copy actually does a copy" do
+    MapRedus::FileSystem.rpush(@process.map_key("test_key"), "whatever")
+    MapRedus::FileSystem.rpush(@process.map_key("test_key"), "yeah")
+    MapRedus::Identity.perform(@process.pid, "test_key")
+
+    @process.map_values("test_key").should == ["whatever", "yeah"]
+    @process.reduce_values("test_key").should == ["whatever", "yeah"]
+  end
+
+  it "should properly do a count" do
+    MapRedus::FileSystem.rpush(@process.map_key("test_key"), "whatever")
+    MapRedus::FileSystem.rpush(@process.map_key("test_key"), "yeah")
+    MapRedus::Counter.perform(@process.pid, "test_key")
+
+    @process.reduce_values("test_key").should == ["2"]
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mapredus
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: false
   segments: 
   - 0
   - 0
-  - 5
-  version: 0.0.5
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - John Le
@@ -16,11 +16,13 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-02 00:00:00 -07:00
+date: 2010-08-05 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: redis
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -31,12 +33,12 @@ dependencies:
         - 0
         - 4
         version: 1.0.4
+  requirement: *id001
   type: :runtime
-  name: redis
-  prerelease: false
-  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: resque
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -46,12 +48,12 @@ dependencies:
         - 1
         - 8
         version: "1.8"
+  requirement: *id002
   type: :runtime
-  name: resque
-  prerelease: false
-  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: resque-scheduler
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -60,12 +62,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  requirement: *id003
   type: :runtime
-  name: resque-scheduler
-  prerelease: false
-  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: redis_support
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -74,10 +76,8 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  requirement: *id004
   type: :runtime
-  name: redis_support
-  prerelease: false
-  version_requirements: *id004
 description: simple mapreduce framework using redis and resque
 email: john@doloreslabs.com
 executables: []
@@ -102,9 +102,9 @@ files:
 - lib/mapredus/support.rb
 - LICENSE
 - README.md
-- spec/helper.rb
 - spec/helper_classes.rb
 - spec/mapredus_spec.rb
+- spec/helper.rb
 has_rdoc: true
 homepage: http://github.com/dolores/mapredus
 licenses: []
@@ -140,6 +140,6 @@ signing_key:
 specification_version: 3
 summary: mapredus initial
 test_files: 
-- spec/helper.rb
 - spec/helper_classes.rb
 - spec/mapredus_spec.rb
+- spec/helper.rb