mapredus 0.0.2 → 0.0.3

data/README.md

@@ -114,10 +114,13 @@ an example:
       end
     end
 
-The process.keyname refers the final result key that is stored in
-redis.  The outputter is needed to define how exactly that encoding is
-defined.  We provided an outputter that encodes your data into a redis
-hash.
+The process.result_key refers the final result key that is stored in
+redis.  The result_key may take arguments which define the output of
+the key.  The process will also incorporate initially given key
+arguments into the result_key.  result_key's are defined exactly as a
+redis_key in the redis_support gem. The outputter is needed to define
+how exactly that encoding is defined.  We provided an outputter that
+encodes your data into a redis hash.
 
     class RedisHasher < MapRedus::Outputter
       def encode(result_key, k, v)
@@ -166,6 +169,7 @@ Requirements
 TODO
 ----
 not necessarily in the given order
+* Ensure that the type that is inputted is the type that is outputted
 
 * if a process fails we do what we are supposed to do i.e. add a
   failure_hook which does something if your process fails

data/lib/mapredus/default_classes.rb

@@ -39,6 +39,10 @@ module MapRedus
   end
 
   class RedisHasher < Outputter
+    def self.values(result_key)
+      FileSystem.hvals(result_key)
+    end
+
     def self.keys(result_key)
       FileSystem.hkeys(result_key)
     end

data/lib/mapredus/keys.rb

@@ -3,7 +3,7 @@ module MapRedus
   ProcessInfo = RedisKey
   
   #### USED WITHIN process.rb ####
-  
+ 
   # Holds the current map reduce processes that are either running or which still have data lying around
   #
   redis_key :processes, "mapredus:processes"
@@ -36,6 +36,10 @@ module MapRedus
   #
   redis_key :temp, "mapredus:process:PID:temp_reduce_key:HASHED_KEY:UNIQUE_REDUCE_HOSTNAME:UNIQUE_REDUCE_PROCESS_ID"
 
+  # The default location for the result
+  #
+  DEFAULT_RESULT_KEY = "mapredus:process:PID:result"
+
   #### USED WITHIN master.rb ####
   
   # Keeps track of the current slaves (by appending "1" to a redis list)

data/lib/mapredus/master.rb

@@ -3,7 +3,7 @@ module MapRedus
   # a master interface with Resque
   #
   # Does bookkeeping to keep track of how many slaves are doing work. If we have
-  # no slaves doing work for a process then the process is done. While there is work available
+  # no slaves doing work for a process then the process is donex. While there is work available
   # the slaves will always be doing work.
   #
   class Master < QueueProcess
@@ -49,9 +49,23 @@ module MapRedus
       enslave( process, process.inputter, process.pid, data_object )
     end
     
+    # Enslave the reducers:
+    # 
+    # For each key, enslave a reducer to process the values on that
+    # key. If there were no keys produced during the map operation we
+    # must set off the finalizer.
+    #
+    # TODO: inject optimizations here for special reducers like the
+    # identity reduce
+    #
+    # returns nothing
     def self.enslave_reducers( process )
-      process.map_keys.each do |key|
-        enslave_reduce( process, key )
+      if( process.num_keys > 0 )
+        process.map_keys.each do |key|
+          enslave_reduce( process, key )
+        end
+      else
+        process.next_state
       end
     end
 
@@ -79,7 +93,7 @@ module MapRedus
     #
     def self.enslave( process, klass, *args )
       FileSystem.rpush(ProcessInfo.slaves(process.pid), 1)
-      
+
       if( process.synchronous )
         klass.perform(*args)
       else

data/lib/mapredus/process.rb

@@ -236,6 +236,14 @@ module MapRedus
       end
     end
 
+    def num_keys()
+      if( not @ordered )
+        FileSystem.scard( ProcessInfo.keys(@pid) )
+      else
+        FileSystem.zcard( ProcessInfo.keys(@pid) )
+      end
+    end
+
     # values that the map operation produced, for a key
     #
     # Examples
@@ -265,12 +273,23 @@ module MapRedus
       FileSystem.lrange( ProcessInfo.reduce(@pid, hashed_key), 0, -1 )
     end
 
+    # functions to manage the location of the result in the FileSystem
+    #
+    # Examples
+    #   process.result_key(extra, arguments)
+    #   Process.result_key(all, needed, arguments)
+    #   # => "something:that:uses:the:extra:arguments"
+    # 
+    #   SomeProcessSubclass.set_result_key("something:ARG:something:VAR")
+    #   # sets the result key for (CAPITAL require arguments to fill in the values)
     def result_key(*args)
       Helper.class_get(@type).result_key(*[@key_args, args].flatten)
     end
 
     def self.result_key(*args)
-      ProcessInfo.send( "#{self.to_s.gsub(/\W/,"_")}_result_cache", *args )
+      key_maker = "#{self.to_s.gsub(/\W/,"_")}_result_cache"
+      key_maker = ProcessInfo.respond_to?(key_maker) ? key_maker : "#{MapRedus::Process.to_s.gsub(/\W/,"_")}_result_cache"
+      ProcessInfo.send( key_maker, *args )
     end
 
     def self.set_result_key(key_struct)
@@ -303,11 +322,11 @@ module MapRedus
     #
     # Example
     #   class AnswerDistribution < MapRedus::Process
-    #     inputter = JudgmentStream
-    #     mapper = ResponseFrequencyMap
-    #     reducer = Adder
-    #     finalizer = AnswerCount
-    #     outputter = MapRedus::RedisHasher
+    #     inputter JudgmentStream
+    #     mapper ResponseFrequencyMap
+    #     reducer Adder
+    #     finalizer AnswerCount
+    #     outputter MapRedus::RedisHasher
     #   end
     class << self; attr_reader *ATTRS; end
 
@@ -343,6 +362,7 @@ module MapRedus
     finalizer ToRedisHash
     outputter RedisHasher
     type Process
+    set_result_key DEFAULT_RESULT_KEY
     
     # This function returns all the redis keys produced associated
     # with a process's process id.

data/spec/helper.rb

@@ -35,6 +35,7 @@ end
 #
 # Set the redis server
 #
+RedisSupport.redis = 'localhost:9736:0'
 MapRedus.redis = 'localhost:9736:0'
 Resque.redis = MapRedus.redis
 require 'resque/failure/redis'

data/spec/helper_classes.rb

@@ -18,18 +18,49 @@ class CharCounter < MapRedus::Mapper
   end
 end
 
+class ExtraResultKeyHash < MapRedus::Finalizer
+  def self.finalize(process)
+    process.each_key_reduced_value do |key, value|
+      process.outputter.encode(process.result_key("extra"), key, value)
+    end
+  end
+end
+
 class GetCharCount < MapRedus::Process
   EXPECTED_ANSWER = {"k"=>2, "v"=>1, " "=>54, ","=>3, "w"=>7, "a"=>17, "l"=>12, "b"=>2, "m"=>4, "c"=>3, "."=>2, "y"=>3, "n"=>18, "D"=>1, "d"=>15, "o"=>13, "p"=>14, "e"=>34, "f"=>6, "r"=>13, "g"=>6, "S"=>1, "s"=>12, "h"=>19, "H"=>1, "t"=>20, "i"=>16, "u"=>5, "j"=>1}
   inputter CharStream
   mapper CharCounter
 end
 
+class CharCountTest < MapRedus::Process
+  inputter CharStream
+  mapper CharCounter
+end
+
 class GetWordCount < MapRedus::Process
   TEST = "He pointed his finger in friendly jest and went over to the parapet laughing to himself. Stephen Dedalus stepped up, followed him wearily halfway and sat down on the edge of the gunrest, watching him still as he propped his mirror on the parapet, dipped the brush in the bowl and lathered cheeks and neck."
   EXPECTED_ANSWER = {"gunrest"=>1, "over"=>1, "still"=>1, "of"=>1, "him"=>2, "and"=>4, "bowl"=>1, "himself"=>1, "went"=>1, "friendly"=>1, "finger"=>1, "propped"=>1, "cheeks"=>1, "dipped"=>1, "down"=>1, "wearily"=>1, "up"=>1, "stepped"=>1, "dedalus"=>1, "to"=>2, "in"=>2, "sat"=>1, "the"=>6, "pointed"=>1, "as"=>1, "followed"=>1, "stephen"=>1, "laughing"=>1, "his"=>2, "he"=>2, "brush"=>1, "jest"=>1, "neck"=>1, "mirror"=>1, "edge"=>1, "on"=>2, "parapet"=>2, "lathered"=>1, "watching"=>1, "halfway"=>1}
   set_result_key "test:result"
 end
 
+class TestHash < MapRedus::Finalizer
+  def self.finalize(process)
+    process.each_key_reduced_value do |key, value|
+      process.outputter.encode(process.result_key("extra_arg"), key, value)
+    end
+  end
+end
+
+class TestResultKeyArguments < MapRedus::Process
+  #
+  # EXTRA_KEY_ARG is not known at the time the process is run
+  # but it is known by the time the finalizer is running
+  #
+  finalizer TestHash
+  set_result_key "test:KEY_ARG:test:EXTRA_KEY_ARG"
+  key_args ["key_argument"]
+end
+
 class Document
   include MapRedus::Support
   mapreduce_process :char_count, GetCharCount, "document:count:ID"

data/spec/mapredus_spec.rb

@@ -78,6 +78,59 @@ describe "MapRedus" do
       @process.outputter.decode(@process.result_key, key).to_i.should == GetWordCount::EXPECTED_ANSWER[key]
     end
   end
+
+  it "runs the default process" do
+    process = MapRedus::Process.create
+    process.update(:key_args => [process.pid])
+    process.result_key.should == "mapredus:process:#{process.pid}:result"
+    process.run("wordstream:test")
+    work_off
+    
+    process.map_keys.size.should == GetWordCount::EXPECTED_ANSWER.size
+    process.map_keys.each do |key|
+      reduce_values = process.reduce_values(key)
+      reduce_values.size.should == 1
+    end
+
+    process.each_key_reduced_value do |key, value|
+      process.outputter.decode(process.result_key, key).to_i.should == GetWordCount::EXPECTED_ANSWER[key]
+    end
+  end
+
+  it "runs a process without result_key being set (using the default key location)" do
+    process = CharCountTest.create
+    process.update(:key_args => [process.pid])
+    process.result_key.should == "mapredus:process:#{process.pid}:result"
+    process.run("wordstream:test")
+    work_off
+    
+    process.map_keys.size.should == GetCharCount::EXPECTED_ANSWER.size
+    process.map_keys.each do |key|
+      reduce_values = process.reduce_values(key)
+      reduce_values.size.should == 1
+    end
+
+    process.each_key_reduced_value do |key, value|
+      process.outputter.decode(process.result_key, key).to_i.should == GetCharCount::EXPECTED_ANSWER[key]
+    end
+  end
+
+  it "runs a process where key arguments exist and extra arguments are used" do 
+    process = TestResultKeyArguments.create
+    process.result_key("extra_arg").should == "test:key_argument:test:extra_arg"
+    process.run("wordstream:test")
+    work_off
+    
+    process.map_keys.size.should == GetWordCount::EXPECTED_ANSWER.size
+    process.map_keys.each do |key|
+      reduce_values = process.reduce_values(key)
+      reduce_values.size.should == 1
+    end
+
+    process.each_key_reduced_value do |key, value|
+      process.outputter.decode(process.result_key("extra_arg"), key).to_i.should == GetWordCount::EXPECTED_ANSWER[key]
+    end
+  end
 end
 
 describe "MapRedus Process" do
@@ -154,19 +207,40 @@ describe "MapRedus Process" do
     end
 
     MapRedus::Process.kill_all
-    Resque.peek(:mapredus, 0, -1) == []
+    Resque.peek(:mapredus, 0, 100) == []
   end
 
-  it "responses to next state correctly" do
+  it "responds to next state correctly" do
     @process.state.should == MapRedus::NOT_STARTED
     @process.next_state
     @process.state.should == MapRedus::INPUT_MAP_IN_PROGRESS
     work_off
 
+    ##
+    ## Since there are no map keys produced in this the next state
+    ## should go directly to the finalizer
+    ##
+    
     @process.next_state
-    @process.state.should == MapRedus::REDUCE_IN_PROGRESS
+    @process.state.should == MapRedus::FINALIZER_IN_PROGRESS
     work_off
+    
+    @process.next_state
+    @process.state.should == MapRedus::COMPLETE
+  end
 
+  it "responds to next state correcty when keys are produced" do
+    @process.state.should == MapRedus::NOT_STARTED
+    @process.next_state
+    @process.state.should == MapRedus::INPUT_MAP_IN_PROGRESS
+    work_off
+
+    @process.emit_intermediate("hell", "yeah")
+
+    @process.next_state
+    @process.state.should == MapRedus::REDUCE_IN_PROGRESS
+    work_off
+    
     @process.next_state
     @process.state.should == MapRedus::FINALIZER_IN_PROGRESS
     work_off
@@ -237,7 +311,7 @@ describe "MapRedus Master" do
 
   it "handles slaves (enslaving) correctly" do
     MapRedus::Master.enslave(@process, MapRedus::WordCounter, @process.pid, "test")
-    Resque.peek(:mapredus, 0, -1).should == [{"args"=>[@process.pid, "test"], "class"=>"MapRedus::WordCounter"}]
+    Resque.peek(:mapredus, 0, 1).should == {"args"=>[@process.pid, "test"], "class"=>"MapRedus::WordCounter"}
     MapRedus::Master.slaves(@process.pid).should == ["1"]
   end
 
@@ -266,12 +340,14 @@ describe "MapRedus Mapper/Reducer/Finalizer" do
     @process.update(:state => MapRedus::INPUT_MAP_IN_PROGRESS)
     @process.state.should == MapRedus::INPUT_MAP_IN_PROGRESS
     @process.inputter.perform(@process.pid, "wordstream:test")
-    Resque.peek(:mapredus, 0, -1).should == [{"args"=>[@process.pid, 0], "class"=>"MapRedus::WordCounter"}]
+    Resque.peek(:mapredus, 0, 1).should == {"args"=>[@process.pid, 0], "class"=>"MapRedus::WordCounter"}
     Resque.pop(:mapredus)
     @process.mapper.perform(@process.pid, 0)
     @process.reload
     @process.state.should == MapRedus::REDUCE_IN_PROGRESS
-    Resque.peek(:mapredus, 0, -1).should == [{"args"=>[@process.pid, "data"], "class"=>"MapRedus::Adder"}]
+    Resque.peek(:mapredus, 0, 1).should == {"args"=>[@process.pid, "data"], "class"=>"MapRedus::Adder"}
+
+    MapRedus::Process.open(@process.pid).state.should == MapRedus::REDUCE_IN_PROGRESS
   end
 
   it "runs a reduce correctly proceeding to the correct next state" do
@@ -281,7 +357,9 @@ describe "MapRedus Mapper/Reducer/Finalizer" do
     @process.reducer.perform(@process.pid, "data")
     @process.reload
     @process.state.should == MapRedus::FINALIZER_IN_PROGRESS
-    Resque.peek(:mapredus, 0, -1).should == [{"args"=>[@process.pid], "class"=>"MapRedus::ToRedisHash"}]
+    Resque.peek(:mapredus, 0, 1).should == {"args"=>[@process.pid], "class"=>"MapRedus::ToRedisHash"}
+
+    MapRedus::Process.open(@process.pid).state.should == MapRedus::FINALIZER_IN_PROGRESS
   end
 
   it "should test that the finalizer correctly saves" do
@@ -292,8 +370,10 @@ describe "MapRedus Mapper/Reducer/Finalizer" do
     @process.finalizer.perform(@process.pid)
     @process.reload
     @process.state.should == MapRedus::COMPLETE
-    Resque.peek(:mapredus, 0, -1).should == []
+    Resque.peek(:mapredus, 0, 100).should == []
     @process.outputter.decode("test:result", "data").should == "1"
+
+    MapRedus::Process.open(@process.pid).state.should == MapRedus::COMPLETE
   end
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mapredus
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 25
   prerelease: false
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - John Le
@@ -16,11 +16,13 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-09 00:00:00 -07:00
+date: 2010-07-15 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: redis
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -31,12 +33,12 @@ dependencies:
         - 0
         - 4
         version: 1.0.4
+  requirement: *id001
   type: :runtime
-  name: redis
-  prerelease: false
-  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: resque
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -46,12 +48,12 @@ dependencies:
         - 1
         - 8
         version: "1.8"
+  requirement: *id002
   type: :runtime
-  name: resque
-  prerelease: false
-  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: resque-scheduler
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -60,12 +62,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  requirement: *id003
   type: :runtime
-  name: resque-scheduler
-  prerelease: false
-  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: redis_support
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -74,10 +76,8 @@ dependencies:
         segments: 
         - 0
         version: "0"
+  requirement: *id004
   type: :runtime
-  name: redis_support
-  prerelease: false
-  version_requirements: *id004
 description: simple mapreduce framework using redis and resque
 email: john@doloreslabs.com
 executables: []
@@ -102,9 +102,9 @@ files:
 - lib/mapredus/support.rb
 - LICENSE
 - README.md
-- spec/helper.rb
 - spec/helper_classes.rb
 - spec/mapredus_spec.rb
+- spec/helper.rb
 has_rdoc: true
 homepage: http://github.com/dolores/mapredus
 licenses: []
@@ -140,6 +140,6 @@ signing_key:
 specification_version: 3
 summary: mapredus initial
 test_files: 
-- spec/helper.rb
 - spec/helper_classes.rb
 - spec/mapredus_spec.rb
+- spec/helper.rb