mapredus 0.0.1

data/spec/helper_classes.rb

@@ -0,0 +1,102 @@
+class GetCharCount < MapRedus::Process
+  EXPECTED_ANSWER = {"k"=>2, "v"=>1, " "=>54, ","=>3, "w"=>7, "a"=>17, "l"=>12, "b"=>2, "m"=>4, "c"=>3, "."=>2, "y"=>3, "n"=>18, "D"=>1, "d"=>15, "o"=>13, "p"=>14, "e"=>34, "f"=>6, "r"=>13, "g"=>6, "S"=>1, "s"=>12, "h"=>19, "H"=>1, "t"=>20, "i"=>16, "u"=>5, "j"=>1}
+  def self.specification
+    {
+      :inputter => CharStream,
+      :mapper => CharCounter,
+      :reducer => Adder,
+      :finalizer => ToRedisHash,
+      :outputter => MapRedus::RedisHasher,
+      :ordered => false
+    }
+  end
+end
+
+class GetWordCount < MapRedus::Process
+  TEST = "He pointed his finger in friendly jest and went over to the parapet laughing to himself. Stephen Dedalus stepped up, followed him wearily halfway and sat down on the edge of the gunrest, watching him still as he propped his mirror on the parapet, dipped the brush in the bowl and lathered cheeks and neck."
+  EXPECTED_ANSWER = {"gunrest"=>1, "over"=>1, "still"=>1, "of"=>1, "him"=>2, "and"=>4, "bowl"=>1, "himself"=>1, "went"=>1, "friendly"=>1, "finger"=>1, "propped"=>1, "cheeks"=>1, "dipped"=>1, "down"=>1, "wearily"=>1, "up"=>1, "stepped"=>1, "dedalus"=>1, "to"=>2, "in"=>2, "sat"=>1, "the"=>6, "pointed"=>1, "as"=>1, "followed"=>1, "stephen"=>1, "laughing"=>1, "his"=>2, "he"=>2, "brush"=>1, "jest"=>1, "neck"=>1, "mirror"=>1, "edge"=>1, "on"=>2, "parapet"=>2, "lathered"=>1, "watching"=>1, "halfway"=>1}
+  def self.specification
+    {
+      :inputter => WordStream,
+      :mapper => WordCounter,
+      :reducer => Adder,
+      :finalizer => ToRedisHash,
+      :outputter => MapRedus::RedisHasher,
+      :ordered => false,
+      :keyname => "test:result"
+    }
+  end
+end
+
+class WordStream < MapRedus::InputStream
+  def self.scan(data_object)
+    #
+    # The data_object should be a reference to an object that is
+    # stored on your system.  The scanner is used to break up what you
+    # need from the object into manageable pieces for the mapper.  In
+    # this example, the data object is a reference to a redis string.
+    #
+    test_string = MapRedus::FileSystem.get(data_object)
+    
+    test_string.split.each_slice(10).each_with_index do |word_set, i|
+      yield(i, word_set.join(" "))
+    end
+  end
+end
+
+class CharStream < MapRedus::InputStream
+  def self.scan(data_object)
+    test_string = MapRedus::FileSystem.get(data_object)
+    
+    0.step(test_string.size, 30) do |index|
+      char_set = test_string[index...(index+30)]
+      next if char_set.empty?
+      yield(index, char_set)
+    end
+  end
+end
+
+class WordCounter < MapRedus::Mapper
+  def self.map(map_data)
+    map_data.split(/\W/).each do |word|
+      next if word.empty?
+      yield(word.downcase, 1)
+    end
+  end
+end
+
+class CharCounter < MapRedus::Mapper
+  def self.map(map_data)
+    map_data.each_char do |char|
+      yield(char, 1)
+    end
+  end
+end
+
+class Adder < MapRedus::Reducer
+  def self.reduce(value_list)
+    yield( value_list.reduce(0) { |r, v| r += v.to_i } )
+  end
+end
+
+class ToRedisHash < MapRedus::Finalizer
+  def self.finalize(process)
+    process.each_key_reduced_value do |key, value|
+      process.outputter.encode(process.keyname, key, value)
+    end
+  end
+end
+
+class Document
+  include MapRedus::Support
+  mapreduce_process :char_count, GetCharCount, "document:count:ID"
+
+  attr_accessor :id
+  def initialize(id)
+    @id = id
+  end
+
+  def calculate_chars(data_reference)
+    mapreduce.char_count(data_reference, id)
+  end
+end

data/spec/mapredus_spec.rb

@@ -0,0 +1,295 @@
+require File.dirname(__FILE__) + '/helper'
+
+describe "MapRedus" do
+  # this is called before each test case
+  before(:each) do
+    MapRedus::FileSystem.flushall
+    @process = GetWordCount.create
+    MapRedus::FileSystem.setnx("wordstream:test", GetWordCount::TEST)
+  end
+
+  it "creates a process successfully" do
+    process = GetWordCount.open(@process.pid)
+    
+    process.inputter.should == WordStream
+    process.mapper.should == WordCounter
+    process.reducer.should == Adder
+    process.finalizer.should == ToRedisHash
+    process.outputter.should == MapRedus::RedisHasher
+  end
+
+  it "runs a map reduce process synchronously" do
+    ##
+    ## In general map reduce shouldn't be running operations synchronously
+    ##
+    @process.run("wordstream:test", synchronously = true)
+    @process.map_keys.size.should == GetWordCount::EXPECTED_ANSWER.size
+
+    @process.map_keys.each do |key|
+      reduce_values = @process.reduce_values(key)
+      reduce_values.size.should == 1
+    end
+
+    @process.each_key_reduced_value do |key, value|
+      @process.outputter.decode(@process.keyname, key).to_i.should == GetWordCount::EXPECTED_ANSWER[key]
+    end
+  end
+
+  it "runs a map reduce process asynchronously" do
+    @process.run("wordstream:test", synchronously = false)
+    work_off
+
+    @process.map_keys.size.should == GetWordCount::EXPECTED_ANSWER.size
+    @process.map_keys.each do |key|
+      reduce_values = @process.reduce_values(key)
+      reduce_values.size.should == 1
+    end
+
+    @process.each_key_reduced_value do |key, value|
+      @process.outputter.decode(@process.keyname, key).to_i.should == GetWordCount::EXPECTED_ANSWER[key]
+    end
+  end
+end
+
+describe "MapRedus Process" do
+  before(:each) do
+    MapRedus::FileSystem.flushall
+    @process = GetWordCount.create
+  end
+
+  it "saves a process" do
+    @process.mapper = CharCounter
+    @process.synchronous = true
+    @process.save
+
+    @process = MapRedus::Process.open(@process.pid)
+    
+    @process.mapper.should == CharCounter
+    @process.synchronous.should == true
+  end
+  
+  it "updates a process" do
+    @process.update(:mapper => CharCounter, :ordered => true)
+    @process = MapRedus::Process.open(@process.pid)
+    
+    @process.mapper.should == CharCounter
+    @process.ordered.should == true
+  end
+  
+  it "deletes a process" do
+    @process.delete
+    
+    proc = MapRedus::Process.open(@process.pid)
+    proc.should == nil
+  end
+  
+  it "kills a process" do
+    @process.run(GetWordCount::TEST)
+    MapRedus::Process.kill(@process.pid)
+    Resque.size(:mapredus).should == 0
+  end
+
+  it "kills a process that is started" do
+    @process.run(GetWordCount::TEST)
+
+    worker = Resque::Worker.new("*")
+    worker.perform(worker.reserve)   # do some work
+    
+    MapRedus::Process.kill(@process.pid)
+    Resque.size(:mapredus).should == 0
+  end
+  
+  it "kills all process" do
+    proc_1 = GetWordCount.create
+    proc_2 = GetWordCount.create
+    proc_3 = GetWordCount.create
+    proc_4 = GetWordCount.create
+    proc_5 = GetWordCount.create
+    proc_6 = GetWordCount.create
+
+    proc_1.run(GetWordCount::TEST)
+    proc_2.run(GetWordCount::TEST)
+    proc_3.run(GetWordCount::TEST)
+
+    worker = Resque::Worker.new("*")
+    6.times do 
+      worker.perform(worker.reserve)
+    end
+    
+    proc_4.run(GetWordCount::TEST)
+    proc_5.run(GetWordCount::TEST)
+    proc_6.run(GetWordCount::TEST)
+    
+    6.times do
+      worker.perform(worker.reserve)
+    end
+
+    MapRedus::Process.kill_all
+    Resque.peek(:mapredus, 0, -1) == []
+  end
+
+  it "responses to next state correctly" do
+    @process.state.should == MapRedus::NOT_STARTED
+    @process.next_state
+    @process.state.should == MapRedus::INPUT_MAP_IN_PROGRESS
+    work_off
+
+    @process.next_state
+    @process.state.should == MapRedus::REDUCE_IN_PROGRESS
+    work_off
+
+    @process.next_state
+    @process.state.should == MapRedus::FINALIZER_IN_PROGRESS
+    work_off
+    
+    @process.next_state
+    @process.state.should == MapRedus::COMPLETE
+  end
+
+  it "emit_intermediate unordered successfully" do
+    @process.emit_intermediate("hell", "yeah")
+    result = []
+    @process.each_key_nonreduced_value do |key, value|
+      result << [key, value]
+    end
+
+    result.should == [["hell", "yeah"]]
+  end
+
+  it "emit_intermediate on an ordered process" do
+    @process.update(:ordered => true)
+    @process.emit_intermediate(1, "number", "one")
+    @process.emit_intermediate(2, "place", "two")
+    res = []
+    @process.each_key_nonreduced_value do |key, value|
+      res << [key, value]
+    end
+    
+    res.should == [["number", "one"], ["place", "two"]]
+  end
+
+  it "emit successfully" do
+    @process.emit_intermediate("something", "non_reduced_value")
+    @process.emit("something", "reduced")
+    result = []
+    @process.each_key_reduced_value do |key, rv|
+      result << [key, rv]
+    end
+    result.should == [["something", "reduced"]]
+  end
+
+  it "produces the correct map keys" do
+    @process.emit_intermediate("map key 1", "value")
+    @process.emit_intermediate("map key 1", "value")
+    @process.emit_intermediate("map key 2", "value")
+
+    @process.map_keys.sort.should == ["map key 1", "map key 2"]
+  end
+
+  it "produces the correct map/reduce values" do
+    MapRedus::FileSystem.setnx("wordstream:test", GetWordCount::TEST)
+    @process.run("wordstream:test")
+    work_off
+    @process.map_keys.sort.should == GetWordCount::EXPECTED_ANSWER.keys.sort
+    
+    @process.each_key_reduced_value do |key, reduced_value|
+      reduced_value.to_i.should == GetWordCount::EXPECTED_ANSWER[key]
+      @process.map_values(key).should == ["1"] * reduced_value.to_i
+    end
+  end
+end
+
+describe "MapRedus Master" do
+  before(:each) do
+    MapRedus::FileSystem.flushall
+    MapRedus::FileSystem.setnx("test", "some data")
+    @process = GetWordCount.create
+  end
+
+  it "handles slaves (enslaving) correctly" do
+    MapRedus::Master.enslave(@process, WordCounter, @process.pid, "test")
+    Resque.peek(:mapredus, 0, -1).should == [{"args"=>[@process.pid, "test"], "class"=>"WordCounter"}]
+    MapRedus::Master.slaves(@process.pid).should == ["1"]
+  end
+
+  it "handles slaves (freeing) correctly" do
+    MapRedus::Master.enslave(@process, WordCounter, @process.pid, "test")
+    MapRedus::Master.enslave(@process, WordCounter, @process.pid, "test")
+
+    MapRedus::Master.slaves(@process.pid).should == ["1", "1"]
+
+    MapRedus::Master.free_slave(@process.pid)
+    MapRedus::Master.free_slave(@process.pid)
+    MapRedus::Master.slaves(@process.pid).should == []
+  end
+
+  it "handles redundant multiple workers (same output regardless of how many workers complete)"
+end
+
+describe "MapRedus Mapper/Reducer/Finalizer" do
+  before(:each) do
+    MapRedus::FileSystem.flushall
+    MapRedus::FileSystem.setnx("wordstream:test", "data")
+    @process = GetWordCount.create
+  end
+
+  it "runs a map correctly proceeding to the next state" do
+    @process.update(:state => MapRedus::INPUT_MAP_IN_PROGRESS)
+    @process.state.should == MapRedus::INPUT_MAP_IN_PROGRESS
+    @process.inputter.perform(@process.pid, "wordstream:test")
+    Resque.peek(:mapredus, 0, -1).should == [{"args"=>[@process.pid, 0], "class"=>"WordCounter"}]
+    Resque.pop(:mapredus)
+    @process.mapper.perform(@process.pid, 0)
+    @process.reload
+    @process.state.should == MapRedus::REDUCE_IN_PROGRESS
+    Resque.peek(:mapredus, 0, -1).should == [{"args"=>[@process.pid, "data"], "class"=>"Adder"}]
+  end
+
+  it "runs a reduce correctly proceeding to the correct next state" do
+    @process.update(:state => MapRedus::REDUCE_IN_PROGRESS)
+    @process.state.should == MapRedus::REDUCE_IN_PROGRESS
+    @process.emit_intermediate("data", "1")
+    @process.reducer.perform(@process.pid, "data")
+    @process.reload
+    @process.state.should == MapRedus::FINALIZER_IN_PROGRESS
+    Resque.peek(:mapredus, 0, -1).should == [{"args"=>[@process.pid], "class"=>"ToRedisHash"}]
+  end
+
+  it "should test that the finalizer correctly saves" do
+    @process.update(:state => MapRedus::FINALIZER_IN_PROGRESS)
+    @process.state.should == MapRedus::FINALIZER_IN_PROGRESS
+    @process.emit_intermediate("data", "1")
+    @process.emit("data", "1")
+    @process.finalizer.perform(@process.pid)
+    @process.reload
+    @process.state.should == MapRedus::COMPLETE
+    Resque.peek(:mapredus, 0, -1).should == []
+    @process.outputter.decode("test:result", "data").should == "1"
+  end
+end
+
+describe "MapReduce Support" do
+  before(:each) do
+    MapRedus::FileSystem.flushall
+    @doc = Document.new(10)
+    @other_doc = Document.new(15)
+  end
+
+  it "should be simple to create a mapredus as a part of a job" do
+    MapRedus::FileSystem.setnx("wordstream:test", GetWordCount::TEST)
+    MapRedus::FileSystem.setnx("charstream:test", "simpler test")
+    other_answer = {" "=>1, "l"=>1, "m"=>1, "e"=>2, "p"=>1, "r"=>1, "s"=>2, "t"=>2, "i"=>1}
+
+    @doc.calculate_chars("wordstream:test")
+    @other_doc.calculate_chars("charstream:test")
+    work_off
+
+    GetCharCount::EXPECTED_ANSWER.keys.each do |char|
+      @doc.mapreduce.char_count_result(char).should == GetCharCount::EXPECTED_ANSWER[char].to_s
+    end
+    
+    other_answer.keys.each do |char|
+      @other_doc.mapreduce.char_count_result(char).should == other_answer[char].to_s
+    end
+  end
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification 
+name: mapredus
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- John Le
+- Brian O'Rourke
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-07-06 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 1
+        - 0
+        - 4
+        version: 1.0.4
+  type: :runtime
+  name: redis
+  prerelease: false
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 1
+        - 8
+        version: "1.8"
+  type: :runtime
+  name: resque
+  prerelease: false
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: resque-scheduler
+  prerelease: false
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: redis_support
+  prerelease: false
+  version_requirements: *id004
+description: simple mapreduce framework using redis and resque
+email: john@doloreslabs.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.md
+files: 
+- lib/mapredus.rb
+- lib/mapredus/filesystem.rb
+- lib/mapredus/finalizer.rb
+- lib/mapredus/inputter.rb
+- lib/mapredus/keys.rb
+- lib/mapredus/mapper.rb
+- lib/mapredus/master.rb
+- lib/mapredus/outputter.rb
+- lib/mapredus/process.rb
+- lib/mapredus/reducer.rb
+- lib/mapredus/support.rb
+- LICENSE
+- README.md
+- spec/helper.rb
+- spec/helper_classes.rb
+- spec/mapredus_spec.rb
+has_rdoc: true
+homepage: http://github.com/dolores/mapredus
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: mapredus initial
+test_files: 
+- spec/helper.rb
+- spec/helper_classes.rb
+- spec/mapredus_spec.rb