answer-factory 0.1.2 → 0.1.3.4

data/spec/machines/evaluate_with_test_cases_spec.rb

@@ -0,0 +1,296 @@
+require File.join(File.dirname(__FILE__), "./../spec_helper")
+include Machines
+
+FakeWeb.allow_net_connect = false
+
+
+describe "Machines::TestCase" do
+  describe "inputs Array" do
+    before(:each) do
+      @tc = TestCase.new
+    end
+    
+    describe "bindings" do
+      it "should have an attribute called #inputs" do
+        @tc.should respond_to(:inputs)
+      end
+      
+      it "should accept a Hash as an initialization argument for #inputs" do
+        lambda{TestCase.new(inputs:{})}.should_not raise_error
+        TestCase.new(inputs:{"x1" => [:int, 12]}).inputs.should ==
+          {"x1" => [:int, 12]}
+      end
+      
+      it "should default :inputs to an empty Hash" do
+        TestCase.new.inputs.should == {}
+      end
+      
+      it "should have an attribute called #outputs" do
+        @tc.should respond_to(:outputs)
+      end
+      
+      it "should accept a Hash as an initialization argument for #outputs" do
+        lambda{TestCase.new(outputs:{})}.should_not raise_error
+        TestCase.new(outputs:{"y1" => [:bool, false]}).outputs.should ==
+          {"y1" => [:bool, false]}
+      end
+      
+      it "should default :inputs to an empty Hash" do
+        TestCase.new.outputs.should == {}
+      end
+    end
+  end
+end
+
+
+describe "EvaluateWithTestCases" do
+  
+  before(:each) do
+    @canned = '{"total_rows":1,"offset":0,"rows":[{"id":"0f60c293ad736abfdb083d33f71ef9ab","key":"ws1","value":{"_id":"0f60c293ad736abfdb083d33f71ef9ab","_rev":"1-473467b6dc1a4cba3498dd6eeb8e3206","blueprint":"do bar","tags":[],"scores":{},"progress":12,"timestamp":"2010/04/14 17:09:14 +0000"}}]}'
+    
+    FakeWeb.register_uri(:any, "http://127.0.0.1:5984/foo_training/_design/tester/_view/test_cases",
+      :body => @canned, :status => [200, "OK"])
+    @factory = Factory.new(name:"foo")
+    @tester = EvaluateWithTestCases.new(name: :tester)
+  end
+  
+  describe "initialization" do
+    
+    describe "name" do
+      it "should have a name" do
+        @tester.should respond_to(:name)
+      end
+    end
+    
+    describe "data" do
+      it "should call #load_training_data! if" do
+        EvaluateWithTestCases.new(name: :tester)
+      end
+    end
+    
+    describe "sensors" do
+      it "should have a #sensors attribute" do
+        EvaluateWithTestCases.new(name: :tester).should respond_to(:sensors)
+      end
+
+      it "should default to an empty Hash" do
+        EvaluateWithTestCases.new(name: :tester).sensors.should == {}
+      end
+      
+      describe ":build_sensor" do
+        before(:each) do
+          @m1 = EvaluateWithTestCases.new(name: :tester)
+        end
+        
+        it "should respond to :build_sensor" do
+          @m1.should respond_to(:build_sensor)
+        end
+        
+        it "should use the name argument as the Hash key in #sensors" do
+          @m1.build_sensor("harbor_master_score")
+          @m1.sensors.keys.should include("harbor_master_score")
+        end
+        
+        it "should take a block and store it as a Proc as the value in #sensors" do
+          block = lambda {|a| 9}
+          @m1.build_sensor("harbor_master_score", &block)
+          @m1.sensors["harbor_master_score"].should == block
+        end
+      end
+      
+    end
+  end
+  
+  
+  describe "#score method" do
+    it "should respond to :score" do
+      @tester.should respond_to(:score)
+    end
+    
+    it "should only accept a Batch" do
+      lambda{@tester.score(99)}.should raise_error(ArgumentError)
+      lambda{@tester.score(Batch.new)}.should_not raise_error(ArgumentError)
+    end
+    
+    describe "running Interpreter" do
+      before(:each) do
+        @tester.test_cases = [TestCase.new(inputs: {"x1:int" => 1}, outputs: {"y1" => 2})]
+        @batch = Batch.[](Answer.new("do a"))
+      end
+      
+      it "should create one Interpreter for each TestCase of #data_in_hand" do
+        i = Interpreter.new("")
+        Interpreter.should_receive(:new).at_least(1).times.and_return(i)
+        @tester.score(@batch)
+      end
+      
+      it "should register all the sensors" do
+        @tester.sensors = {"y1" => Proc.new{|interpreter| interpreter.peek_value(:int)} }
+        i = Interpreter.new
+        Interpreter.should_receive(:new).and_return(i)
+        i.should_receive(:register_sensor).exactly(1).times
+        @tester.score(@batch)
+      end
+
+      it "should set up Interpreters correctly" do
+        i = Interpreter.new
+        Interpreter.should_receive(:new).with(
+          "do a", {:name=>:tester, :target_size_in_points=>99}).and_return(i)
+        @tester.score(@batch,target_size_in_points:99)
+      end
+      
+      
+      
+    end
+    
+    describe "install_training_data_from_csv!" do
+      before(:each) do
+        FakeWeb.register_uri(:any,
+          "http://127.0.0.1:5984/dammit_training/_bulk_docs", 
+          :body => @canned, :status => [200, "OK"])
+        FakeWeb.register_uri(:any,
+          "http://127.0.0.1:5984/dammit_training/_design/tester/_view/test_cases", 
+          :body => @canned, :status => [200, "OK"])
+        
+        @f1 = Factory.new(name: "dammit")
+        @my_csv = "./spec/fixtures/my_data_source.csv"
+        @m1 = EvaluateWithTestCases.new(name: :tester, training_data_csv: @my_csv)
+        @training_db = "http://127.0.0.1:5984/dammit_training"
+        
+      end
+      
+      it "should get the filename as an initialization parameter" do
+        EvaluateWithTestCases.new(name: :tester, training_data_csv: "foo.csv").
+          csv_filename.should == "foo.csv"
+        EvaluateWithTestCases.new(name: :tester).csv_filename.should == nil
+      end
+      
+      it "should open a csv file" do
+        f = File.open(@my_csv)
+        File.should_receive(:open).and_return(f)
+        c = CSV.new(f, headers: true)
+        CSV.should_receive(:new).with(f, headers: true).and_return(c)
+        @m1.install_training_data_from_csv(@my_csv)
+      end
+      
+      it "should be the training_data default db" do
+        db = CouchRest.database!(@training_db)
+        CouchRest.should_receive(:database!).with(@training_db).and_return(db)
+        @m1.install_training_data_from_csv(@my_csv)
+      end
+      
+      it "makes one doc for every row" do
+        db = CouchRest.database!(@training_db)
+        CouchRest.should_receive(:database!).with(@training_db).and_return(db)
+        db.should_receive(:bulk_save_doc).exactly(3).times
+        @m1.install_training_data_from_csv(@my_csv)
+      end
+      
+      
+      it "should raise an error if every header doesn't contain a colon and a type string" do
+        lambda{@m1.header_prep("x1")}.should raise_error ArgumentError
+        lambda{@m1.header_prep("x1:")}.should raise_error ArgumentError
+        lambda{@m1.header_prep("x1:int")}.should_not raise_error ArgumentError
+        lambda{@m1.header_prep(":int")}.should raise_error ArgumentError
+      end
+      
+    end
+    
+    describe "load_training_data!" do
+      
+      before(:each) do
+        @factoreee = Factory.new(name:"dammit")
+        @m1 = EvaluateWithTestCases.new(name: :tester)
+        @design_doc = "tester/test_cases"  # we'll assume this has been set up!
+        @expected = {"total_rows"=>1, "offset"=>0, "rows"=>[{"id"=>"05d195b7bb436743ee36b4223008c4ce", "key"=>"05d195b7bb436743ee36b4223008c4ce", "value"=>{"_id"=>"05d195b7bb436743ee36b4223008c4ce", "_rev"=>"1-c9fae927001a1d4789d6396bcf0cd5a7", "inputs"=>{"x1:int"=>7}, "outputs"=>{"y1:grault"=>12}}}]}
+        
+      end
+      
+      it "should get the couch_db uri from configatron" do
+        @m1.training_data_view.should ==
+          "http://127.0.0.1:5984/dammit_training/_design/#{@m1.name}/_view/test_cases"
+      end
+      
+      it "should respond to :load_training_data!" do
+        @m1.should respond_to(:load_training_data!)
+      end
+      
+      it "should access the couch_db uri" do
+        FakeWeb.register_uri(:any,
+          "http://127.0.0.1:5984/dammit_training/_design/tester/_view/test_cases", 
+          :body => @canned, :status => [200, "OK"])
+        db = CouchRest.database!(@m1.training_datasource)
+        CouchRest.should_receive(:database!).and_return(db)
+        db.should_receive(:view).with(@design_doc).and_return(@expected)
+        @m1.load_training_data!
+      end
+      
+      it "should throw a useful error if the view isn't available"
+      
+      it "should ask for the view document" do
+        db = CouchRest.database!(@m1.training_datasource)
+        CouchRest.should_receive(:database!).and_return(db)
+        db.should_receive(:view).with(@design_doc).and_return(@expected)
+        @m1.load_training_data!
+      end
+      
+      it "should store Array of TestCases in @test_cases" do
+        db = CouchRest.database!(@m1.training_datasource)
+        CouchRest.should_receive(:database!).and_return(db)
+        db.should_receive(:view).with(@design_doc).and_return(@expected)
+        
+        @m1.load_training_data!
+        @m1.test_cases.should be_a_kind_of(Array)
+        @m1.test_cases.length.should == 1
+      end
+    end
+    
+    describe "scoring" do
+      before(:each) do
+        @m1 = EvaluateWithTestCases.new(name: :tester)
+        @m1.test_cases = (0...10).collect do |i|
+          TestCase.new(inputs: {"x1:int" => i}, outputs: {"y1" => 2*i, "y2" => 3*i})
+        end
+        @m1.build_sensor("y1") {|a| 777}
+        @m1.build_sensor("y2") {|a| 666}
+        
+        @batch = Batch.[](Answer.new("do a"))
+        @i1 = Interpreter.new
+      end
+      
+      it "should make an Interpreter for each row of training data" do
+        @m1.stub!(:load_training_data!)
+        Interpreter.should_receive(:new).exactly(10).times.and_return(@i1)
+        @m1.score(@batch)
+      end
+      
+      it "should run all the Interpreters" do
+        @m1.stub!(:load_training_data!)
+        Interpreter.should_receive(:new).exactly(10).times.and_return(@i1)
+        @i1.should_receive(:run).at_least(1).times.and_return({})
+        @m1.score(@batch)
+      end
+      
+      it "should register its sensors before each Interpreter run" do
+        Interpreter.stub!(:new).and_return(@i1)
+        @i1.should_receive(:register_sensor).at_least(1).times
+        @m1.score(@batch)
+      end
+      
+      it "should have a score for each sensor" do
+        @m1.score(@batch)
+        @batch.first.scores["y1"].should_not == nil
+        @batch.first.scores["y2"].should_not == nil
+      end
+      
+      it "should return sum of absolute errors" do
+        @m1.stub!(:load_training_data!)
+        @m1.score(@batch)
+        @batch[0].scores["y1"].should == 777+775+773+771+769+767+765+763+761+759
+        @batch[0].scores["y2"].should == 666+663+660+657+654+651+648+645+642+639
+      end
+    end
+    
+    
+  end
+end

data/spec/machines/select_by_summed_rank_spec.rb

@@ -0,0 +1,134 @@
+require File.join(File.dirname(__FILE__), "./../spec_helper")
+
+describe "Machines::SelectBySummedRank" do
+  before(:each) do
+    @best = Machines::SelectBySummedRank.new
+    @highlander = Batch.[](Answer.new("foo"), Answer.new("bar"), Answer.new("baz"))
+    @highlander[0].scores = {e1:15, e2:12}  # 3,3 = 6
+    @highlander[1].scores = {e1:5,  e2:5}   # 2,2 = 4
+    @highlander[2].scores = {e1:1,  e2:1}   # 1,1 = 2
+    
+    @lowlander = Batch.[](Answer.new("foo"), Answer.new("bar"), Answer.new("baz"))
+    @lowlander[0].scores = {e1:1, e2:3} # 1,3 = 4
+    @lowlander[1].scores = {e1:2, e2:2} # 2,2 = 4
+    @lowlander[2].scores = {e1:3, e2:1} # 3,1 = 4
+    
+    @allover = Batch.[](Answer.new("foo"), Answer.new("bar"), Answer.new("baz"))
+    @allover[0].scores = {e1:1, e2:3}         # 1,2,- 
+    @allover[1].scores = {e1:2, e2:2, e3:2}   # 2,1,1 
+    @allover[2].scores = {e1:3}               # 3,-,- 
+    
+    @separate = Batch.[](Answer.new("foo"), Answer.new("bar"))
+    @separate[0].scores = {e1:1}
+    @separate[1].scores = {e2:2, e3:2}
+  end
+  
+  
+  describe "#screen method" do
+    it "should respond to :screen" do
+      @best.should respond_to(:screen)
+    end
+    
+    it "should only accept a Batch as its argument" do
+      lambda{@best.screen(129)}.should raise_error
+      lambda{@best.screen([Answer.new("foo")])}.should raise_error
+      lambda{@best.screen(@highlander, comparison_criteria:[:e1])}.should_not raise_error
+    end
+    
+    it "should produce a Batch" do
+      @best.screen(@highlander, comparison_criteria:[:e2]).should be_a_kind_of(Batch)
+    end
+    
+    it "should use the :comparison_criteria option as a template Array of score keys" do
+      ignore_most = @best.screen(@lowlander, comparison_criteria:[:e1])
+      ignore_most.length.should == 1
+      ignore_most.should include(@lowlander[0])
+      
+      ignore_most = @best.screen(@lowlander, comparison_criteria:[:e2])
+      ignore_most.length.should == 1
+      ignore_most.should include(@lowlander[2])
+      
+      ignore_most = @best.screen(@allover, comparison_criteria:[:e1])
+      ignore_most.length.should == 1
+      ignore_most.should include(@allover[0])
+    end
+    
+    it "should use the intersection of all the score keys in the :batch as a default for criteria" do
+      @best.should_receive(:shared_goals).and_return([:e1])
+      @best.screen(@highlander)
+    end
+    
+    it "should accept (and store) an initialization :comparison_criteria option" do
+      just_one = Machines::SelectBySummedRank.new(comparison_criteria:[:e2]).
+        screen(@lowlander)
+      just_one.length.should == 1
+      just_one.should include(@lowlander[2])
+    end
+    
+    it "should include all Answers lacking a given score (since they can't be ranked)" do
+      dunno = @best.screen(@allover, comparison_criteria:[:e2])
+      dunno.length.should == 2
+      dunno.should include(@allover[1])
+      dunno.should include(@allover[2])
+      dunno.should_not include(@allover[0])
+    end
+    
+    it "should return the entire batch if no scores are shared" do
+      @best.screen(@separate).length.should == 2
+    end
+    
+    it "should override its initialization if given a #build option" do
+      overridden = Machines::SelectBySummedRank.new(comparison_criteria:[:e2]).
+        screen(@lowlander,comparison_criteria:[:e1])
+      overridden.length.should == 1
+      overridden[0].scores.should == {e1:1, e2:3}
+    end
+    
+    it "should return the lowest-ranking subset of the argument Batch" do
+      @best.screen(@lowlander).length.should == 3
+    end
+    
+    it "should return objects from the argument, not clones" do
+      (@highlander.collect {|a| a.object_id}).should include(@best.screen(@highlander)[0].object_id)
+    end
+    
+    it "should return a new Batch object" do
+      @best.screen(@highlander).object_id.should_not == @highlander.object_id
+    end
+    
+    it "should not change the :progress of the Answers" do
+      @best.screen(@highlander).each {|a| a.progress.should == 0}
+    end
+    
+  end
+  
+  
+  it "should respond to :generate as an alias to :screen" do
+    Machines::SelectBySummedRank.new.should respond_to(:generate)
+  end
+  
+  
+  describe "all_goals" do
+    it "should return an Array of score keys" do
+      @best.all_goals(@highlander).should == [:e1,:e2]
+    end
+    
+    it "should include every score key in the batch passed in" do
+      @best.all_goals(@allover).should == [:e1,:e2, :e3]
+    end
+    
+    it "should have one copy of each score name" do
+      @best.all_goals(@highlander).find_all {|e| e == :e1}.length.should == 1
+    end
+  end
+  
+  
+  describe "shared_goals" do
+    it "should return an Array of only the shared score keys from the Batch passed in" do
+      @best.shared_goals(@highlander).should == [:e1,:e2]
+      @best.shared_goals(@allover).should == [:e1]
+      @best.shared_goals(@separate).should == []
+    end
+  end
+end
+

data/spec/machines/select_nondominated_spec.rb

@@ -4,6 +4,7 @@ describe "Machines::SelectNondominated" do
   describe "#screen method" do
     before(:each) do
       @best = Machines::SelectNondominated.new
+      
       @highlander = Batch.[](Answer.new("foo"), Answer.new("bar"), Answer.new("baz"))
       @highlander[0].scores = {e1:5, e2:15}
       @highlander[1].scores = {e1:15,e2:5}
@@ -18,6 +19,10 @@ describe "Machines::SelectNondominated" do
       @allover[0].scores = {e1:1, e2:3}
       @allover[1].scores = {e1:2, e2:2, e3:2}
       @allover[2].scores = {e1:3}
+      
+      @separate = Batch.[](Answer.new("foo"), Answer.new("bar"))
+      @separate[0].scores = {e1:1}
+      @separate[1].scores = {e2:2, e3:2}
     end
     
     it "should respond to :screen" do
@@ -37,6 +42,7 @@ describe "Machines::SelectNondominated" do
     it "should accept a template Array of score keys" do
       @best.screen(@lowlander).length.should == 3
       @best.screen(@lowlander,comparison_criteria:[:e2]).length.should == 1
+      @best.screen(@lowlander,comparison_criteria:[:e2]).should include(@lowlander[2])
     end
     
     it "should use an initialization template as well" do
@@ -56,6 +62,8 @@ describe "Machines::SelectNondominated" do
       @best.screen(@allover, comparison_criteria:[:e1]).length.should == 1
       @best.screen(@allover, comparison_criteria:[:e2]).length.should == 2
       @best.screen(@allover, comparison_criteria:[:e3]).length.should == 3
+      
+      @best.screen(@separate).should == @separate
     end
     
     it "should return the nondominated subset of the argument" do