ci_reporter 1.0 → 1.6.5

data/spec/ci/reporter/cucumber_spec.rb

@@ -0,0 +1,230 @@
+# Copyright (c) 2006-2010 Nick Sieger <nicksieger@gmail.com>
+# See the file LICENSE.txt included with the distribution for
+# software license details.
+
+require File.dirname(__FILE__) + "/../../spec_helper.rb"
+require 'ci/reporter/cucumber'
+
+describe "The Cucumber reporter" do
+  describe CI::Reporter::CucumberFailure do
+    before(:each) do
+      @klass = mock("class")
+      @klass.stub!(:name).and_return("Exception name")
+
+      @exception = mock("exception")
+      @exception.stub!(:class).and_return(@klass)
+      @exception.stub!(:message).and_return("Exception message")
+      @exception.stub!(:backtrace).and_return(["First line", "Second line"])
+
+      @step = mock("step")
+      @step.stub!(:exception).and_return(@exception)
+
+      @cucumber_failure = CI::Reporter::CucumberFailure.new(@step)
+    end
+
+    it "should always return true for failure?" do
+      @cucumber_failure.should be_failure
+    end
+
+    it "should always return false for error?" do
+      @cucumber_failure.should_not be_error
+    end
+
+    it "should propagate the name as the underlying exception's class name" do
+      @step.should_receive(:exception)
+      @exception.should_receive(:class)
+      @klass.should_receive(:name)
+
+      @cucumber_failure.name.should == "Exception name"
+    end
+
+    it "should propagate the message as the underlying exception's message" do
+      @step.should_receive(:exception)
+      @exception.should_receive(:message)
+
+      @cucumber_failure.message.should == "Exception message"
+    end
+
+    it "should propagate and format the exception's backtrace" do
+      @step.should_receive(:exception)
+      @exception.should_receive(:backtrace)
+
+      @cucumber_failure.location.should == "First line\nSecond line"
+    end
+  end
+
+  describe CI::Reporter::Cucumber do
+    before(:each) do
+      @step_mother = mock("step_mother")
+      @io = mock("io")
+
+      @report_manager = mock("report_manager")
+      CI::Reporter::ReportManager.stub!(:new).and_return(@report_manager)
+    end
+
+    def new_instance
+      CI::Reporter::Cucumber.new(@step_mother, @io, {})
+    end
+
+    it "should create a new report manager to report on test success/failure" do
+      CI::Reporter::ReportManager.should_receive(:new)
+      new_instance
+    end
+
+    it "should record the feature name when a new feature is visited" do
+      cucumber = new_instance
+      cucumber.feature_name(nil, "Some feature name")
+      cucumber.name.should == "Some feature name"
+    end
+
+    it "should record only the first line of a feature name" do
+      cucumber = new_instance
+      cucumber.feature_name(nil, "Some feature name\nLonger description")
+      cucumber.name.should == "Some feature name"
+    end
+
+    context "applied to a feature" do
+      before(:each) do
+        @cucumber = new_instance
+        @cucumber.feature_name(nil, "Demo feature")
+
+        @test_suite = mock("test_suite", :start => nil, :finish => nil, :name= => nil)
+        CI::Reporter::TestSuite.stub!(:new).and_return(@test_suite)
+
+        @feature = mock("feature")
+
+        @report_manager.stub!(:write_report)
+      end
+
+      context "before" do
+        it "should create a new test suite" do
+          CI::Reporter::TestSuite.should_receive(:new).with(/Demo feature/)
+          @cucumber.before_feature(@feature)
+        end
+
+        it "should indicate that the test suite has started" do
+          @test_suite.should_receive(:start)
+          @cucumber.before_feature(@feature)
+        end
+      end
+
+      context "after" do
+        before :each do
+          @cucumber = new_instance
+          @cucumber.feature_name(nil, "Demo feature")
+
+          @test_suite = mock("test_suite", :start => nil, :finish => nil, :name= => nil)
+          CI::Reporter::TestSuite.stub!(:new).and_return(@test_suite)
+
+          @feature = mock("feature")
+
+          @report_manager.stub!(:write_report)
+
+          @cucumber.before_feature(@feature)
+        end
+
+        it "should indicate that the test suite has finished" do
+          @test_suite.should_receive(:finish)
+          @cucumber.after_feature(@feature)
+        end
+
+        it "should ask the report manager to write a report" do
+          @report_manager.should_receive(:write_report).with(@test_suite)
+          @cucumber.after_feature(@feature)
+        end
+      end
+    end
+
+    context "inside a scenario" do
+      before(:each) do
+        @testcases = []
+
+        @test_suite = mock("test_suite", :testcases => @testcases)
+
+        @cucumber = new_instance
+        @cucumber.stub!(:test_suite).and_return(@test_suite)
+
+        @test_case = mock("test_case", :start => nil, :finish => nil, :name => "Step Name")
+        CI::Reporter::TestCase.stub!(:new).and_return(@test_case)
+
+        @step = mock("step", :status => :passed)
+        @step.stub!(:name).and_return("Step Name")
+      end
+
+      context "before steps" do
+        it "should create a new test case" do
+          CI::Reporter::TestCase.should_receive(:new).with("Step Name")
+          @cucumber.scenario_name(nil, "Step Name")
+          @cucumber.before_steps(@step)
+        end
+
+        it "should indicate that the test case has started" do
+          @test_case.should_receive(:start)
+          @cucumber.before_steps(@step)
+        end
+      end
+
+      context "after steps" do
+        before :each do
+          @cucumber.before_steps(@step)
+        end
+
+        it "should indicate that the test case has finished" do
+          @test_case.should_receive(:finish)
+          @cucumber.after_steps(@step)
+        end
+
+        it "should add the test case to the suite's list of cases" do
+          @testcases.should be_empty
+          @cucumber.after_steps(@step)
+          @testcases.should_not be_empty
+          @testcases.first.should == @test_case
+        end
+
+        it "should alter the name of a test case that is pending to include '(PENDING)'" do
+          @step.stub!(:status).and_return(:pending)
+          @test_case.should_receive(:name=).with("Step Name (PENDING)")
+          @cucumber.after_steps(@step)
+        end
+
+        it "should alter the name of a test case that is undefined to include '(PENDING)'" do
+          @step.stub!(:status).and_return(:undefined)
+          @test_case.should_receive(:name=).with("Step Name (PENDING)")
+          @cucumber.after_steps(@step)
+        end
+
+        it "should alter the name of a test case that was skipped to include '(SKIPPED)'" do
+          @step.stub!(:status).and_return(:skipped)
+          @test_case.should_receive(:name=).with("Step Name (SKIPPED)")
+          @cucumber.after_steps(@step)
+        end
+      end
+
+      describe "that fails" do
+        before(:each) do
+          @step.stub!(:status).and_return(:failed)
+
+          @failures = []
+          @test_case.stub!(:failures).and_return(@failures)
+
+          @cucumber.before_steps(@step)
+
+          @cucumber_failure = mock("cucumber_failure")
+          CI::Reporter::CucumberFailure.stub!(:new).and_return(@cucumber_failure)
+        end
+
+        it "should create a new cucumber failure with that step" do
+          CI::Reporter::CucumberFailure.should_receive(:new).with(@step)
+          @cucumber.after_steps(@step)
+        end
+
+        it "should add the failure to the suite's list of failures" do
+          @failures.should be_empty
+          @cucumber.after_steps(@step)
+          @failures.should_not be_empty
+          @failures.first.should == @cucumber_failure
+        end
+      end
+    end
+  end
+end

data/spec/ci/reporter/output_capture_spec.rb

@@ -0,0 +1,57 @@
+# Copyright (c) 2006-2010 Nick Sieger <nicksieger@gmail.com>
+# See the file LICENSE.txt included with the distribution for
+# software license details.
+
+require File.dirname(__FILE__) + "/../../spec_helper.rb"
+require 'rexml/document'
+
+describe "Output capture" do
+  before(:each) do
+    @suite = CI::Reporter::TestSuite.new "test"
+  end
+
+  it "should save stdout and stderr messages written during the test run" do
+    @suite.start
+    puts "Hello"
+    $stderr.print "Hi"
+    @suite.finish
+    @suite.stdout.should == "Hello\n"
+    @suite.stderr.should == "Hi"
+  end
+
+  it "should include system-out and system-err elements in the xml output" do
+    @suite.start
+    puts "Hello"
+    $stderr.print "Hi"
+    @suite.finish
+
+    root = REXML::Document.new(@suite.to_xml).root
+    root.elements.to_a('//system-out').length.should == 1
+    root.elements.to_a('//system-err').length.should == 1
+    root.elements.to_a('//system-out').first.texts.first.to_s.strip.should == "Hello"
+    root.elements.to_a('//system-err').first.texts.first.to_s.strip.should == "Hi"
+  end
+
+  it "should return $stdout and $stderr to original value after finish" do
+    out, err = $stdout, $stderr
+    @suite.start
+    $stdout.object_id.should_not == out.object_id
+    $stderr.object_id.should_not == err.object_id
+    @suite.finish
+    $stdout.object_id.should == out.object_id
+    $stderr.object_id.should == err.object_id
+  end
+  
+  it "should capture only during run of owner test suite" do
+    $stdout.print "A"
+    $stderr.print "A"
+    @suite.start
+    $stdout.print "B"
+    $stderr.print "B"
+    @suite.finish
+    $stdout.print "C"
+    $stderr.print "C"
+    @suite.stdout.should == "B"
+    @suite.stderr.should == "B"
+  end
+end

data/spec/ci/reporter/rake/rake_tasks_spec.rb

@@ -0,0 +1,100 @@
+# Copyright (c) 2006-2010 Nick Sieger <nicksieger@gmail.com>
+# See the file LICENSE.txt included with the distribution for
+# software license details.
+
+require File.dirname(__FILE__) + "/../../../spec_helper.rb"
+require 'rake'
+
+def save_env(v)
+  ENV["PREV_#{v}"] = ENV[v]
+end
+def restore_env(v)
+  ENV[v] = ENV["PREV_#{v}"]
+  ENV.delete("PREV_#{v}")
+end
+
+describe "ci_reporter ci:setup:testunit task" do
+  before(:each) do
+    @rake = Rake::Application.new
+    Rake.application = @rake
+    load CI_REPORTER_LIB + '/ci/reporter/rake/test_unit.rb'
+    save_env "CI_REPORTS"
+    save_env "TESTOPTS"
+    ENV["CI_REPORTS"] = "some-bogus-nonexistent-directory-that-wont-fail-rm_rf"
+  end
+  after(:each) do
+    restore_env "TESTOPTS"
+    restore_env "CI_REPORTS"
+    Rake.application = nil
+  end
+
+  it "should set ENV['TESTOPTS'] to include test/unit setup file" do
+    @rake["ci:setup:testunit"].invoke
+    ENV["TESTOPTS"].should =~ /test_unit_loader/
+  end
+  
+  it "should append to ENV['TESTOPTS'] if it already contains a value" do
+    ENV["TESTOPTS"] = "somevalue".freeze
+    @rake["ci:setup:testunit"].invoke
+    ENV["TESTOPTS"].should =~ /somevalue.*test_unit_loader/
+  end
+end
+
+describe "ci_reporter ci:setup:rspec task" do
+  before(:each) do
+    @rake = Rake::Application.new
+    Rake.application = @rake
+    load CI_REPORTER_LIB + '/ci/reporter/rake/rspec.rb'
+    save_env "CI_REPORTS"
+    save_env "SPEC_OPTS"
+    ENV["CI_REPORTS"] = "some-bogus-nonexistent-directory-that-wont-fail-rm_rf"
+  end
+  after(:each) do
+    restore_env "SPEC_OPTS"
+    restore_env "CI_REPORTS"
+    Rake.application = nil
+  end
+  
+  it "should set ENV['SPEC_OPTS'] to include rspec formatter args" do
+    @rake["ci:setup:rspec"].invoke
+    ENV["SPEC_OPTS"].should =~ /--require.*rspec_loader.*--format.*CI::Reporter::RSpec/
+  end
+
+  it "should set ENV['SPEC_OPTS'] to include rspec doc formatter if task is ci:setup:rspecdoc" do
+    @rake["ci:setup:rspecdoc"].invoke
+    ENV["SPEC_OPTS"].should =~ /--require.*rspec_loader.*--format.*CI::Reporter::RSpecDoc/
+  end
+  
+  it "should append to ENV['SPEC_OPTS'] if it already contains a value" do
+    ENV["SPEC_OPTS"] = "somevalue".freeze
+    @rake["ci:setup:rspec"].invoke
+    ENV["SPEC_OPTS"].should =~ /somevalue.*--require.*rspec_loader.*--format.*CI::Reporter::RSpec/
+  end
+end
+
+describe "ci_reporter ci:setup:cucumber task" do
+  before(:each) do
+    @rake = Rake::Application.new
+    Rake.application = @rake
+    load CI_REPORTER_LIB + '/ci/reporter/rake/cucumber.rb'
+    save_env "CI_REPORTS"
+    save_env "CUCUMBER_OPTS"
+    ENV["CI_REPORTS"] = "some-bogus-nonexistent-directory-that-wont-fail-rm_rf"
+  end
+  after(:each) do
+    restore_env "CUCUMBER_OPTS"
+    restore_env "CI_REPORTS"
+    Rake.application = nil
+  end
+  
+  it "should set ENV['CUCUMBER_OPTS'] to include cucumber formatter args" do
+    @rake["ci:setup:cucumber"].invoke
+    ENV["CUCUMBER_OPTS"].should =~ /--require.*cucumber_loader.*--format.*CI::Reporter::Cucumber/
+  end
+
+  it "should append to ENV['CUCUMBER_OPTS'] if it already contains a value" do
+    ENV["CUCUMBER_OPTS"] = "somevalue".freeze
+    @rake["ci:setup:cucumber"].invoke
+    ENV["CUCUMBER_OPTS"].should =~ /somevalue.*--require.*cucumber_loader.*--format.*CI::Reporter::Cucumber/
+  end
+end

data/spec/ci/reporter/report_manager_spec.rb

@@ -1,35 +1,39 @@
+# Copyright (c) 2006-2010 Nick Sieger <nicksieger@gmail.com>
+# See the file LICENSE.txt included with the distribution for
+# software license details.
+
 require File.dirname(__FILE__) + "/../../spec_helper.rb"
 
-context "The ReportManager" do
-  setup do
+describe "The ReportManager" do
+  before(:each) do
     @reports_dir = REPORTS_DIR
   end
 
-  teardown do
+  after(:each) do
     FileUtils.rm_rf @reports_dir
     ENV["CI_REPORTS"] = nil
   end
 
-  specify "should create the report directory according to the given prefix" do
+  it "should create the report directory according to the given prefix" do
     CI::Reporter::ReportManager.new("spec")
-    File.directory?(@reports_dir).should_be true
+    File.directory?(@reports_dir).should be_true
   end
   
-  specify "should create the report directory based on CI_REPORTS environment variable if set" do
+  it "should create the report directory based on CI_REPORTS environment variable if set" do
     @reports_dir = "#{Dir.getwd}/dummy"
     ENV["CI_REPORTS"] = @reports_dir
     CI::Reporter::ReportManager.new("spec")
-    File.directory?(@reports_dir).should_be true
+    File.directory?(@reports_dir).should be_true
   end
   
-  specify "should write reports based on name and xml content of a test suite" do
+  it "should write reports based on name and xml content of a test suite" do
     reporter = CI::Reporter::ReportManager.new("spec")
     suite = mock("test suite")
     suite.should_receive(:name).and_return("some test suite name")
     suite.should_receive(:to_xml).and_return("<xml></xml>")
     reporter.write_report(suite)
     filename = "#{REPORTS_DIR}/SPEC-some-test-suite-name.xml"
-    File.exist?(filename).should_be true
+    File.exist?(filename).should be_true
     File.open(filename) {|f| f.read.should == "<xml></xml>"}
   end
-end
+end

data/spec/ci/reporter/rspec_spec.rb

@@ -1,35 +1,148 @@
+# Copyright (c) 2006-2011 Nick Sieger <nicksieger@gmail.com>
+# See the file LICENSE.txt included with the distribution for
+# software license details.
+
 require File.dirname(__FILE__) + "/../../spec_helper.rb"
 require 'stringio'
 
-context "The RSpec reporter" do
-  setup do
+describe "The RSpec reporter" do
+  before(:each) do
     @error = mock("error")
-    @error.stub!(:exception).and_return do
-      begin
-        raise StandardError, "error message"
-      rescue => e
-        e
-      end
-    end
     @error.stub!(:expectation_not_met?).and_return(false)
+    @error.stub!(:pending_fixed?).and_return(false)
+    @error.stub!(:exception).and_return(StandardError.new)
     @report_mgr = mock("report manager")
-    @fmt = CI::Reporter::RSpec.new(StringIO.new(""), false, false, @report_mgr)
+    @options = mock("options")
+    @args = [@options, StringIO.new("")]
+    @args.shift unless defined?(::Spec) && ::Spec::VERSION::MAJOR == 1 && ::Spec::VERSION::MINOR >= 1
+    @fmt = CI::Reporter::RSpec.new *@args
+    @fmt.report_manager = @report_mgr
+    @formatter = mock("formatter")
+    @fmt.formatter = @formatter
+  end
+
+  it "should use a progress bar formatter by default" do
+    fmt = CI::Reporter::RSpec.new *@args
+    fmt.formatter.should be_instance_of(CI::Reporter::RSpecFormatters::ProgressFormatter)
+  end
+
+  it "should use a specdoc formatter for RSpecDoc" do
+    fmt = CI::Reporter::RSpecDoc.new *@args
+    fmt.formatter.should be_instance_of(CI::Reporter::RSpecFormatters::DocFormatter)
+  end
+
+  it "should create a test suite with one success, one failure, and one pending" do
+    @report_mgr.should_receive(:write_report).and_return do |suite|
+      suite.testcases.length.should == 3
+      suite.testcases[0].should_not be_failure
+      suite.testcases[0].should_not be_error
+      suite.testcases[1].should be_error
+      suite.testcases[2].name.should =~ /\(PENDING\)/
+    end
+
+    example_group = mock "example group"
+    example_group.stub!(:description).and_return "A context"
+
+    @formatter.should_receive(:start).with(3)
+    @formatter.should_receive(:example_group_started).with(example_group)
+    @formatter.should_receive(:example_started).exactly(3).times
+    @formatter.should_receive(:example_passed).once
+    @formatter.should_receive(:example_failed).once
+    @formatter.should_receive(:example_pending).once
+    @formatter.should_receive(:start_dump).once
+    @formatter.should_receive(:dump_failure).once
+    @formatter.should_receive(:dump_summary).once
+    @formatter.should_receive(:dump_pending).once
+    @formatter.should_receive(:close).once
+
+    @fmt.start(3)
+    @fmt.example_group_started(example_group)
+    @fmt.example_started("should pass")
+    @fmt.example_passed("should pass")
+    @fmt.example_started("should fail")
+    @fmt.example_failed("should fail", 1, @error)
+    @fmt.example_started("should be pending")
+    @fmt.example_pending("A context", "should be pending", "Not Yet Implemented")
+    @fmt.start_dump
+    @fmt.dump_failure(1, mock("failure"))
+    @fmt.dump_summary(0.1, 3, 1, 1)
+    @fmt.dump_pending
+    @fmt.close
+  end
+
+  it "should support RSpec 1.0.8 #add_behavior" do
+    @formatter.should_receive(:start)
+    @formatter.should_receive(:add_behaviour).with("A context")
+    @formatter.should_receive(:example_started).once
+    @formatter.should_receive(:example_passed).once
+    @formatter.should_receive(:dump_summary)
+    @report_mgr.should_receive(:write_report)
+
+    @fmt.start(2)
+    @fmt.add_behaviour("A context")
+    @fmt.example_started("should pass")
+    @fmt.example_passed("should pass")
+    @fmt.dump_summary(0.1, 1, 0, 0)
   end
 
-  specify "should create a test suite with one success and one failure" do
+  it "should use the example #description method when available" do
+    group = mock "example group"
+    group.stub!(:description).and_return "group description"
+    example = mock "example"
+    example.stub!(:description).and_return "should do something"
+
+    @formatter.should_receive(:start)
+    @formatter.should_receive(:example_group_started).with(group)
+    @formatter.should_receive(:example_started).with(example).once
+    @formatter.should_receive(:example_passed).once
+    @formatter.should_receive(:dump_summary)
     @report_mgr.should_receive(:write_report).and_return do |suite|
-      suite.testcases.length.should == 2
-      suite.testcases.first.should_not_be_failure
-      suite.testcases.first.should_not_be_error
-      suite.testcases.last.should_be_error
+      suite.testcases.last.name.should == "should do something"
     end
 
     @fmt.start(2)
-    @fmt.add_context("A context", true)
-    @fmt.spec_started("should pass")
-    @fmt.spec_passed("should pass")
-    @fmt.spec_started("should fail")
-    @fmt.spec_failed("should fail", 1, @error)
-    @fmt.dump_summary(0.1, 2, 1)
-  end
-end
+    @fmt.example_group_started(group)
+    @fmt.example_started(example)
+    @fmt.example_passed(example)
+    @fmt.dump_summary(0.1, 1, 0, 0)
+  end
+
+  it "should create a test suite with failure in before(:all)" do
+    example_group = mock "example group"
+    example_group.stub!(:description).and_return "A context"
+
+    @formatter.should_receive(:start)
+    @formatter.should_receive(:example_group_started).with(example_group)
+    @formatter.should_receive(:example_started).once
+    @formatter.should_receive(:example_failed).once
+    @formatter.should_receive(:dump_summary)
+    @report_mgr.should_receive(:write_report)
+
+    @fmt.start(2)
+    @fmt.example_group_started(example_group)
+    @fmt.example_failed("should fail", 1, @error)
+    @fmt.dump_summary(0.1, 1, 0, 0)
+  end
+
+  describe 'RSpec2Failure' do
+    before(:each) do
+      @formatter = mock "formatter"
+      @formatter.should_receive(:format_backtrace).and_return("backtrace")
+      @rspec20_example = mock('RSpec2.0 Example', :execution_result => {:exception_encountered => StandardError.new('rspec2.0 ftw')})
+      @rspec22_example = mock('RSpec2.2 Example', :execution_result => {:exception => StandardError.new('rspec2.2 ftw')})
+    end
+
+    it 'should handle rspec (< 2.2) execution results' do
+      failure = CI::Reporter::RSpec2Failure.new(@rspec20_example, @formatter)
+      failure.name.should_not be_nil
+      failure.message.should == 'rspec2.0 ftw'
+      failure.location.should_not be_nil
+    end
+    it 'should handle rspec (>= 2.2) execution results' do
+      failure = CI::Reporter::RSpec2Failure.new(@rspec22_example, @formatter)
+      failure.name.should_not be_nil
+      failure.message.should == 'rspec2.2 ftw'
+      failure.location.should_not be_nil
+    end
+  end
+end