qunited 0.3.1 → 0.4.0

data/lib/qunited/application.rb

@@ -1,5 +1,4 @@
 require 'optparse'
-require 'ostruct'
 
 module QUnited
   class Application
@@ -12,7 +11,8 @@ module QUnited
 
     def run_tests
       js_source_files, js_test_files = ARGV.join(' ').split('--').map { |file_list| file_list.split(' ') }
-      exit QUnited::Runner.new(js_source_files, js_test_files, options).run
+      passed = QUnited::Runner.new(js_source_files, js_test_files, options).run
+      exit (passed ? 0 : 10)
     end
 
     # Client options generally parsed from the command line
@@ -80,21 +80,19 @@ Options:
     def handle_exceptions
       begin
         yield
-      rescue SystemExit
-        exit
       rescue UsageError => ex
         $stderr.puts ex.message
         exit 1
       rescue OptionParser::InvalidOption => ex
         $stderr.puts ex.message
         exit 1
-      rescue Exception => ex
-        display_error_message ex
+      rescue StandardError => ex
+        display_crash_message ex
         exit 1
       end
     end
 
-    def display_error_message(ex)
+    def display_crash_message(ex)
       msg = <<MSG
 QUnited has aborted! If this is unexpected, you may want to open an issue at
 github.com/aaronroyer/qunited to get a possible bug fixed. If you do, please

data/lib/qunited/driver/base.rb

@@ -4,7 +4,12 @@ module QUnited
       # Path of the common (to all drivers) supporting files directory
       SUPPORT_DIR = File.expand_path('../support', __FILE__)
 
+      TEST_RESULT_START_TOKEN = 'QUNITED_TEST_RESULT_START_TOKEN'
+      TEST_RESULT_END_TOKEN   = 'QUNITED_TEST_RESULT_END_TOKEN'
+      TEST_RESULT_REGEX       = /#{TEST_RESULT_START_TOKEN}(.*?)#{TEST_RESULT_END_TOKEN}/m
+
       attr_reader :results, :source_files, :test_files
+      attr_accessor :formatter
 
       # Finds an executable on the PATH. Returns the absolute path of the
       # executable if found, otherwise nil.
@@ -49,6 +54,12 @@ module QUnited
       def name
         self.class.name.split('::')[-1]
       end
+
+      protected
+
+      def send_to_formatter(method, *args)
+        formatter.send(method, *args) if formatter
+      end
     end
   end
 end

data/lib/qunited/driver/phantomjs/phantomjs.rb

@@ -16,7 +16,7 @@ module QUnited
       end
 
       def name
-        "PhantomJS" # Slightly more accurate than our class name
+        'PhantomJS'
       end
 
       def run
@@ -24,20 +24,37 @@ module QUnited
         tests_file.write(tests_page_content)
         tests_file.close
 
-        results_file = Tempfile.new('qunited_results')
-        results_file.close
+        send_to_formatter(:start)
 
-        cmd = %{phantomjs "#{File.join(SUPPORT_DIR, 'runner.js')}" }
-        cmd << %{#{tests_file.path} #{results_file.path}}
+        cmd = %|phantomjs "#{File.join(SUPPORT_DIR, 'runner.js')}" "#{tests_file.path}"|
+
+        @results = []
 
         Open3.popen3(cmd) do |stdin, stdout, stderr|
-          # PhantomJS sometimes puts error messages to stdout - redirect them to stderr
-          [stdout, stderr].each do |io|
-            unless (io_str = io.read).strip.empty? then $stderr.puts(io_str) end
+          results_collector = ResultsCollector.new(stdout)
+
+          results_collector.on_test_result do |result|
+            @results << result
+            method = result.passed? ? :test_passed : :test_failed
+            send_to_formatter(method, result)
+          end
+
+          results_collector.on_non_test_result_line do |line|
+            # PhantomJS sometimes puts error messages to stdout. If we are not reading
+            # a test result then redirect any output to stderr
+            $stderr.puts(line)
           end
+
+          results_collector.collect_results
+
+          err = stderr.read
+          unless err.nil? || err.strip.empty? then $stderr.puts(err) end
         end
 
-        @results = ::QUnited::Results.from_javascript_produced_json(IO.read(results_file.path))
+        send_to_formatter(:stop)
+        send_to_formatter(:summarize)
+
+        @results
       end
 
       private

data/lib/qunited/driver/phantomjs/support/runner.js

@@ -1,102 +1,97 @@
 /*
-Portions of this file are from the PhantomJS project from Ofi Labs.
-
-  Copyright (C) 2011 Ariya Hidayat <ariya.hidayat@gmail.com>
-  Copyright (C) 2011 Ivan De Marino <ivan.de.marino@gmail.com>
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+ * Runs QUnit tests in PhantomJS and outputs test result data, in JSON format, to stdout.
+ *
+ * Tokens are placed around each test result to allow them to be parsed individually. The tokens
+ * match the constants in QUnited::Driver::ResultsCollector.
+ *
+ * Usage:
+ *   phantomjs runner.js PATH_TO_TESTS_HTML_PAGE
+ */
 
-  * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-  * Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-  * Neither the name of the <organization> nor the
-    names of its contributors may be used to endorse or promote products
-    derived from this software without specific prior written permission.
+var system = require('system'),
+    webpage = require('webpage');
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
+if (system.args.length < 2) {
+  console.log('No tests file specified');
+  phantom.exit(1);
+}
 
-var system = require('system'), fs = require("fs");
+var page = webpage.create(),
+    testsHtmlFile = system.args[1],
+    config = {
+      resultsCheckInterval: 200,        // Check for new results at this interval
+      testsCompletedCheckInterval: 100, // Check for all tests completing at this interval
+      testsTimeout: 10001               // Time out and indicate error after this many millis
+    };
 
-/**
- * Wait until the test condition is true or a timeout occurs. Useful for waiting
- * on a server response or for a ui change (fadeIn, etc.) to occur.
+/*
+ * Writes any collected QUnit results that are pending output to stdout (this is done with
+ * console.log in PhantomJS).
  *
- * @param testFx javascript condition that evaluates to a boolean,
- * it can be passed in as a string (e.g.: "1 == 1" or "$('#bar').is(':visible')" or
- * as a callback function.
- * @param onReady what to do when testFx condition is fulfilled,
- * it can be passed in as a string (e.g.: "1 == 1" or "$('#bar').is(':visible')" or
- * as a callback function.
- * @param timeOutMillis the max amount of time to wait. If not specified, 3 sec is used.
+ * Tokens are placed around each test result to allow them to be parsed out later. Note that the
+ * tokens must match the constants in QUnited::Driver::ResultsCollector.
+ *
+ * JSON.stringify must be called in the context of the page to properly serialize null values in
+ * results. If it is called here in the PhantomJS interpreter code then null values are serialized
+ * as empty strings. Finding out exactly why this happens would take more investigation. For now
+ * it seems that stringifying on the page is a decent solution, though it is slightly less robust
+ * since JSON serialization may be tampered with in user code.
  */
-function waitFor(testFx, onReady, timeOutMillis) {
-	var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 3001, //< Default Max Timeout is 3s
-	start = new Date().getTime(),
-	condition = false,
-	interval = setInterval(function() {
-		if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) {
-			// If not time-out yet and condition not yet fulfilled
-			condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code
-		} else {
-			if (!condition) {
-				// If condition still not fulfilled (timeout but condition is 'false')
-				console.log("ERROR: Timeout waiting for tests to complete");
-				phantom.exit(1);
-			} else {
-				// Condition fulfilled (timeout and/or condition is 'true')
-				//console.log("'waitFor()' finished in " + (new Date().getTime() - start) + "ms.");
-				typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condition is fulfilled
-				clearInterval(interval); //< Stop this interval
-			}
-		}
-	}, 100);
-};
-
+function writePendingTestResults() {
+  var serializedResults = page.evaluate(function() {
+    var pendingResults = [];
+    while (QUnited.testResultsPendingOutput.length > 0) {
+      pendingResults.push(QUnited.util.jsonStringify(QUnited.testResultsPendingOutput.shift()));
+    }
+    return pendingResults;
+  });
 
-if (system.args.length < 2) {
-	console.log('No tests file specified');
-	phantom.exit(1);
-} else if (system.args.length < 3) {
-	console.log('No results output file specified');
-	phantom.exit(1);
+  var i, output;
+  for (i = 0; i < serializedResults.length; i++) {
+    output = 'QUNITED_TEST_RESULT_START_TOKEN';
+    output += serializedResults[i];
+    output += 'QUNITED_TEST_RESULT_END_TOKEN';
+    console.log(output);
+  }
 }
 
-var page = require('webpage').create(),
-		tests_html_file = system.args[1],
-		results_output_file = system.args[2];
+/*
+ * Executes the given function once all tests have completed. If a timeout occurs then exit
+ * with a status code of 1.
+ */
+function whenTestsHaveCompleted(fn) {
+  var start = new Date().getTime(),
+      testsHaveCompleted = false,
+      interval = setInterval(function() {
+        if ( (new Date().getTime() - start < config.testsTimeout) && !testsHaveCompleted ) {
+          testsHaveCompleted = page.evaluate(function() { return QUnited.testsHaveCompleted; });
+        } else {
+          if (testsHaveCompleted) {
+            fn();
+            clearInterval(interval);
+          } else {
+            // Tests took too long
+            console.log("ERROR: Timeout waiting for tests to complete");
+            phantom.exit(1);
+          }
+        }
+      }, config.testsCompletedCheckInterval);
+};
 
-page.open(tests_html_file, function(status) {
+/*
+ * Open the HTML page that contains all of our QUnit tests. As it is running, check for collected
+ * test results and output them if we have any. Also check whether tests have completed. Once they
+ * have completed, output any remaining results and exit.
+ */
+page.open(testsHtmlFile, function(status) {
     if (status !== "success") {
         console.log("Could not open tests file");
         phantom.exit(1);
     } else {
-        waitFor(function(){
-						// Done when all tests have run (the results have been rendered)
-            return page.evaluate(function(){
-                var el = document.getElementById('qunit-testresult');
-                if (el && el.innerText.match('completed')) {
-                    return true;
-                }
-                return false;
-            });
-        }, function(){
-						// Results should have been collected with code in qunited.js. Check that file
-						// for more details. Grab the YAML it outputs and write it to the results file.
-            var results = page.evaluate(function() { return QUnited.collectedTestResultsAsJson(); });
-						fs.write(results_output_file, results, 'a');
+        setInterval(writePendingTestResults, config.resultsCheckInterval);
+
+        whenTestsHaveCompleted(function() {
+            writePendingTestResults();
             phantom.exit(0);
         });
     }

data/lib/qunited/driver/phantomjs/support/tests_page.html.erb

@@ -21,6 +21,9 @@
 	<%= script_tag source_file %>
 <% end %>
 <% test_files.each do |test_file| %>
+  <script>
+    QUnited.currentTestFile = "<%= test_file %>";
+  </script>
 	<%= script_tag test_file %>
 <% end %>
 

data/lib/qunited/driver/results_collector.rb

@@ -0,0 +1,105 @@
+module QUnited
+  module Driver
+
+    # Collects test results from lines of JavaScript interpreter output.
+    #
+    # QUnited test running drivers may run a JavaScript interpreter in a separate process and
+    # observe the output (say, on stdout) for text containing test results. These results may
+    # be delimited by tokens that allow ResultsCollector to recognize the beginning and end of
+    # JSON test results. The test running driver must be properly configured to emit the correct
+    # tokens, matching TEST_RESULT_START_TOKEN and TEST_RESULT_END_TOKEN before and after strings
+    # of test results serialized as valid JSON.
+    #
+    # If everything is set up correctly the recognized results are parsed and QUnitTestResult
+    # objects are produced for each.
+    #
+    #
+    # To use, initialize with the IO object that provides the output from the test running
+    # process. Then call on_test_result with a block to be called when a test is collected.
+    # A QUnited::QUnitTestResult object will be passed to the block for each test.
+    #
+    #   rc = ResultsCollector.new(stdout_from_test_runner)
+    #   rc.on_test_result {|test_result| puts "I've got a result: #{test_result.inspect}" }
+    #
+    # If you need to capture output that is not part of any test result, you can call
+    # on_non_test_result_line with another block to do this. Each line of output that is not part
+    # of test result JSON is passed to the block.
+    #
+    #   rc.on_non_test_result_line {|line| puts "This line is not part of a test result: #{line}"}
+    #
+    class ResultsCollector
+      TEST_RESULT_START_TOKEN    = 'QUNITED_TEST_RESULT_START_TOKEN'
+      TEST_RESULT_END_TOKEN      = 'QUNITED_TEST_RESULT_END_TOKEN'
+      ONE_LINE_TEST_RESULT_REGEX = /#{TEST_RESULT_START_TOKEN}(.*?)#{TEST_RESULT_END_TOKEN}/
+
+      def initialize(io)
+        @io = io
+        @results = []
+        @on_test_result_block = nil
+        @on_non_test_result_line_block = nil
+        @partial_test_result = ''
+      end
+
+      # Set a block to be called when a test result has been parsed. The block is passed a
+      # QUnitTestResult object.
+      def on_test_result(&block)
+        raise ArgumentError.new('must provide a block') unless block_given?
+        @on_test_result_block = block
+      end
+
+      # Set a block to be called when a line of output is read from the IO object that is not
+      # part of a test result. The block is passed the line of output.
+      def on_non_test_result_line(&block)
+        raise ArgumentError.new('must provide a block') unless block_given?
+        @on_non_test_result_line_block = block
+      end
+
+      # Read all available lines from the IO and parse results from it. If blocks have been set
+      # with on_test_result and/or on_non_test_result_line they will be called when appropriate.
+      def collect_results
+        while collect_next_line; end
+      end
+
+      # Read the next line from the IO and parse results from it, if applicable. If blocks have
+      # been set with on_test_result and/or on_non_test_result_line they will be called when
+      # appropriate.
+      #
+      # Usually collect_results should be used unless lines need to be read one at a time for
+      # some reason.
+      def collect_next_line
+        line = @io.gets
+        return nil unless line
+
+        if line =~ ONE_LINE_TEST_RESULT_REGEX
+          process_test_result $1
+
+        elsif line.include? TEST_RESULT_START_TOKEN
+          @partial_test_result << line.sub(TEST_RESULT_START_TOKEN, '')
+
+        elsif line.include? TEST_RESULT_END_TOKEN
+          @partial_test_result << line.sub(TEST_RESULT_END_TOKEN, '')
+          process_test_result @partial_test_result
+          @partial_test_result = ''
+
+        elsif !@partial_test_result.empty?
+          # Middle of a test result
+          @partial_test_result << line
+
+        else
+          @on_non_test_result_line_block.call(line) if @on_non_test_result_line_block
+
+        end
+
+        line
+      end
+
+      private
+
+      def process_test_result(test_result_json)
+        result = ::QUnited::QUnitTestResult.from_json(test_result_json)
+        @results << result
+        @on_test_result_block.call(result) if @on_test_result_block
+      end
+    end
+  end
+end

data/lib/qunited/driver/rhino/rhino.rb

@@ -28,23 +28,33 @@ module QUnited
         source_files_args = @source_files.map { |sf| %{"#{sf}"} }.join(' ')
         test_files_args = @test_files.map { |tf| %{"#{tf}"} }.join(' ')
 
-        results_file = Tempfile.new('qunited_results')
-        results_file.close
+        send_to_formatter(:start)
 
         cmd = %{java -jar "#{js_jar}" -opt -1 "#{runner}" }
-        cmd << %{"#{QUnited::Driver::Base::SUPPORT_DIR}" "#{SUPPORT_DIR}" "#{results_file.path}"}
+        cmd << %{"#{QUnited::Driver::Base::SUPPORT_DIR}" "#{SUPPORT_DIR}"}
         cmd << " #{source_files_args} -- #{test_files_args}"
 
-        # Swallow stdout but allow stderr to get blasted out to console - if there are uncaught
-        # exceptions or anything else that goes wrong with the JavaScript interpreter the user
-        # will probably want to know but we are not particularly interested in it.
+        @results = []
+
         Open3.popen3(cmd) do |stdin, stdout, stderr|
-          stdout.each {||} # Ignore; this is just here to make sure we block
-                           # while waiting for tests to finish
+          results_collector = ResultsCollector.new(stdout)
+          results_collector.on_test_result do |result|
+            @results << result
+            method = result.passed? ? :test_passed : :test_failed
+            send_to_formatter(method, result)
+          end
+
+          results_collector.collect_results
+
+          # Allow stderr to get blasted out to console - if there are uncaught exceptions or
+          # anything else that goes wrong with Rhino the user will probably want to know.
           unless (err = stderr.read).strip.empty? then $stderr.puts(err) end
         end
 
-        @results = ::QUnited::Results.from_javascript_produced_json(IO.read(results_file.path))
+        send_to_formatter(:stop)
+        send_to_formatter(:summarize)
+
+        @results
       end
     end
   end

data/lib/qunited/driver/rhino/support/runner.js

@@ -1,22 +1,25 @@
-// Runs QUnit tests with Envjs and outputs test results as
-// an array of data serialized in YAML format.
-//
-// The first argument should be the lib directory containing common QUnited dependencies. The second
-// argument is the directory containing Rhino driver specific dependencies. The third argument
-// is the file to use for test results output. The next arguments are source JavaScript files to
-// test, until "--" is encountered. After the "--" the rest of the arguments are QUnit
-// test files.
-//
-// Example:
-//   java -jar js.jar -opt -1 runner.js commonlibdir libdir outfile.json source.js -- test1.js test2.js
-//                                            ^ our args start here
+/*
+ * Runs QUnit tests on Rhino with Envjs and outputs test result data, in JSON format, to stdout.
+ *
+ * Tokens are placed around each test result to allow them to be parsed individually. The tokens
+ * match the constants in QUnited::Driver::ResultsCollector.
+ *
+ * When run, the first argument is the lib directory containing common QUnited dependencies. The
+ * second argument is the directory containing Rhino driver specific dependencies. The next
+ * arguments are source JavaScript files to test, until "--" is encountered. After the "--" the
+ * rest of the arguments are QUnit test files.
+ *
+ * Example:
+ *   java -jar js.jar -opt -1 runner.js commonlibdir libdir source.js -- test1.js test2.js
+ *                                      ^ our args start here
+ */
 
 var QUnited = { sourceFiles: [], testFiles: [] };
 
+// Process command line arguments
 (function(args) {
 	var commonLibDir = args.shift(),
 			libDir = args.shift();
-	QUnited.outputFile = args.shift();
 
 	load(libDir + '/env.rhino.js');
 
@@ -90,21 +93,32 @@ QUnited.testFiles.forEach(function(file) {
 			QUnited.modulesMap[defaultModuleName] = module;
 		}
 
-		// Push our failed test data into the default module
-		module.tests.push({
-			name: "Nonexistent test",
+		// Put our failed test data into the default module
+		var failingTest = {
+			name: "Nonexistent tests",
 			assertion_data: [{
 				result: false, message: "Test file did not contain any tests (or there was an error loading it)"
 			}],
 			start: new Date(), duration: 0,
 			assertions: 1, failed: 1, total: 1,
 			file: file
-		});
+		}
+
+		module.tests.push(failingTest);
+		QUnited.testResultsPendingOutput.push(failingTest);
 	}
-});
 
-(function() {
-	var writer = new java.io.PrintWriter(QUnited.outputFile);
-	writer.write(QUnited.collectedTestResultsAsJson());
-	writer.close();
-})();
+	var results = [];
+	while (QUnited.testResultsPendingOutput.length > 0) {
+		results.push(QUnited.testResultsPendingOutput.shift());
+	}
+
+	var i, output;
+	for (i = 0; i < results.length; i++) {
+		// Beginning and end tokens should match the constants in QUnited::Driver::ResultsCollector
+		output = 'QUNITED_TEST_RESULT_START_TOKEN\n';
+		output += JSON.stringify(results[i], null, 1);
+		output += '\nQUNITED_TEST_RESULT_END_TOKEN';
+		java.lang.System.out.println(output);
+	}
+});