spassky 0.1.36 → 0.1.37

data/README.md

@@ -1,8 +1,42 @@
 Spassky
 =======
-A distributed web testing tool. We use it at the BBC for testing our web apps on a wide range of devices.
+Spassky is a test framework that allows us to automate the process of running JavaScript Unit tests on various browsers and devices.
+It currently supports running tests in QUnit, but in theory, will support any JavaScript framework that the browser supports.
+
+Spassky was a protagonist in the greatest Chess match of last century - which being played by an American and a Soviet Russian, was a Cold War metaphor. Spassky eventually resigned and Fischer won the championship.
+
+Architecture
+------------
+```
+                   +-----+
+                   | You |
+                   +--+--+
+                      |
+                      |
+                      v
+ +------------------------------------------+
+ |         Spassky Central Server           |
+ +------------------------------------------+
+           |           |          |
+           |           |          |
+           v           v          v
+       +-------+   +-------+   +-----+
+       |iPhone3|   |iPhone4|   |Nokia|
+       |-------|   |-------|   |-----|
+       |       |   |       |   | +++ |
+       |       |   |       |   | +++ |
+       |   +   |   |   +   |   | +++ |
+       +-------+   +-------+   +-----+
+```
+
+
+
+Developers push JS unit tests to the Spassky server through a command line interface.
+Multiple devices connected to the central Spassky server will poll the server for a suite of tests.
+The browser will be redirected to the test page, run the tests, and then are redirected to the idle loop.
+
+
 
-![Spassky](https://github.com/BBC/spassky/raw/master/spassky.jpg)
 
 Installation
 ------------
@@ -11,6 +45,11 @@ Installation
 gem install spassky
 ```
 
+
+
+![Spassky](https://github.com/BBC/spassky/raw/master/spassky.jpg)
+
+
 Usage
 -----
 
@@ -20,7 +59,7 @@ Start the server:
 spassky server 9191
 ```
 
-Connect some devices by browsing to http://localhost:9191/device/connect on the device. The device will stay in an idle meta refresh loop until it receives a test to run.
+Connect test devices by browsing to http://localhost:9191/device/connect on the device. The device will stay in an idle meta refresh loop until it receives a test to run.
 
 Check what devices are connected to the server:
 
@@ -52,7 +91,7 @@ We need to run automated tests on a wide range of web-enabled devices with very
 
 How it works
 ------------
-Physical devices act as test agents, connected permanently to a central server using meta refresh tags. Using a command-line utility, developers execute tests against those browsers by posting to the central server. The tests themselves are plain HTML pages, that are expected to call an assert URL (e.g. by embedding an image) within a time frame. That means even devices without any JavaScript can run automated tests of some kind.
+Physical devices act as test agents, connected permanently to a central server using meta refresh tags. Using a command-line utility, developers execute tests against those browsers by posting to the central server. The tests themselves are plain HTML pages, that are expected to call an assert URL (e.g. by embedding an image) within a time frame.
 
 Test structure
 --------------

data/lib/spassky/client/cli.rb

@@ -2,7 +2,7 @@ require 'spassky'
 require 'spassky/version'
 require 'spassky/server/app'
 require 'spassky/client/device_list_retriever'
-require 'spassky/client/test_runner'
+require 'spassky/client/test_suite_runner'
 require 'spassky/client/pusher'
 require 'spassky/client/directory_reader'
 require 'commandable'
@@ -18,8 +18,8 @@ module Spassky::Client
     def run(pattern, test, server = DEFAULT_SERVER, colour = false)
       writer = colour ? ColouredWriter : DefaultWriter
       pusher = Pusher.new(server)
-      test_runner = TestRunner.new(pusher, writer.new(STDOUT), DirectoryReader.new(pattern))
-      test_runner.run_tests(pattern, test)
+      test_suite_runner = TestSuiteRunner.new(pusher, writer.new(STDOUT), DirectoryReader.new(pattern))
+      test_suite_runner.run_test_suite(pattern, test)
     end
 
     command "list devices"

data/lib/spassky/client/directory_reader.rb

@@ -1,31 +1,35 @@
 module Spassky::Client
   class DirectoryReader
-
     def initialize(pattern)
       @pattern = pattern
     end
 
+    def read_files
+      if File.file? @pattern
+        read_file
+      elsif File.directory? @pattern
+        read_directory
+      end
+    end
+
+    private
+
     def read_directory
-      Dir.glob(@pattern + "/**/*").inject({}) do |hash, path|
+      files = {}
+      Dir.glob(@pattern + "/**/*").each do |path|
         if File.file? path
-          key = path.gsub(/^#{@pattern}\//, "")
-          hash[key] = File.read(path)
+          files[remove_pattern_from_file(path)] = File.read(path)
         end
-        hash
       end
+      files
+    end
 
+    def remove_pattern_from_file path
+      path.gsub(/^#{@pattern}\//, "")
     end
 
     def read_file
       { @pattern => File.read(@pattern) }
     end
-
-    def read_files
-      if File.file? @pattern
-        read_file
-      elsif File.directory? @pattern
-        read_directory
-      end
-    end
   end
 end

data/lib/spassky/client/pusher.rb

@@ -1,5 +1,5 @@
 require 'rest-client'
-require 'spassky/test_result'
+require 'spassky/test_suite_result'
 
 module Spassky::Client
   class Pusher
@@ -9,31 +9,43 @@ module Spassky::Client
     end
 
     def push(options)
-      location = post_test(options)
-      result = nil
-      begin
-        result = Spassky::TestResult.from_json(RestClient.get(location))
+      each_test_suite_result(post_test(options)) do |result|
         yield result
-        @sleeper.sleep 0.4 if result.status == 'in progress'
-      end while result.status == 'in progress'
+        @sleeper.sleep 0.4
+      end
     end
 
     private
 
+    def each_test_suite_result location
+      while (result = get_test_suite_result(location)).status == 'in progress'
+        yield result
+      end
+      yield get_test_suite_result(location)
+    end
+
+    def get_test_suite_result location
+      Spassky::TestSuiteResult.from_json(RestClient.get(location))
+    end
+
     def test_runs_url
       test_runs_url = @server_url.gsub(/\/$/, "") + "/test_runs"
     end
 
-    def post_test(options)
+    def post_test options
       RestClient.post(test_runs_url, options) do |response, request, result|
+        process_test_post_response response
         get_redirect_location response
       end
     end
 
-    def get_redirect_location response
+    def process_test_post_response response
       if response.code == 500
         raise response.to_str
       end
+    end
+
+    def get_redirect_location response
       location = response.headers[:location]
       raise "Expected #{test_runs_url} to respond with 302" unless location
       location

data/lib/spassky/client/test_suite_runner.rb

@@ -0,0 +1,62 @@
+require 'spassky/client/writer'
+
+module Spassky::Client
+  class TestSuiteRunner
+    def initialize(pusher, writer, directory_reader)
+      @pusher = pusher
+      @writer = writer
+      @directory_reader = directory_reader
+    end
+
+    def run_test_suite(pattern, test_name)
+      begin
+        @pusher.push(:name => test_name, :contents => @directory_reader.read_files.to_json) do |result|
+          handle_test_suite_result(result)
+        end
+      rescue => error
+        fail_with_error error
+      end
+    end
+
+    private
+
+    def fail_with_error error
+      @writer.write_failing(error.message)
+      Kernel.exit(1)
+    end
+
+    def handle_test_suite_result(test_suite_result)
+      write_in_progress_status test_suite_result
+      unless test_suite_result.status == "in progress"
+        write(test_suite_result.status, test_suite_result.summary)
+      end
+      @previous_test_suite_result = test_suite_result
+      write_exit_code(test_suite_result)
+    end
+
+    def write_in_progress_status test_suite_result
+      test_suite_result.completed_since(@previous_test_suite_result).each do |device_test_status|
+        write_completed_test_status device_test_status
+      end
+    end
+
+    def write_completed_test_status device_test_status
+      write(device_test_status.status, completion_status(device_test_status))
+      write(device_test_status.status, device_test_status.message)
+    end
+
+    def completion_status device_test_status
+      "#{device_test_status.status.upcase} #{device_test_status.test_name} on #{device_test_status.device_id}"
+    end
+
+    def write status, message
+      method = status == 'pass' ? :write_passing : :write_failing
+      @writer.send(method, message)
+    end
+
+    def write_exit_code(result)
+      Kernel.exit(1) if result.status == 'fail'
+      Kernel.exit(2) if result.status == 'timed out'
+    end
+  end
+end

data/lib/spassky/device_test_status.rb

@@ -16,5 +16,14 @@ module Spassky
     def completed?
       @status != "in progress"
     end
+
+    def self.from_hash hash
+      DeviceTestStatus.new({
+        :device_id => hash["device_id"],
+        :test_name => hash["test_name"],
+        :status    => hash["status"],
+        :message   => hash["message"]}
+      )
+    end
   end
 end

data/lib/spassky/server/app.rb

@@ -38,35 +38,22 @@ module Spassky::Server
       end
     end
 
-    def create_test_run
-      TestRun.create({
-        :name => params[:name],
-        :contents => JSON.parse(params[:contents]),
-        :devices => @device_list.recently_connected_devices
-      })
-    end
-
     post '/test_runs' do
       recently_connected_devices = @device_list.recently_connected_devices
-      if recently_connected_devices.size > 0
-        redirect "/test_runs/#{create_test_run.id}"
-      else
+      if recently_connected_devices.empty?
         halt 500, "There are no connected devices"
       end
+      redirect "/test_runs/#{create_test_run.id}"
     end
 
     get '/test_runs/:id' do
-      run = TestRun.find(params[:id])
+      run = test_run
       run.update_connected_devices(@device_list.recently_connected_devices)
       run.result.to_json
     end
 
     get '/test_runs/:id/run/:random/assert' do
-      TestRun.find(params[:id]).save_result_for_device(
-        :device_identifier => get_device_identifier,
-        :status => params[:status],
-        :message => params[:message]
-      )
+      save_test_result
     end
 
     get "/test_runs/:id/run/:random/*" do
@@ -76,11 +63,34 @@ module Spassky::Server
 
     private
 
+    def test_run
+      TestRun.find(params[:id])
+    end
+
+    def create_test_run
+      TestRun.create({
+        :name     => params[:name],
+        :contents => test_contents,
+        :devices  => @device_list.recently_connected_devices
+      })
+    end
+
+    def test_contents
+      JSON.parse(params[:contents])
+    end
+
     def get_test_file_contents test_run_id, file_name
-      test_run = TestRun.find(params[:id])
       HtmlTest.new(test_run.contents, idle_url, 1).get_file(file_name)
     end
 
+    def save_test_result
+      test_run.save_result_for_device(
+        :device_identifier  => get_device_identifier,
+        :status             => params[:status],
+        :message            => params[:message]
+      )
+    end
+
     def redirect_to_run_tests(test_run)
       redirect "/test_runs/#{test_run.id}/run/#{RandomStringGenerator.random_string}/#{test_run.name}"
     end

data/lib/spassky/server/device_database.rb

@@ -15,6 +15,7 @@ module Spassky::Server
     def initialize
       download_wurfl_file unless File.exist?(WURFL_FILE)
       @wurfl = WURFL.new(WURFL_FILE)
+      @stored_device_identifiers = {}
     end
 
     def download_wurfl_file
@@ -22,27 +23,33 @@ module Spassky::Server
       Kernel.puts("Downloading WURFL database")
       RestClient.proxy = ENV["http_proxy"] if ENV["http_proxy"]
       content = RestClient.get(LATEST)
-      File.open(WURFL_FILE, "w") do |file|
-        file.write(content)
-      end
+      save_wurfl_file content
     end
 
     def device_identifier user_agent
-      @stored_device_identifiers ||= {}
+      cached_device_identifier(user_agent) or uncached_device_identifier(user_agent)
+    end
+
+    private
 
-      unless @stored_device_identifiers[user_agent]
-        device = @wurfl[user_agent]
-        if device.nil?
-          @stored_device_identifiers[user_agent] = user_agent
-        else
-          @stored_device_identifiers[user_agent] = "#{device.model_name} (id = #{device.id}, mobile_browser = #{device.mobile_browser}, device_os_version = #{device.device_os_version})"
-        end
+    def save_wurfl_file content
+      File.open(WURFL_FILE, "w") do |file|
+        file.write(content)
       end
+    end
+
+    def cached_device_identifier user_agent
       @stored_device_identifiers[user_agent]
     end
 
-    def device user_agent
-      @wurfl[user_agent]
+    def uncached_device_identifier user_agent
+      @stored_device_identifiers ||= {}
+      if device = @wurfl[user_agent]
+        @stored_device_identifiers[user_agent] = "#{device.model_name} (id = #{device.id}, mobile_browser = #{device.mobile_browser}, device_os_version = #{device.device_os_version})"
+      else
+        @stored_device_identifiers[user_agent] = user_agent
+      end
+      cached_device_identifier user_agent
     end
   end
 

data/lib/spassky/server/device_list.rb

@@ -10,12 +10,18 @@ module Spassky::Server
 
     def recently_connected_devices
       @devices_and_time_last_connected.keys.select do |device_id|
-        Time.now.to_f - @devices_and_time_last_connected[device_id].to_f < 3
+        recent? @devices_and_time_last_connected[device_id]
       end
     end
 
     def clear
       @devices_and_time_last_connected = {}
     end
+
+    private
+
+    def recent? time
+      Time.now.to_f - time.to_f < 3
+    end
   end
 end

data/lib/spassky/server/test_run.rb

@@ -1,4 +1,4 @@
-require 'spassky/test_result'
+require 'spassky/test_suite_result'
 
 module Spassky::Server
   class TestRun
@@ -19,9 +19,7 @@ module Spassky::Server
     end
 
     def save_result_for_device(options)
-      unless ['pass', 'fail'].include? options[:status]
-        raise "#{options[:status]} is not a valid status"
-      end
+      validate_status options[:status]
       @status_by_device_id[options[:device_identifier]] = options[:status]
       @message_by_device_id[options[:device_identifier]] = options[:message]
     end
@@ -35,7 +33,7 @@ module Spassky::Server
     end
 
     def result
-      Spassky::TestResult.new(@status_by_device_id.map { |device_id, status|
+      Spassky::TestSuiteResult.new(@status_by_device_id.map { |device_id, status|
         Spassky::DeviceTestStatus.new(:device_id => device_id, :status => status, :message => @message_by_device_id[device_id], :test_name => @name)
       })
     end
@@ -55,12 +53,20 @@ module Spassky::Server
       test_runs.find { |test_run| test_run.run_by_device_id?(device_id) == false }
     end
 
+    private
+
+    def self.test_runs
+      @test_runs ||= []
+    end
+
     def self.delete_all
       @test_runs = []
     end
 
-    def self.test_runs
-      @test_runs ||= []
+    def validate_status status
+      unless ['pass', 'fail'].include?(status)
+        raise "#{status} is not a valid status"
+      end
     end
   end
 end

data/lib/spassky/test_suite_result.rb

@@ -0,0 +1,70 @@
+require 'spassky/test_suite_result_summariser'
+require 'spassky/device_test_status'
+require 'json'
+
+module Spassky
+  class TestSuiteResult
+    attr_reader :device_statuses
+
+    def initialize device_statuses
+      @device_statuses = device_statuses
+    end
+
+    def status
+      statuses = @device_statuses.map { |s| s.status }
+      return "in progress" if statuses.empty?
+      ["in progress", "fail", "timed out", "pass"].find {|s| statuses.include? s}
+    end
+
+    def completed_since(older_test_suite_result)
+      if older_test_suite_result.nil?
+        device_statuses.select { |s| s.completed? }
+      else
+        find_newly_completed_device_results(older_test_suite_result)
+      end
+    end
+
+    def summary
+      TestSuiteResultSummariser.new(@device_statuses).summary
+    end
+
+    def to_json
+      {
+        :status => "pass",
+        :device_statuses => @device_statuses.map do |status|
+          {
+            :device_id => status.device_id,
+            :test_name => status.test_name,
+            :status => status.status,
+            :message => status.message
+          }
+        end
+      }.to_json
+    end
+
+    def self.from_json json
+      device_test_statuses = JSON.parse(json)['device_statuses'].map do |device_test_status|
+        DeviceTestStatus.from_hash(device_test_status)
+      end
+      test_suite_result = TestSuiteResult.new(device_test_statuses)
+    end
+
+    private
+
+    def find_newly_completed_device_results(older_test_suite_result)
+      completed = []
+      before_and_after(older_test_suite_result) do |before, after|
+        if before.in_progress? && after.completed?
+          completed << after
+        end
+      end
+      completed
+    end
+
+    def before_and_after(older_test_suite_result)
+      device_statuses.each_with_index do |s, i|
+        yield older_test_suite_result.device_statuses[i], s
+      end
+    end
+  end
+end

data/lib/spassky/{test_result_summariser.rb → test_suite_result_summariser.rb}

@@ -1,17 +1,17 @@
 module Spassky
-  class TestResultSummariser
+  class TestSuiteResultSummariser
     def initialize device_statuses
       @device_statuses = device_statuses
     end
 
     def summary
       statuses = []
-      passes = status_count "pass"
-      fails = status_count "fail"
-      timeouts = status_count "timed out"
-      statuses << "#{passes} passed" if passes > 0
-      statuses << "#{fails} failed" if fails > 0
-      statuses << "#{timeouts} timed out" if timeouts > 0
+      {"pass" => "passed", "fail" => "failed", "timed out" => "timed out"}.each do |status, description|
+        status_count = status_count status
+        if status_count > 0
+          statuses << "#{status_count} #{description}"
+        end
+      end
       statuses.join ", "
     end
 

data/lib/spassky/version.rb

@@ -1,3 +1,3 @@
 module Spassky
-  VERSION = '0.1.36'
+  VERSION = '0.1.37'
 end

data/spassky.gemspec

@@ -31,6 +31,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'capybara'
   s.add_development_dependency 'aruba'
   s.add_development_dependency 'fakefs'
-  s.add_development_dependency 'ruby-debug19'
   s.add_development_dependency 'factory_girl'
 end

data/spec/spassky/client/cli_spec.rb

@@ -8,25 +8,25 @@ module Spassky::Client
     end
 
     let :runner do
-      mock(:runner, :run_tests => true)
+      mock(:runner, :run_test_suite => true)
     end
 
     before do
       Pusher.stub!(:new).and_return(pusher)
-      TestRunner.stub!(:new).and_return(runner)
+      TestSuiteRunner.stub!(:new).and_return(runner)
     end
 
     describe "spassky run" do
       context "with a server url" do
         it "creates a pusher" do
           Pusher.should_receive(:new).with("server_name").and_return(pusher)
-          TestRunner.should_receive(:new).with(pusher, anything(), anything()).and_return(runner)
+          TestSuiteRunner.should_receive(:new).with(pusher, anything(), anything()).and_return(runner)
           Cli.new.run "test_pattern", "test_name", "server_name"
         end
       end
 
       it "runs a test" do
-        runner.should_receive(:run_tests).with("test_pattern", "test_name")
+        runner.should_receive(:run_test_suite).with("test_pattern", "test_name")
         Cli.new.run "test_pattern", "test_name", "server_name"
       end
 
@@ -34,7 +34,7 @@ module Spassky::Client
         it "creates a test runner with a default writer" do
           default_writer = mock :default_writer
           DefaultWriter.should_receive(:new).with(STDOUT).and_return(default_writer)
-          TestRunner.should_receive(:new).with(anything(), default_writer, anything())
+          TestSuiteRunner.should_receive(:new).with(anything(), default_writer, anything())
           Cli.new.run "test_pattern", "test_name", "server_name"
         end
       end
@@ -43,8 +43,8 @@ module Spassky::Client
         it "creates a test runner with a colour writer" do
           coloured_writer = mock :coloured_writer
           ColouredWriter.should_receive(:new).with(STDOUT).and_return(coloured_writer)
-          TestRunner.should_receive(:new).with(anything(), coloured_writer, anything())
-          Cli.new.run "test_pattern", "test_name", "server_name", "--colour"
+          TestSuiteRunner.should_receive(:new).with(anything(), coloured_writer, anything())
+          Cli.new.run "test_pattern", "test_name", "server_name", true
         end
       end
     end

data/spec/spassky/client/pusher_spec.rb

@@ -15,15 +15,15 @@ module Spassky::Client
     end
 
     def in_progress_status
-      Spassky::TestResult.new([FactoryGirl.build(:device_test_status, :status => 'in progress')]).to_json
+      Spassky::TestSuiteResult.new([FactoryGirl.build(:device_test_status, :status => 'in progress')]).to_json
     end
 
     def passed_status
-      Spassky::TestResult.new([FactoryGirl.build(:device_test_status)]).to_json
+      Spassky::TestSuiteResult.new([FactoryGirl.build(:device_test_status)]).to_json
     end
 
     def failed_status
-      Spassky::TestResult.new([FactoryGirl.build(:device_test_status, :status => 'fail')]).to_json
+      Spassky::TestSuiteResult.new([FactoryGirl.build(:device_test_status, :status => 'fail')]).to_json
     end
 
     it "pushes a test to the server" do
@@ -78,7 +78,7 @@ module Spassky::Client
 
     it "sleeps while looping during get requests" do
       RestClient.stub!(:get).and_return(in_progress_status, in_progress_status, in_progress_status, passed_status)
-      @sleeper.should_receive(:sleep).with(0.4).exactly(3).times
+      @sleeper.should_receive(:sleep).with(0.4).exactly(4).times
       @pusher.push("test contents") { |result| }
     end
   end