chemistrykit 3.0.1 → 3.1.0

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+#3.1.0 (2013-07-07)
+Updated logging mechanism for concurrent tests and to output junit xml.
+
+- Bumped version to 3.1.0 to prepare for release.
+- cleaned out some random debug code
+- fixed code quality issues and removed some debugging information
+- updated concurency to use uuid for partial logs, cleaned up tests
+- fixed other features so they would pass
+- fixed more code quality issues
+- added facility that would create partial test output for concurrent tests
+- ensured all log files could be directed to the same directory
+- fixed code quality issue and added test to ensure html report output would work
+- added in configurable junit export with stdout
+- added logging feature and tests, and expanded the configuration for logging
+
 #3.0.1 (2013-07-05)
 Made the base url available in the env for formula usage
 

data/chemistrykit.gemspec

@@ -2,13 +2,13 @@
 
 Gem::Specification.new do |s|
   s.name          = "chemistrykit"
-  s.version       = "3.0.1"
+  s.version       = "3.1.0"
   s.platform      = Gem::Platform::RUBY
   s.authors       = ["Dave Haeffner", "Jason Fox"]
   s.email         = ["dave@arrgyle.com", "jason@arrgyle.com"]
   s.homepage      = "https://github.com/arrgyle/chemistrykit"
   s.summary       = "A simple and opinionated web testing framework for Selenium that follows convention over configuration."
-  s.description   = "Made the base url available in the env for formula usage"
+  s.description   = "Updated logging mechanism for concurrent tests and to output junit xml."
   s.license       = 'MIT'
 
   s.files         = `git ls-files`.split($/)
@@ -20,15 +20,15 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>=1.9'
 
   s.add_dependency "thor", "~> 0.17.0"
-  s.add_dependency "rspec", "~> 2.12.0"
+  s.add_dependency "rspec", "~> 2.13.0"
+  s.add_dependency "yarjuf", "~> 1.0.5"
   s.add_dependency "selenium-webdriver", "~> 2.29.0"
-  s.add_dependency "ci_reporter", "~> 1.8.3"
   s.add_dependency "rest-client", "~> 1.6.7"
   s.add_dependency "selenium-connect", "~> 2.1.1"
   s.add_dependency "parallel_tests", "~> 0.15.0"
   s.add_dependency "parallel", "~> 0.7.0"
 
-  s.add_development_dependency "rspec", "~> 2.12.0"
+  s.add_development_dependency "rspec", "~> 2.13.0"
   s.add_development_dependency "aruba", "~> 0.5.1"
   s.add_development_dependency "cucumber", "~> 1.2.1"
   s.add_development_dependency "rake", "~> 10.0.3"

data/features/brew.feature

@@ -29,7 +29,6 @@ Feature: Brewing a ChemistryKit project
         end
       end
       """
-
   Scenario: Localhost
     Given a file named "config.yaml" with:
       """
@@ -40,7 +39,7 @@ Feature: Brewing a ChemistryKit project
     When I run `ckit brew`
     Then the stdout should contain "1 example, 0 failures"
     And the following files should exist:
-      | evidence/SPEC-Bookie.xml  |
+      | evidence/results_junit.xml  |
       | evidence/server.log       |
 
 

data/features/concurrency.feature

@@ -28,6 +28,7 @@ Feature: Support for concurency
       end
     end
     """
+
   Scenario: I can run the tests in parallel
     When I overwrite config.yaml with:
       """
@@ -43,6 +44,8 @@ Feature: Support for concurency
     When I run `ckit brew`
     Then the stdout should contain "4 processes for 2 beakers"
     And the stdout should contain "3 examples, 0 failures"
+    And the file "evidence/results_junit.xml" should not exist
+    And there should be "2" unique results files in the "evidence" directory
 
   Scenario: I can run a specific beaker in parallel
     When I overwrite config.yaml with:

data/features/logging.feature

@@ -0,0 +1,94 @@
+#logging.feature
+Feature: Log handling
+  In order to examine the results of my test suite
+  As a test harness user
+  I want to see all of the logs in a central location
+
+  Background: Setup the project
+    Given I run `ckit new logging-test`
+    And I cd to "logging-test"
+    And a file named "formulas/bookie.rb" with:
+      """
+      module Formulas
+        class Bookie < Formula
+          def open(url)
+            @driver.get url
+          end
+        end
+      end
+      """
+    And a file named "beakers/first_beaker.rb" with:
+      """
+      describe "First", :depth => 'shallow' do
+        let(:book) { Formulas::Bookie.new(@driver) }
+
+        it "loads an external web page" do
+          book.open "http://www.google.com"
+        end
+      end
+      """
+    And a file named "beakers/second_beaker.rb" with:
+      """
+      describe "Second", :depth => 'shallow' do
+        let(:book) { Formulas::Bookie.new(@driver) }
+
+        it "loads an external web page" do
+          book.open "http://www.google.com"
+        end
+      end
+      """
+
+  Scenario: I can output junit xml results by default
+    When I run `ckit brew`
+    Then the stdout should contain "2 examples, 0 failures"
+    And the following files should exist:
+      | evidence/results_junit.xml  |
+
+  Scenario: I can output custom junit xml results
+    Given a file named "config.yaml" with:
+      """
+      log:
+          path: 'my_evidence'
+          results_file: 'my_results.xml'
+          format: 'junit'
+      selenium_connect:
+          log: 'evidence'
+          host: 'localhost'
+      """
+    When I run `ckit brew`
+    Then the stdout should contain "2 examples, 0 failures"
+    And the following files should exist:
+      | my_evidence/my_results.xml  |
+
+    Scenario: I can output html results
+      Given a file named "config.yaml" with:
+        """
+        log:
+            results_file: 'report.html'
+            format: 'html'
+        """
+      And I run `ckit brew`
+      Then the stdout should contain "2 examples, 0 failures"
+      And the following files should exist:
+        | evidence/report.html  |
+
+    Scenario: I define one log location to rule them all
+      Given a file named "config.yaml" with:
+        """
+          log:
+              path: 'my_evidence'
+          selenium_connect:
+              host: 'localhost'
+        """
+      When I run `ckit brew`
+        Then the stdout should contain "2 examples, 0 failures"
+        And the following files should exist:
+          | my_evidence/results_junit.xml |
+          | my_evidence/server.log        |
+
+    Scenario: I can runtime override the results output file name
+      When I run `ckit brew --results_file results_junit_01.xml`
+      Then the stdout should contain "2 examples, 0 failures"
+      And the following files should exist:
+        | evidence/results_junit_01.xml  |
+

data/features/new.feature

@@ -15,5 +15,4 @@ Feature: ckit new
       | evidence                |
     And the following files should exist:
       | config.yaml             |
-      | .rspec                  |
       | formulas/lib/formula.rb |

data/features/step_definitions/steps.rb

@@ -10,3 +10,12 @@ end
 Then(/^the exit code should be (\d+)$/) do |exit_status|
   @last_exit_status.should == exit_status.to_i
 end
+
+Then(/^there should be "(.*?)" unique results files in the "(.*?)" directory$/) do |number_files, logs_path|
+  files = Dir.glob(File.join(current_dir, logs_path, '*.xml'))
+  count = 0
+  files.each do |file|
+    count += 1 if file =~ /parallel_part_\w{8}-(\w{4}-){3}\w{12}\.xml/
+  end
+  count.should == number_files.to_i
+end

data/lib/chemistrykit/cli/cli.rb

@@ -2,7 +2,7 @@
 
 require 'thor'
 require 'rspec'
-require 'ci/reporter/rake/rspec_loader'
+require 'yarjuf'
 require 'chemistrykit/cli/new'
 require 'chemistrykit/cli/formula'
 require 'chemistrykit/cli/beaker'
@@ -11,6 +11,8 @@ require 'chemistrykit/catalyst'
 require 'chemistrykit/formula/base'
 require 'selenium-connect'
 require 'chemistrykit/configuration'
+require 'parallel_tests'
+require 'chemistrykit/parallel_tests_mods'
 
 module ChemistryKit
   module CLI
@@ -39,14 +41,17 @@ module ChemistryKit
       method_option :beakers, type: :array
       # This is set if the thread is being run in parallel so as not to trigger recursive concurency
       method_option :parallel, default: false
+      method_option :results_file, default: false
 
       def brew
         config = load_config options['config']
         # TODO perhaps the params should be rolled into the available
         # config object injected into the system?
         pass_params if options['params']
-        turn_stdout_stderr_on_off
-        set_logs_dir
+
+        # replace certain config values with run time flags as needed
+        config = override_configs options, config
+
         load_page_objects
         setup_tags
         # configure rspec
@@ -59,10 +64,19 @@ module ChemistryKit
         else
           run_rspec beakers
         end
+
       end
 
       protected
 
+      def override_configs(options, config)
+        # TODO expand this to allow for more overrides as needed
+        if options['results_file']
+          config.log.results_file = options['results_file']
+        end
+        config
+      end
+
       def pass_params
         options['params'].each_pair do |key, value|
           ENV[key] = value
@@ -74,14 +88,6 @@ module ChemistryKit
         loader.get_formulas(File.join(Dir.getwd, 'formulas')).each { |file| require file }
       end
 
-      def set_logs_dir
-        ENV['CI_REPORTS'] = File.join(Dir.getwd, 'evidence')
-      end
-
-      def turn_stdout_stderr_on_off
-        ENV['CI_CAPTURE'] = 'on'
-      end
-
       def load_config(file_name)
         config_file = File.join(Dir.getwd, file_name)
         ChemistryKit::Configuration.initialize_with_yaml config_file
@@ -128,12 +134,16 @@ module ChemistryKit
           c.order = 'random'
           c.default_path = 'beakers'
           c.pattern = '**/*_beaker.rb'
+          c.output_stream = $stdout
+          c.add_formatter 'progress'
+          if config.concurrency == 1 || options['parallel']
+            c.add_formatter(config.log.format, File.join(Dir.getwd, config.log.path, config.log.results_file))
+          end
         end
       end
 
       def run_in_parallel(beakers, concurrency)
-        require 'parallel_tests'
-        require 'chemistrykit/parallel_tests_mods'
+
         ParallelTests::CLI.new.run(%w(--type rspec) + ['-n', concurrency.to_s] + %w(-o --beakers=) + beakers)
       end
 

data/lib/chemistrykit/configuration.rb

@@ -1,22 +1,46 @@
 # Encoding: utf-8
 
 require 'yaml'
+require 'ostruct'
 
 module ChemistryKit
   # Default configuration class
   class Configuration
 
-    attr_accessor :concurrency,
-                  :base_url,
-                  :selenium_connect
+    attr_accessor :base_url,
+                  :concurrency
+
+    attr_reader   :log
+
+    attr_writer   :selenium_connect
 
     def initialize(hash)
       # set defaults
       @concurrency = 1
+      @selenium_connect = {}
+      @log = OpenStruct.new
+      @log.path = 'evidence'
+      @log.results_file = 'results_junit.xml'
+      @log.format = 'JUnit'
 
       # overide with argument
       populate_with_hash hash
-      validate_config
+    end
+
+    def log=(log_hash)
+      log_hash.each do |key, value|
+        value = 'JUnit' if key == :format && value =~ /junit/i
+        @log.send("#{key}=", value) unless value.nil?
+      end
+    end
+
+    def selenium_connect
+      # return the default log unless the sc log is set
+      if @selenium_connect[:log].nil?
+        @selenium_connect[:log] = @log.path
+        return @selenium_connect
+      end
+      @selenium_connect
     end
 
     def self.initialize_with_yaml(file)
@@ -35,12 +59,6 @@ module ChemistryKit
         end
       end
 
-      def validate_config
-        if @selenium_connect && @selenium_connect[:host] != 'saucelabs' && @concurrency > 1
-          raise ArgumentError.new 'Concurrency is only supported for the host: "saucelabs"!'
-        end
-      end
-
       # private static to symbolize recursivly a loaded yaml
       def self.symbolize_keys(hash)
         hash.reduce({}) do |result, (key, value)|

data/lib/chemistrykit/parallel_tests_mods.rb

@@ -2,12 +2,12 @@
 
 require 'parallel_tests/rspec/runner'
 require 'parallel_tests/test/runner'
+require 'securerandom'
 
 module ParallelTests
   module RSpec
     # Monkey Patching the ParallelTest RSpec Runner class to work with CKit's config and binary
     class Runner < ParallelTests::Test::Runner
-
       class << self
 
         def run_tests(test_files, process_number, num_processes, options)
@@ -20,7 +20,6 @@ module ParallelTests
 
           cmd = [exe, options[:test_options]].compact.join(' ')
           cmd << test_files.join(' ')
-          puts cmd
 
         # This concatenates the command into `bundle exec ckit brew --beakers=beaker1 beaker2 beaker3 etc`
         # Which enables each test group to be run in its own command
@@ -33,7 +32,9 @@ module ParallelTests
         end
 
         def determine_executable
-          'bundle exec ckit brew --parallel'
+          uuid = SecureRandom.uuid
+          file_name = "parallel_part_#{uuid}.xml"
+          "bundle exec ckit brew --parallel --results_file #{file_name}"
         end
 
         def test_file_name

data/lib/templates/chemistrykit/config.yaml.tt

@@ -1,7 +1,11 @@
 # Default Chemisty Kit Configuration File
 
 base_url: http://google.com
-concurrency: 2
+concurrency: 1
+log:
+    path: #you may override the default log path 'evidence'
+    results_file: #you may override the default file name 'results_junit.xml'
+    format: #you may override the default format 'junit'
 selenium_connect:
     #Setup & Debugging
     jar:

data/spec/chemistrykit/configuration_spec.rb

@@ -7,25 +7,38 @@ describe ChemistryKit::Configuration do
   VALID_BASE_URL = 'http://google.com'
   VALID_CONCURRENCY = 1
   VALID_CONFIG_FILE = 'config.yaml'
+  VALID_LOG_PATH = 'evidence'
+  VALID_JUNIT = 'results_junit.xml'
+  VALID_FORMAT_JUNIT = 'junit'
+  VALID_JUNIT_FORMAT_OUT = 'JUnit'
 
   before(:each) do
     @valid_selenium_connect_hash = { log: 'evidence', host: 'localhost' }
+    @valid_log_hash = { path: VALID_LOG_PATH, results_file: VALID_JUNIT, format: VALID_FORMAT_JUNIT }
     @valid_config_hash = {
       base_url: VALID_BASE_URL,
       concurrency: VALID_CONCURRENCY,
-      selenium_connect: @valid_selenium_connect_hash
+      selenium_connect: @valid_selenium_connect_hash,
+      log: @valid_log_hash
     }
   end
 
   def validate_config(config)
     config.concurrency.should eq VALID_CONCURRENCY
     config.base_url.should eq VALID_BASE_URL
+    config.log.path.should eq VALID_LOG_PATH
+    config.log.results_file.should eq VALID_JUNIT
+    config.log.format.should eq VALID_JUNIT_FORMAT_OUT
     config.selenium_connect.should eq @valid_selenium_connect_hash
   end
 
   it 'should initialize with sane defaults' do
     config = ChemistryKit::Configuration.new({})
-    config.concurrency.should eq 1
+    config.concurrency.should eq VALID_CONCURRENCY
+    config.log.path.should eq VALID_LOG_PATH
+    config.log.results_file.should eq VALID_JUNIT
+    config.log.format.should eq VALID_JUNIT_FORMAT_OUT
+    config.selenium_connect.should eq({ log: VALID_LOG_PATH })
   end
 
   it 'should initialize with a hash of configurations' do
@@ -43,14 +56,18 @@ describe ChemistryKit::Configuration do
     end.to raise_error ArgumentError, 'The config key: "bad" is unknown!'
   end
 
-  it 'should throw an error if concurrency is configured and the host is not saucelabs' do
-    expect do
-      config_hash = {
-      base_url: VALID_BASE_URL,
-      concurrency: 2,
-      selenium_connect: @valid_selenium_connect_hash
-      }
-      ChemistryKit::Configuration.new(config_hash)
-    end.to raise_error ArgumentError, 'Concurrency is only supported for the host: "saucelabs"!'
+  it 'should correct the format to JUnit' do
+    config = ChemistryKit::Configuration.new(@valid_config_hash)
+    config.log.format.should eq 'JUnit'
+  end
+
+  it 'selenium_connect log should default to the main log' do
+    config =  ChemistryKit::Configuration.new({ log: { path: 'main-path' } })
+    config.selenium_connect.should eq({ log: 'main-path' })
+  end
+
+  it 'mainlog should not overide selenium_connect log' do
+    config =  ChemistryKit::Configuration.new({ log: { path: 'main-path' }, selenium_connect: { log: 'sc-log' } })
+    config.selenium_connect.should eq({ log: 'sc-log' })
   end
 end

data/spec/support/config.yaml

@@ -2,6 +2,10 @@
 
 base_url: http://google.com
 concurrency: 1
+log:
+    path: 'evidence'
+    results_file: 'results_junit.xml'
+    format: 'junit'
 selenium_connect:
     log: 'evidence'
     host: 'localhost'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chemistrykit
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-05 00:00:00.000000000 Z
+date: 2013-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -35,7 +35,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: 2.13.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -43,15 +43,15 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: 2.13.0
 - !ruby/object:Gem::Dependency
-  name: selenium-webdriver
+  name: yarjuf
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.29.0
+        version: 1.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -59,15 +59,15 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.29.0
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
-  name: ci_reporter
+  name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 2.29.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -75,7 +75,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 2.29.0
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
@@ -147,7 +147,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: 2.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -155,7 +155,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: 2.13.0
 - !ruby/object:Gem::Dependency
   name: aruba
   requirement: !ruby/object:Gem::Requirement
@@ -220,7 +220,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.9.0
-description: Made the base url available in the env for formula usage
+description: Updated logging mechanism for concurrent tests and to output junit xml.
 email:
 - dave@arrgyle.com
 - jason@arrgyle.com
@@ -250,6 +250,7 @@ files:
 - features/exit_status.feature
 - features/global-config.feature
 - features/load_page_objects.feature
+- features/logging.feature
 - features/multi-config.feature
 - features/new.feature
 - features/step_definitions/steps.rb
@@ -266,7 +267,6 @@ files:
 - lib/chemistrykit/parallel_tests_mods.rb
 - lib/templates/beaker.tt
 - lib/templates/beaker_with_formula.tt
-- lib/templates/chemistrykit/.rspec
 - lib/templates/chemistrykit/beakers/.gitkeep
 - lib/templates/chemistrykit/config.yaml.tt
 - lib/templates/chemistrykit/evidence/.gitkeep
@@ -301,7 +301,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4251143556691358263
+      hash: 3010255625255682660
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25
@@ -316,6 +316,7 @@ test_files:
 - features/exit_status.feature
 - features/global-config.feature
 - features/load_page_objects.feature
+- features/logging.feature
 - features/multi-config.feature
 - features/new.feature
 - features/step_definitions/steps.rb

data/lib/templates/chemistrykit/.rspec

@@ -1 +0,0 @@
---format CI::Reporter::RSpec