buildr 1.2.10 → 1.3.0

data/lib/buildr/core/test.rb

@@ -0,0 +1,690 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership.  The ASF
+# licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+
+require 'buildr/core/project'
+require 'buildr/core/build'
+require 'buildr/core/compile'
+
+
+module Buildr
+
+  # The underlying test framework used by TestTask.
+  # To add a new test framework, extend TestFramework::Base and add your framework using:
+  #   Buildr::TestFramework << MyFramework
+  class TestFramework
+
+    class << self
+
+      # Returns true if the specified test framework exists.
+      def has?(name)
+        frameworks.any? { |framework| framework.to_sym == name.to_sym }
+      end
+
+      # Select a test framework by its name.
+      def select(name)
+        frameworks.detect { |framework| framework.to_sym == name.to_sym }
+      end
+
+      # Identify which test framework applies for this project.
+      def select_from(project)
+        # Look for a suitable test framework based on the compiled language,
+        # which may return multiple candidates, e.g. JUnit and TestNG for Java.
+        # Pick the one used in the parent project, if not, whichever comes first.
+        candidates = frameworks.select { |framework| framework.applies_to?(project) }
+        parent = project.parent
+        parent && candidates.detect { |framework| framework.to_sym == parent.test.framework } || candidates.first
+      end
+
+      # Adds a test framework to the list of supported frameworks.
+      #   
+      # For example:
+      #   Buildr::TestFramework << Buildr::JUnit
+      def add(framework)
+        @frameworks ||= []
+        @frameworks |= [framework]
+      end
+      alias :<< :add
+
+      # Returns a list of available test frameworks.
+      def frameworks
+        @frameworks ||= []
+      end
+
+    end
+
+    # Base class for all test frameworks, with common functionality.  Extend and over-ride as you see fit
+    # (see JUnit as an example).
+    class Base
+
+      class << self
+
+        # The framework's identifier (e.g. :junit).  Inferred from the class name.
+        def to_sym
+          @symbol ||= name.split('::').last.downcase.to_sym
+        end
+
+        # Returns true if this framework applies to the current project.  For example, JUnit returns
+        # true if the tests are written in Java.
+        def applies_to?(project)
+          raise 'Not implemented'
+        end
+
+        # Returns a list of dependencies for this framework.  Defaults to obtaining a list of
+        # artifact specifications from the REQUIRES constant.
+        def dependencies
+          @dependencies ||= FileList[*(const_get('REQUIRES') rescue [])]
+        end
+
+      end
+
+      # Construct a new test framework with the specified options.  Note that options may
+      # change before the framework is run.
+      def initialize(test_task, options)
+        @options = options
+        @task = test_task
+      end
+
+      # Options for this test framework.
+      attr_reader :options
+      # The test task we belong to
+      attr_reader :task
+
+      # Returns a list of dependenices for this framework.  Defaults to calling the #dependencies
+      # method on the class.
+      def dependencies
+        self.class.dependencies
+      end
+
+      # TestTask calls this method to return a list of test names that can be run in this project.
+      # It then applies the include/exclude patterns to arrive at the list of tests that will be
+      # run, and call the #run method with that list.
+      #
+      # This method should return a list suitable for using with the #run method, but also suitable
+      # for the user to manage.  For example, JUnit locates all the tests in the test.compile.target
+      # directory, and returns the class names, which are easier to work with than file names.
+      def tests(dependencies)
+        raise 'Not implemented'
+      end
+
+      # TestTask calls this method to run the named (and only those) tests.  This method returns
+      # the list of tests that ran successfully.
+      def run(tests, dependencies)
+        raise 'Not implemented'
+      end
+
+    end
+  
+  end
+
+
+  # The test task controls the entire test lifecycle.
+  #
+  # You can use the test task in three ways. You can access and configure specific test tasks,
+  # e.g. enhance the #compile task, or run code during #setup/#teardown.
+  #
+  # You can use convenient methods that handle the most common settings. For example,
+  # add dependencies using #with, or include only specific tests using #include.
+  #
+  # You can also enhance this task directly. This task will first execute the #compile task, followed
+  # by the #setup task, run the unit tests, any other enhancements, and end by executing #teardown.
+  #
+  # The test framework is determined based on the available test files, for example, if the test
+  # cases are written in Java, then JUnit is selected as the test framework.  You can also select
+  # a specific test framework, for example, to use TestNG instead of JUnit:
+  #   test.using :testng
+  class TestTask < Rake::Task
+
+    class << self
+
+      # Used by the local test and integration tasks to
+      # a) Find the local project(s),
+      # b) Find all its sub-projects and narrow down to those that have either unit or integration tests,
+      # c) Run all the (either unit or integration) tests, and
+      # d) Ignore failure if necessary.
+      def run_local_tests(integration) #:nodoc:
+        Project.local_projects do |project|
+          # !(foo ^ bar) tests for equality and accepts nil as false (and select is less obfuscated than reject on ^).
+          projects = ([project] + project.projects).select { |project| !(project.test.options[:integration] ^ integration) }
+          projects.each do |project|
+            puts "Testing #{project.name}" if verbose
+            begin
+              project.test.invoke
+            rescue
+              raise unless Buildr.options.test == :all
+            end
+          end
+        end
+      end
+
+      # Used by the test/integration rule to only run tests that match the specified names.
+      def only_run(tests) #:nodoc:
+        tests = tests.map { |name| name =~ /\*/ ? name : "*#{name}*" }
+        # Since the tests may reside in a sub-project, we need to set the include/exclude pattern on
+        # all sub-projects, but only invoke test on the local project.
+        Project.projects.each { |project| project.test.send :only_run, tests }
+      end
+    end
+
+    # Default options already set on each test task.
+    DEFAULT_OPTIONS = { :fail_on_failure=>true, :fork=>:once, :properties=>{}, :environment=>{} }
+
+    def initialize(*args) #:nodoc:
+      super
+      @dependencies = FileList[]
+      @include = []
+      @exclude = []
+      parent_task = Project.parent_task(name)
+      if parent_task.respond_to?(:options)
+        @options = OpenObject.new { |hash, key| parent_task.options[key] }
+      else
+        @options = OpenObject.new(DEFAULT_OPTIONS)
+      end
+      enhance do
+        run_tests if framework
+      end
+    end
+
+    # The dependencies used for running the tests. Includes the compiled files (compile.target)
+    # and their dependencies. Will also include anything you pass to #with, shared between the
+    # testing compile and run dependencies.
+    attr_reader :dependencies
+
+    # *Deprecated*: Use dependencies instead.
+    def classpath
+      Buildr.application.deprecated 'Use dependencies instead.'
+      dependencies
+    end
+
+    # *Deprecated*: Use dependencies= instead.
+    def classpath=(artifacts)
+      Buildr.application.deprecated 'Use dependencies= instead.'
+      self.dependencies = artifacts
+    end
+
+    def execute(args) #:nodoc:
+      setup.invoke
+      begin
+        super
+      rescue RuntimeError
+        raise if options[:fail_on_failure]
+      ensure
+        teardown.invoke
+      end
+    end
+
+    # :call-seq:
+    #   compile(*sources) => CompileTask
+    #   compile(*sources) { |task| .. } => CompileTask
+    #
+    # The compile task is similar to the Project's compile task. However, it compiles all
+    # files found in the src/test/{source} directory into the target/test/{code} directory.
+    # This task is executed by the test task before running any tests.
+    #
+    # Once the project definition is complete, all dependencies from the regular
+    # compile task are copied over, so you only need to specify dependencies
+    # specific to your tests. You can do so by calling #with on the test task.
+    # The dependencies used here are also copied over to the junit task.
+    def compile(*sources, &block)
+      @project.task('test:compile').from(sources).enhance &block
+    end
+ 
+    # :call-seq:
+    #   resources(*prereqs) => ResourcesTask
+    #   resources(*prereqs) { |task| .. } => ResourcesTask
+    #
+    # Executes by the #compile task to copy resource files over. See Project#resources.
+    def resources(*prereqs, &block)
+      @project.task('test:resources').enhance prereqs, &block
+    end
+
+    # :call-seq:
+    #   setup(*prereqs) => task
+    #   setup(*prereqs) { |task| .. } => task
+    #
+    # Returns the setup task. The setup task is executed at the beginning of the test task,
+    # after compiling the test files.
+    def setup(*prereqs, &block)
+      @project.task('test:setup').enhance prereqs, &block
+    end
+
+    # :call-seq:
+    #   teardown(*prereqs) => task
+    #   teardown(*prereqs) { |task| .. } => task
+    #
+    # Returns the teardown task. The teardown task is executed at the end of the test task.
+    def teardown(*prereqs, &block)
+      @project.task('test:teardown').enhance prereqs, &block
+    end
+
+    # :call-seq:
+    #   with(*specs) => self
+    #
+    # Specify artifacts (specs, tasks, files, etc) to include in the depdenenciest list
+    # when compiling and running tests.
+    def with(*artifacts)
+      @dependencies |= Buildr.artifacts(artifacts.flatten).uniq
+      compile.with artifacts
+      self
+    end
+
+    # Returns various test options.
+    attr_reader :options
+
+    # :call-seq:
+    #   using(options) => self
+    #
+    # Sets various test options from a hash and returns self.  For example:
+    #   test.using :fork=>:each, :properties=>{ 'url'=>'http://localhost:8080' }
+    #
+    # Can also be used to select the test framework, or to run these tests as
+    # integration tests.  For example:
+    #   test.using :testng
+    #   test.using :integration
+    #
+    # The :fail_on_failure option specifies whether the task should fail if
+    # any of the tests fail (default), or should report the failures but continue
+    # running the build (when set to false).
+    #
+    # All other options depend on the capability of the test framework.  These options
+    # should be used the same way across all frameworks that support them:
+    # * :fork -- Fork once for each project (:once, default), for each test in each
+    #     project (:each), or don't fork at all (false).
+    # * :properties -- Properties pass to the test, e.g. in Java as system properties.
+    # * :environment -- Environment variables.  This hash is made available in the
+    #     form of environment variables.
+    def using(*args)
+      args.pop.each { |key, value| options[key.to_sym] = value } if Hash === args.last
+      args.each do |name|
+        if TestFramework.has?(name)
+          self.framework = name
+        elsif name == :integration
+          options[:integration] = true
+        else
+          Buildr.application.deprecated "Please replace with using(:#{name}=>true)"
+          options[name.to_sym] = true
+        end
+      end 
+      self
+    end
+
+    # :call-seq:
+    #   include(*names) => self
+    #
+    # Include only the specified tests. Unless specified, the default is to include
+    # all tests identified by the test framework. This method accepts multiple arguments
+    # and returns self.
+    #
+    # Tests are specified by their full name, but you can use glob patterns to select
+    # multiple tests, for example:
+    #   test.include 'com.example.FirstTest'  # FirstTest only
+    #   test.include 'com.example.*'          # All tests under com/example
+    #   test.include 'com.example.Module*'    # All tests starting with Module
+    #   test.include '*.{First,Second}Test'   # FirstTest, SecondTest
+    def include(*names)
+      @include += names
+      self
+    end
+
+    # :call-seq:
+    #   exclude(*names) => self
+    #
+    # Exclude the specified tests. This method accepts multiple arguments and returns self.
+    # See #include for the type of arguments you can use.
+    def exclude(*names)
+      @exclude += names
+      self
+    end
+
+    # *Deprecated*: Use tests instead.
+    def classes
+      Buildr.application.deprecated 'Call tests instead of classes'
+      tests
+    end
+
+    # After running the task, returns all tests selected to run, based on availability and include/exclude pattern.
+    attr_reader :tests
+    # After running the task, returns all the tests that failed, empty array if all tests passed.
+    attr_reader :failed_tests
+    # After running the task, returns all the tests that passed, empty array if no tests passed.
+    attr_reader :passed_tests
+
+    # :call-seq:
+    #   framework => symbol
+    #
+    # Returns the test framework, e.g. :junit, :testng.
+    def framework
+      unless @framework
+        # Start with all frameworks that apply (e.g. JUnit and TestNG for Java),
+        # and pick the first (default) one, unless already specified in parent project.
+        candidates = TestFramework.frameworks.select { |cls| cls.applies_to?(@project) }
+        candidate = @project.parent && candidates.detect { |framework| framework.to_sym == @project.parent.test.framework } ||
+          candidates.first
+        self.framework = candidate if candidate
+      end
+      @framework && @framework.class.to_sym
+    end
+
+    # :call-seq:
+    #   report_to => file
+    #
+    # Test frameworks that can produce reports, will write them to this directory.
+    #
+    # This is framework dependent, so unless you use the default test framework, call this method
+    # after setting the test framework.
+    def report_to
+      @report_to ||= file(@project.path_to(:reports, framework)=>self)
+    end
+
+    # The project this task belongs to.
+    attr_reader :project
+
+  protected
+
+    def associate_with(project)
+      @project = project
+    end
+
+    def framework=(name)
+      cls = TestFramework.select(name) or raise ArgumentError, "No #{name} test framework available. Did you install it?"
+      #cls.inherit_options.reject { |name| options.has_key?(name) }.
+      #  each { |name| options[name] = @parent_task.options[name] } if @parent_task.respond_to?(:options)
+      @framework = cls.new(self, options)
+      # Test framework dependency.
+      with @framework.dependencies
+    end
+
+    # :call-seq:
+    #   include?(name) => boolean
+    #
+    # Returns true if the specified test name matches the inclusion/exclusion pattern. Used to determine
+    # which tests to execute.
+    def include?(name)
+      (@include.empty? || @include.any? { |pattern| File.fnmatch(pattern, name) }) &&
+        !@exclude.any? { |pattern| File.fnmatch(pattern, name) }
+    end
+
+    # Runs the tests using the selected test framework.
+    def run_tests
+      dependencies = Buildr.artifacts(self.dependencies).map(&:to_s).uniq
+      rm_rf report_to.to_s
+      @tests = @framework.tests(dependencies).select { |test| include?(test) }.sort
+      if @tests.empty?
+        @passed_tests, @failed_tests = [], []
+      else
+        puts "Running tests in #{@project.name}" if verbose
+        @passed_tests = @framework.run(@tests, dependencies)
+        @failed_tests = @tests - @passed_tests
+        unless @failed_tests.empty?
+          warn "The following tests failed:\n#{@failed_tests.join("\n")}" if verbose
+          fail 'Tests failed!'
+        end
+      end
+    end
+
+    # Limit running tests to specific list.
+    def only_run(tests)
+      @include = Array(tests)
+      @exclude.clear
+    end
+
+    def invoke_prerequisites(args, chain) #:nodoc:
+      @prerequisites |= FileList[@dependencies.uniq]
+      super
+    end
+
+  end
+
+
+  # The integration tests task. Buildr has one such task (see Buildr#integration) that runs
+  # all tests marked with :integration=>true, and has a setup/teardown tasks separate from
+  # the unit tests.
+  class IntegrationTestsTask < Rake::Task
+
+    def initialize(*args) #:nodoc:
+      super
+      @setup = task("#{name}:setup")
+      @teardown = task("#{name}:teardown")
+      enhance do
+        puts 'Running integration tests...'  if verbose
+        TestTask.run_local_tests true
+      end
+    end
+
+    def execute(args) #:nodoc:
+      setup.invoke
+      begin
+        super
+      ensure
+        teardown.invoke
+      end
+    end
+
+    # :call-seq:
+    #   setup(*prereqs) => task
+    #   setup(*prereqs) { |task| .. } => task
+    #
+    # Returns the setup task. The setup task is executed before running the integration tests.
+    def setup(*prereqs, &block)
+      @setup.enhance prereqs, &block
+    end
+
+    # :call-seq:
+    #   teardown(*prereqs) => task
+    #   teardown(*prereqs) { |task| .. } => task
+    #
+    # Returns the teardown task. The teardown task is executed after running the integration tests.
+    def teardown(*prereqs, &block)
+      @teardown.enhance prereqs, &block
+    end
+
+  end
+
+
+  # Methods added to Project to support compilation and running of tests.
+  module Test
+
+    include Extension
+
+    first_time do
+      desc 'Run all tests'
+      task('test') { TestTask.run_local_tests false }
+
+      # This rule takes a suffix and runs that tests in the current project. For example;
+      #   buildr test:MyTest
+      # will run the test com.example.MyTest, if such a test exists for this project.
+      #
+      # If you want to run multiple test, separate tham with a comma. You can also use glob
+      # (* and ?) patterns to match multiple tests, see the TestTask#include method.
+      rule /^test:.*$/ do |task|
+        # The map works around a JRuby bug whereby the string looks fine, but fails in fnmatch.
+        TestTask.only_run task.name.scan(/test:(.*)/)[0][0].split(',').map { |t| "#{t}" }
+        task('test').invoke
+      end
+
+      task 'build' do |task|
+        # Make sure this happens as the last action on the build, so all other enhancements
+        # are made to run before starting the tests.
+        task.enhance do
+          task('test').invoke unless Buildr.options.test == false
+        end
+      end
+
+      IntegrationTestsTask.define_task('integration')
+
+      # Similar to test:[pattern] but for integration tests.
+      rule /^integration:.*$/ do |task|
+        unless task.name.split(':')[1] =~ /^(setup|teardown)$/
+          # The map works around a JRuby bug whereby the string looks fine, but fails in fnmatch.
+          TestTask.only_run task.name[/integration:(.*)/, 1].split(',').map { |t| "#{t}" }
+          task('integration').invoke
+        end
+      end
+
+    end
+    
+    before_define do |project|
+      # Define a recursive test task, and pass it a reference to the project so it can discover all other tasks.
+      test = TestTask.define_task('test')
+      test.send :associate_with, project
+
+      # Similar to the regular resources task but using different paths.
+      resources = ResourcesTask.define_task('test:resources')
+      resources.send :associate_with, project, :test
+      project.path_to(:source, :test, :resources).tap { |dir| resources.from dir if File.exist?(dir) }
+
+      # Similar to the regular compile task but using different paths.
+      compile = CompileTask.define_task('test:compile'=>[project.compile, resources])
+      compile.send :associate_with, project, :test
+      test.enhance [compile]
+
+      # Define these tasks once, otherwise we may get a namespace error.
+      test.setup ; test.teardown
+    end
+
+    after_define do |project|
+      test = project.test
+      # Dependency on compiled code, its dependencies and resources.
+      test.with project.compile.dependencies
+      test.with [project.compile.target, project.resources.target].compact
+      # Dependency on compiled tests and resources.  Dependencies added using with.
+      test.dependencies.concat [test.compile.target, test.resources.target].compact
+      # Picking up the test frameworks adds further dependencies.
+      test.framework
+
+      project.clean do
+        verbose(false) do
+          rm_rf test.compile.target.to_s if test.compile.target
+          rm_rf test.report_to.to_s
+        end
+      end
+    end
+
+
+    # :call-seq:
+    #   test(*prereqs) => TestTask
+    #   test(*prereqs) { |task| .. } => TestTask
+    #
+    # Returns the test task. The test task controls the entire test lifecycle.
+    #
+    # You can use the test task in three ways. You can access and configure specific
+    # test tasks, e.g. enhance the compile task by calling test.compile, setup for
+    # the tests by enhancing test.setup and so forth.
+    #
+    # You can use convenient methods that handle the most common settings. For example,
+    # add dependencies using test.with, or include only specific tests using test.include.
+    #
+    # You can also enhance this task directly. This method accepts a list of arguments
+    # that are used as prerequisites and an optional block that will be executed by the
+    # test task.
+    #
+    # This task compiles the project and the tests (in that order) before running any tests.
+    # It execute the setup task, runs all the tests, any enhancements, and ends with the
+    # teardown tasks.
+    def test(*prereqs, &block)
+      task('test').enhance prereqs, &block
+    end
+  
+    # :call-seq:
+    #   integration { |task| .... }
+    #   integration => IntegrationTestTask
+    #
+    # Use this method to return the integration tests task, or enhance it with a block to execute.
+    #
+    # There is one integration tests task you can execute directly, or as a result of running the package
+    # task (or tasks that depend on it, like install and deploy). It contains all the tests marked with
+    # :integration=>true, all other tests are considered unit tests and run by the test task before packaging.
+    # So essentially: build=>test=>packaging=>integration=>install/deploy.
+    #
+    # You add new tests from projects that define integration tests using the regular test task,
+    # but with the following addition:
+    #   test.using :integration
+    #
+    # Use this method to enhance the setup and teardown tasks that are executed before (and after) all
+    # integration tests are run, for example, to start a Web server or create a database.
+    def integration(*deps, &block)
+      Rake::Task['rake:integration'].enhance deps, &block
+    end
+
+  end
+
+
+  # :call-seq:
+  #   integration { |task| .... }
+  #   integration => IntegrationTestTask
+  #
+  # Use this method to return the integration tests task.
+  def integration(*deps, &block)
+    Rake::Task['rake:integration'].enhance deps, &block
+  end
+
+  class Options
+
+    # Runs tests after the build when true (default). This forces tests to execute
+    # after the build, including when running build related tasks like install, deploy and release.
+    #
+    # Set to false to not run any tests. Set to :all to run all tests, ignoring failures.
+    #
+    # This option is set from the environment variable 'test', so you can also do:
+
+    # Returns the test option (environment variable TEST). Possible values are:
+    # * :false -- Do not run any tests (also accepts 'no' and 'skip').
+    # * :true -- Run all tests, stop on failure (default if not set).
+    # * :all -- Run all tests, ignore failures.
+    def test
+      case value = ENV['TEST'] || ENV['test']
+      when /^(no|off|false|skip)$/i
+        false
+      when /^all$/i
+        :all
+      when /^(yes|on|true)$/i, nil
+        true
+      else
+        warn "Expecting the environment variable test to be 'no' or 'all', not sure what to do with #{value}, so I'm just going to run all the tests and stop at failure."
+        true
+      end
+    end
+
+    # Sets the test option (environment variable TEST). Possible values are true, false or :all.
+    #
+    # You can also set this from the environment variable, e.g.:
+    #
+    #   buildr          # With tests
+    #   buildr test=no  # Without tests
+    #   buildr test=all # Ignore failures
+    #   set TEST=no
+    #   buildr          # Without tests
+    def test=(flag)
+      ENV['test'] = nil
+      ENV['TEST'] = flag.to_s
+    end
+
+  end
+
+  Buildr.help << <<-HELP
+To run a full build without running any tests:
+  buildr test=no
+To run specific test:
+  buildr test:MyTest
+To run integration tests:
+  buildr integration
+    HELP
+
+end
+
+
+class Buildr::Project
+  include Buildr::Test
+end