pdk 0.2.0 → 0.3.0

data/lib/pdk/report.rb

@@ -12,7 +12,7 @@ module PDK
 
     # @return [Symbol] the method name of the default report format.
     def self.default_format
-      :to_text
+      :write_text
     end
 
     # @return [#write] the default target to write the report to.
@@ -47,7 +47,13 @@ module PDK
     #
     # @param target [#write] an IO object that the report will be written to.
     #   Defaults to PDK::Report.default_target.
-    def to_junit(target = self.class.default_target)
+    def write_junit(target = self.class.default_target)
+      # Extra defaulting here, b/c the Class.send method will pass in nil
+      target ||= self.class.default_target
+
+      # Open a File Object for IO if target is a string containing a filename or path
+      target = File.open(target, 'w') if target.is_a? String
+
       document = REXML::Document.new
       document << REXML::XMLDecl.new
       testsuites = REXML::Element.new('testsuites')
@@ -59,6 +65,7 @@ module PDK
         testsuite.attributes['tests'] = testcases.length
         testsuite.attributes['errors'] = testcases.select(&:error?).length
         testsuite.attributes['failures'] = testcases.select(&:failure?).length
+        testsuite.attributes['skipped'] = testcases.select(&:skipped?).length
         testsuite.attributes['time'] = 0
         testsuite.attributes['timestamp'] = Time.now.strftime('%Y-%m-%dT%H:%M:%S')
         testsuite.attributes['hostname'] = Socket.gethostname
@@ -75,6 +82,8 @@ module PDK
 
       document.elements << testsuites
       document.write(target, 2)
+    ensure
+      target.close if target.is_a? File
     end
 
     # Renders the report as plain text.
@@ -84,12 +93,20 @@ module PDK
     #
     # @param target [#write] an IO object that the report will be written to.
     #   Defaults to PDK::Report.default_target.
-    def to_text(target = self.class.default_target)
+    def write_text(target = self.class.default_target)
+      # Extra defaulting here, b/c the Class.send method will pass in nil
+      target ||= self.class.default_target
+
+      # Open a File Object for IO if target is a string containing a filename or path
+      target = File.open(target, 'w') if target.is_a? String
+
       events.each do |_tool, tool_events|
         tool_events.each do |event|
           target.puts(event.to_text) unless event.pass?
         end
       end
+    ensure
+      target.close if target.is_a? File
     end
   end
 end

data/lib/pdk/report/event.rb

@@ -35,6 +35,9 @@ module PDK
       #   :skipped.
       attr_reader :state
 
+      # @return [Array] Array of full stack trace lines associated with event
+      attr_reader :trace
+
       # Initailises a new PDK::Report::Event object.
       #
       # @param data [Hash{Symbol=>Object}
@@ -46,6 +49,7 @@ module PDK
       # @option data [String] :severity (see #severity)
       # @option data [String] :test (see #test)
       # @option data [Symbol] :state (see #state)
+      # @option data [Array] :trace (see #trace)
       #
       # @raise [ArgumentError] (see #sanitise_data)
       def initialize(data)
@@ -77,6 +81,7 @@ module PDK
       end
 
       # Checks if the event is the result of test that was not run.
+      # This includes pending tests (that are run but have an expected failure result).
       #
       # @return [Boolean] true if the test was skipped, otherwise false.
       def skipped?
@@ -88,7 +93,9 @@ module PDK
       # @return [String] The rendered event.
       def to_text
         location = [file, line, column].compact.join(':')
+        location = nil if location.empty?
 
+        # TODO: maybe add trace
         [location, severity, message].compact.join(': ')
       end
 
@@ -271,6 +278,28 @@ module PDK
 
         value.to_i
       end
+
+      # Cleans up provided stack trace by removing entries that are inside gems
+      # or the rspec binstub.
+      #
+      # @param value [Array] Array of stack trace lines
+      #
+      # @return [Array] Array of stack trace lines with less relevant lines excluded
+      def sanitise_trace(value)
+        return nil if value.nil?
+
+        valid_types = [Array]
+
+        unless valid_types.include?(value.class)
+          raise ArgumentError, _('trace must be an Array of stack trace lines')
+        end
+
+        # Drop any stacktrace lines that include '/gems/' in the path or
+        # are the original rspec binstub lines
+        value.reject do |line|
+          (line =~ %r{/gems/}) || (line =~ %r{bin/rspec:})
+        end
+      end
     end
   end
 end

data/lib/pdk/tests/unit.rb

@@ -1,25 +1,130 @@
 require 'pdk'
 require 'pdk/cli/exec'
 require 'pdk/util/bundler'
+require 'json'
 
 module PDK
   module Test
     class Unit
       def self.cmd(_tests)
-        # TODO: actually run the tests
-        # cmd = 'rake spec'
-        # cmd += " #{tests}" if tests
-        cmd = 'pwd'
-        cmd
+        # TODO: test selection
+        [File.join(PDK::Util.module_root, 'bin', 'rake'), 'spec']
       end
 
-      def self.invoke(tests, report = nil)
+      def self.invoke(report, options = {})
         PDK::Util::Bundler.ensure_bundle!
+        PDK::Util::Bundler.ensure_binstubs!('rake')
 
-        puts _('Running unit tests: %{tests}') % { tests: tests }
+        tests = options.fetch(:tests)
 
-        output = PDK::CLI::Exec.execute(cmd(tests))
-        report.write(output) if report
+        cmd_argv = cmd(tests)
+        cmd_argv.unshift('ruby') if Gem.win_platform?
+
+        command = PDK::CLI::Exec::Command.new(*cmd_argv).tap do |c|
+          c.context = :module
+          c.add_spinner('Running unit tests')
+          c.environment['CI_SPEC_OPTIONS'] = '--format j'
+        end
+
+        PDK.logger.debug(_('Running %{cmd}') % { cmd: command.argv.join(' ') })
+
+        result = command.execute!
+
+        # TODO: cleanup rspec and/or beaker output
+        # Iterate through possible JSON documents until we find one that is valid.
+        json_result = nil
+
+        result[:stdout].scan(%r{\{(?:[^{}]|(?:\g<0>))*\}}x) do |str|
+          begin
+            json_result = JSON.parse(str)
+            break
+          rescue JSON::ParserError
+            next
+          end
+        end
+
+        raise PDK::CLI::FatalError, _('Unit test output did not contain a valid JSON result: %{output}') % { output: result[:stdout] } unless json_result
+
+        parse_output(report, json_result)
+
+        result[:exit_code]
+      end
+
+      def self.parse_output(report, json_data)
+        # Output messages to stderr.
+        json_data['messages'] && json_data['messages'].each { |msg| $stderr.puts msg }
+
+        example_results = {
+          # Only possibilities are passed, failed, pending:
+          # https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/example.rb#L548
+          'passed' => [],
+          'failed' => [],
+          'pending' => [],
+        }
+
+        json_data['examples'] && json_data['examples'].each do |ex|
+          example_results[ex['status']] << ex if example_results.key?(ex['status'])
+        end
+
+        example_results.each do |result, examples|
+          # Translate rspec example results to JUnit XML testcase results
+          state = case result
+                  when 'passed' then :passed
+                  when 'failed' then :failure
+                  when 'pending' then :skipped
+                  end
+
+          examples.each do |ex|
+            report.add_event(
+              source: 'rspec',
+              state: state,
+              file: ex['file_path'],
+              line: ex['line_number'],
+              test: ex['full_description'],
+              severity: ex['status'],
+              message: ex['pending_message'] || (ex['exception'] && ex['exception']['message']) || nil,
+              trace: (ex['exception'] && ex['exception']['backtrace']) || nil,
+            )
+          end
+        end
+
+        return unless json_data['summary']
+
+        # TODO: standardize summary output
+        $stderr.puts '  ' << _('Evaluated %{total} tests in %{duration} seconds: %{failures} failures, %{pending} pending') % {
+          total: json_data['summary']['example_count'],
+          duration: json_data['summary']['duration'],
+          failures: json_data['summary']['failure_count'],
+          pending: json_data['summary']['pending_count'],
+        }
+      end
+
+      # @return array of { :id, :full_description }
+      def self.list
+        PDK::Util::Bundler.ensure_bundle!
+        PDK::Util::Bundler.ensure_binstubs!('rspec-core')
+
+        command_argv = [File.join(PDK::Util.module_root, 'bin', 'rspec'), '--dry-run', '--format', 'json']
+        command_argv.unshift('ruby') if Gem.win_platform?
+        list_command = PDK::CLI::Exec::Command.new(*command_argv)
+        list_command.context = :module
+        output = list_command.execute!
+
+        rspec_json_output = JSON.parse(output[:stdout])
+        if rspec_json_output['examples'].empty?
+          rspec_message = rspec_json_output['messages'][0]
+          return [] if rspec_message == 'No examples found.'
+
+          raise PDK::CLI::FatalError, _('Unable to enumerate examples. rspec reported: %{message}' % { message: rspec_message })
+        else
+          examples = []
+          rspec_json_output['examples'].each do |example|
+            examples << { id: example['id'], full_description: example['full_description'] }
+          end
+          examples
+        end
+      rescue JSON::ParserError => e
+        raise PDK::CLI::FatalError, _('Failed to parse output from rspec: %{message}' % { message: e.message })
       end
     end
   end

data/lib/pdk/util/bundler.rb

@@ -62,24 +62,28 @@ module PDK
         end
 
         def installed?
-          output_start(_('Checking for missing Gemfile dependencies'))
+          command = bundle_command('check', "--gemfile=#{gemfile}", "--path=#{bundle_cachedir}").tap do |c|
+            c.add_spinner(_('Checking for missing Gemfile dependencies'))
+          end
 
-          result = invoke('check', "--gemfile=#{gemfile}", "--path=#{bundle_cachedir}")
+          result = command.execute!
 
-          output_end(:success)
+          unless result[:exit_code].zero?
+            $stderr.puts result[:stdout]
+            $stderr.puts result[:stderr]
+          end
 
           result[:exit_code].zero?
         end
 
         def lock!
-          output_start(_('Resolving Gemfile dependencies'))
+          command = bundle_command('lock').tap do |c|
+            c.add_spinner(_('Resolving Gemfile dependencies'))
+          end
 
-          result = invoke('lock')
+          result = command.execute!
 
-          if result[:exit_code].zero?
-            output_end(:success)
-          else
-            output_end(:failure)
+          unless result[:exit_code].zero?
             $stderr.puts result[:stdout]
             $stderr.puts result[:stderr]
           end
@@ -88,14 +92,13 @@ module PDK
         end
 
         def install!
-          output_start(_('Installing missing Gemfile dependencies'))
+          command = bundle_command('install', "--gemfile=#{gemfile}", "--path=#{bundle_cachedir}").tap do |c|
+            c.add_spinner(_('Installing missing Gemfile dependencies'))
+          end
 
-          result = invoke('install', "--gemfile=#{gemfile}", "--path=#{bundle_cachedir}")
+          result = command.execute!
 
-          if result[:exit_code].zero?
-            output_end(:success)
-          else
-            output_end(:failure)
+          unless result[:exit_code].zero?
             $stderr.puts result[:stdout]
             $stderr.puts result[:stderr]
           end
@@ -104,10 +107,12 @@ module PDK
         end
 
         def binstubs!(gems)
-          # FIXME: wrap in progress indicator
-          result = invoke('binstubs', gems.join(' '), '--force')
+          command = bundle_command('binstubs', gems.join(' '), '--force')
+
+          result = command.execute!
 
           unless result[:exit_code].zero?
+            PDK.logger.error(_('Failed to generate binstubs for %{gems}') % { gems: gems.join(' ') })
             $stderr.puts result[:stdout]
             $stderr.puts result[:stderr]
           end
@@ -121,13 +126,10 @@ module PDK
 
         private
 
-        def invoke(*args)
-          bundle_bin = PDK::CLI::Exec.bundle_bin
-          command = PDK::CLI::Exec::Command.new(bundle_bin, *args).tap do |c|
+        def bundle_command(*args)
+          PDK::CLI::Exec::Command.new(PDK::CLI::Exec.bundle_bin, *args).tap do |c|
             c.context = :module
           end
-
-          command.execute!
         end
 
         def gemfile_lock
@@ -137,30 +139,6 @@ module PDK
         def bundle_cachedir
           @bundle_cachedir ||= File.join(PDK::Util.cachedir, 'bundler')
         end
-
-        # These two output_* methods are just a way to not try to do the spinner stuff on Windows for now.
-        def output_start(message)
-          if Gem.win_platform?
-            $stderr.print "#{message}... "
-          else
-            @spinner = TTY::Spinner.new("[:spinner] #{message}")
-            @spinner.auto_spin
-          end
-        end
-
-        def output_end(state)
-          if Gem.win_platform?
-            $stderr.print((state == :success) ? _("done.\n") : _("FAILURE!\n"))
-          else
-            if state == :success
-              @spinner.success
-            else
-              @spinner.error
-            end
-
-            remove_instance_variable(:@spinner)
-          end
-        end
       end
     end
   end

data/lib/pdk/validators/base_validator.rb

@@ -20,8 +20,7 @@ module PDK
         targets.map { |target|
           if respond_to?(:pattern)
             if File.directory?(target)
-              files_glob = Array[pattern].flatten.map { |p| Dir.glob(File.join(target, p)) }
-              files_glob.flatten.empty? ? target : files_glob
+              Array[pattern].flatten.map { |p| Dir.glob(File.join(target, p)) }
             else
               target
             end
@@ -40,13 +39,13 @@ module PDK
       end
 
       def self.invoke(report, options = {})
-        PDK::Util::Bundler.ensure_binstubs!(cmd)
-
         targets = parse_targets(options)
-        cmd_argv = parse_options(options, targets).unshift(cmd_path)
-        cmd_argv.unshift('ruby') if Gem.win_platform?
 
-        PDK.logger.debug(_('Running %{cmd}') % { cmd: cmd_argv.join(' ') })
+        return 0 if targets.empty?
+
+        PDK::Util::Bundler.ensure_binstubs!(cmd)
+        cmd_argv = parse_options(options, targets).unshift(cmd_path)
+        cmd_argv.unshift('ruby', '-W0') if Gem.win_platform?
 
         command = PDK::CLI::Exec::Command.new(*cmd_argv).tap do |c|
           c.context = :module
@@ -55,13 +54,7 @@ module PDK
 
         result = command.execute!
 
-        begin
-          json_data = JSON.parse(result[:stdout])
-        rescue JSON::ParserError
-          json_data = []
-        end
-
-        parse_output(report, json_data)
+        parse_output(report, result, targets)
 
         result[:exit_code]
       end

data/lib/pdk/validators/metadata.rb

@@ -14,6 +14,10 @@ module PDK
         'metadata-json-lint'
       end
 
+      def self.spinner_text
+        _('Checking metadata.json')
+      end
+
       def self.parse_targets(_options)
         [File.join(PDK::Util.module_root, 'metadata.json')]
       end
@@ -24,20 +28,38 @@ module PDK
         cmd_options.concat(targets)
       end
 
-      def self.parse_output(report, json_data)
-        return if json_data.empty?
-
-        json_data.delete('result')
-        json_data.keys.each do |type|
-          json_data[type].each do |offense|
-            report.add_event(
-              file:     'metadata.json',
-              source:   cmd,
-              message:  offense['msg'],
-              test:     offense['check'],
-              severity: type,
-              state:    :failure,
-            )
+      def self.parse_output(report, result, _targets)
+        begin
+          json_data = JSON.parse(result[:stdout])
+        rescue JSON::ParserError
+          json_data = []
+        end
+
+        if json_data.empty?
+          report.add_event(
+            file:     'metadata.json',
+            source:   cmd,
+            state:    :passed,
+            severity: :ok,
+          )
+        else
+          json_data.delete('result')
+          json_data.keys.each do |type|
+            json_data[type].each do |offense|
+              # metadata-json-lint groups the offenses by type, so the type ends
+              # up being `warnings` or `errors`. We want to convert that to the
+              # singular noun for the event.
+              event_type = type[%r{\A(.+?)s?\Z}, 1]
+
+              report.add_event(
+                file:     'metadata.json',
+                source:   cmd,
+                message:  offense['msg'],
+                test:     offense['check'],
+                severity: event_type,
+                state:    :failure,
+              )
+            end
           end
         end
       end