inspec 1.51.0 → 1.51.6

data/lib/inspec/formatters.rb

@@ -0,0 +1,3 @@
+require 'inspec/formatters/base'
+require 'inspec/formatters/json_rspec'
+require 'inspec/formatters/show_progress'

data/lib/inspec/formatters/base.rb

@@ -0,0 +1,208 @@
+require 'rspec/core'
+require 'rspec/core/formatters/base_formatter'
+
+module Inspec::Formatters
+  class Base < RSpec::Core::Formatters::BaseFormatter
+    RSpec::Core::Formatters.register self, :close, :dump_summary, :stop
+
+    attr_accessor :backend, :run_data
+
+    def initialize(output)
+      super(output)
+
+      @run_data = {}
+      @profiles = []
+      @profiles_info = nil
+      @backend = nil
+    end
+
+    # RSpec Override: #dump_summary
+    #
+    # Supply run summary data, such as the InSpec version and the total duration.
+    def dump_summary(summary)
+      run_data[:version] = Inspec::VERSION
+      run_data[:statistics] = {
+        duration: summary.duration,
+      }
+    end
+
+    # RSpec Override: #stop
+    #
+    # Called at the end of a complete RSpec run.
+    # We use this to map tests to controls and flesh out the rest of the run_data
+    # hash to include details about the run, the platform, etc.
+    def stop(notification)
+      # This might be a bit confusing. The results are not actually organized
+      # by control. It is organized by test. So if a control has 3 tests, the
+      # output will have 3 control entries, each one with the same control id
+      # and different test results. An rspec example maps to an inspec test.
+      run_data[:controls] = notification.examples.map do |example|
+        format_example(example).tap do |hash|
+          e = example.exception
+          next unless e
+
+          if example.metadata[:sensitive]
+            hash[:message] = '*** sensitive output suppressed ***'
+          else
+            hash[:message] = exception_message(e)
+          end
+
+          next if e.is_a? RSpec::Expectations::ExpectationNotMetError
+          hash[:exception] = e.class.name
+          hash[:backtrace] = e.backtrace
+        end
+      end
+
+      # include any tests that were run that were not part of a control
+      run_data[:other_checks] = examples_without_controls
+      examples_with_controls.each do |example|
+        control = example2control(example)
+        move_example_into_control(example, control)
+      end
+
+      # flesh out the profiles key with additional profile information
+      run_data[:profiles] = profiles_info
+
+      # add the platform information for this particular target
+      run_data[:platform] = {
+        name: platform(:name),
+        release: platform(:release),
+        target: backend_target,
+      }
+    end
+
+    # Add the current profile to the list of executed profiles.
+    # Called by the runner during example collection.
+    def add_profile(profile)
+      @profiles.push(profile)
+    end
+
+    # Return all the collected output to the caller
+    def results
+      run_data
+    end
+
+    private
+
+    def exception_message(exception)
+      if exception.is_a?(RSpec::Core::MultipleExceptionError)
+        exception.all_exceptions.map(&:message).uniq.join("\n\n")
+      else
+        exception.message
+      end
+    end
+
+    # RSpec Override: #format_example
+    #
+    # Called after test execution, this allows us to populate our own hash with data
+    # for this test that is necessary for the rest of our reports.
+    def format_example(example) # rubocop:disable Metrics/AbcSize
+      if !example.metadata[:description_args].empty? && example.metadata[:skip]
+        # For skipped profiles, rspec returns in full_description the skip_message as well. We don't want
+        # to mix the two, so we pick the full_description from the example.metadata[:example_group] hash.
+        code_description = example.metadata[:example_group][:description]
+      else
+        code_description = example.metadata[:full_description]
+      end
+
+      res = {
+        id: example.metadata[:id],
+        profile_id: example.metadata[:profile_id],
+        status: example.execution_result.status.to_s,
+        code_desc: code_description,
+        run_time: example.execution_result.run_time,
+        start_time: example.execution_result.started_at.to_s,
+        resource_title: example.metadata[:described_class] || example.metadata[:example_group][:description],
+        expectation_message: format_expectation_message(example),
+      }
+
+      unless (pid = example.metadata[:profile_id]).nil?
+        res[:profile_id] = pid
+      end
+
+      if res[:status] == 'pending'
+        res[:status] = 'skipped'
+        res[:skip_message] = example.metadata[:description]
+        res[:resource] = example.metadata[:described_class].to_s
+      end
+
+      res
+    end
+
+    def format_expectation_message(example)
+      if (example.metadata[:example_group][:description_args].first == example.metadata[:example_group][:described_class]) ||
+         example.metadata[:example_group][:described_class].nil?
+        example.metadata[:description]
+      else
+        "#{example.metadata[:example_group][:description]} #{example.metadata[:description]}"
+      end
+    end
+
+    def platform(field)
+      return nil if @backend.nil?
+      @backend.platform[field]
+    end
+
+    def backend_target
+      return nil if @backend.nil?
+      connection = @backend.backend
+      connection.respond_to?(:uri) ? connection.uri : nil
+    end
+
+    def examples
+      run_data[:controls]
+    end
+
+    def examples_without_controls
+      examples.find_all { |example| example2control(example).nil? }
+    end
+
+    def examples_with_controls
+      examples.find_all { |example| !example2control(example).nil? }
+    end
+
+    def profiles_info
+      @profiles_info ||= @profiles.map(&:info!).map(&:dup)
+    end
+
+    def example2control(example)
+      profile = profile_from_example(example)
+      return nil unless profile&.[](:controls)
+      profile[:controls].find { |x| x[:id] == example[:id] }
+    end
+
+    def profile_from_example(example)
+      profiles_info.find { |p| profile_contains_example?(p, example) }
+    end
+
+    def profile_contains_example?(profile, example)
+      profile_name = profile[:name]
+      example_profile_id = example[:profile_id]
+
+      # if either the profile name is nil or the profile in the given example
+      # is nil, assume the profile doesn't contain the example and default
+      # to creating a new profile. Otherwise, for profiles that have no
+      # metadata, this may incorrectly match a profile that does not contain
+      # this example, leading to Ruby exceptions.
+      return false if profile_name.nil? || example_profile_id.nil?
+
+      # The correct profile is one where the name of the profile, and the profile
+      # name in the example match. Additionally, the list of controls in the
+      # profile must contain the example in question (which we match by ID).
+      #
+      # While the profile name match is usually good enough, we must also match by
+      # the control ID in the case where an InSpec runner has multiple profiles of
+      # the same name (i.e. when Test Kitchen is running concurrently using a
+      # single test suite that uses the Flat source reader, in which case InSpec
+      # creates a fake profile with a name like "tests from /path/to/tests")
+      profile_name == example_profile_id && profile[:controls].any? { |control| control[:id] == example[:id] }
+    end
+
+    def move_example_into_control(example, control)
+      control[:results] ||= []
+      example.delete(:id)
+      example.delete(:profile_id)
+      control[:results].push(example)
+    end
+  end
+end

data/lib/inspec/formatters/json_rspec.rb

@@ -0,0 +1,20 @@
+module Inspec::Formatters
+  class RspecJson < RSpec::Core::Formatters::JsonFormatter
+    RSpec::Core::Formatters.register self
+
+    private
+
+    # We are cheating and overriding a private method in RSpec's core JsonFormatter.
+    # This is to avoid having to repeat this id functionality in both dump_summary
+    # and dump_profile (both of which call format_example).
+    # See https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/formatters/json_formatter.rb
+    #
+    # rspec's example id here corresponds to an inspec test's control name -
+    # either explicitly specified or auto-generated by rspec itself.
+    def format_example(example)
+      res = super(example)
+      res[:id] = example.metadata[:id]
+      res
+    end
+  end
+end

data/lib/inspec/formatters/show_progress.rb

@@ -0,0 +1,12 @@
+module Inspec::Formatters
+  class ShowProgress < RSpec::Core::Formatters::ProgressFormatter
+    RSpec::Core::Formatters.register self
+
+    # remove summary output from progress
+    %w{dump_summary dump_failures dump_pending message seed start_dump}.each do |m|
+      define_method(m) do |*args|
+        # override
+      end
+    end
+  end
+end

data/lib/inspec/objects.rb

@@ -4,6 +4,7 @@ module Inspec
   autoload :Attribute, 'inspec/objects/attribute'
   autoload :Tag, 'inspec/objects/tag'
   autoload :Control, 'inspec/objects/control'
+  autoload :Describe, 'inspec/objects/describe'
   autoload :EachLoop, 'inspec/objects/each_loop'
   autoload :List, 'inspec/objects/list'
   autoload :OrTest, 'inspec/objects/or_test'

data/lib/inspec/objects/describe.rb

@@ -0,0 +1,92 @@
+# encoding:utf-8
+
+module Inspec
+  class Describe
+    # Internal helper to structure test objects.
+    # Should not be exposed to the user as it is hidden behind
+    # `add_test`, `to_hash`, and `to_ruby` in Inspec::Describe
+    Test = Struct.new(:its, :matcher, :expectation, :negated) do
+      def negate!
+        self.negated = !negated
+      end
+
+      def to_ruby
+        itsy = its.nil? ? 'it' : 'its(' + its.to_s.inspect + ')'
+        naughty = negated ? '_not' : ''
+        xpect = if expectation.nil?
+                  ''
+                elsif expectation.class == Regexp
+                  # without this, xpect values like / \/zones\// will not be parsed properly
+                  "(#{expectation.inspect})"
+                else
+                  ' ' + expectation.inspect
+                end
+        format('%s { should%s %s%s }', itsy, naughty, matcher, xpect)
+      end
+    end
+
+    # A qualifier describing the resource that will be tested. It may consist
+    # of the resource identification and multiple accessors to pinpoint the data
+    # the user wants to test.
+    attr_accessor :qualifier
+
+    # An array of individual tests for the qualifier. Every entry will be
+    # translated into an `it` or `its` clause.
+    attr_accessor :tests
+
+    # Optional variables which are used by tests.
+    attr_accessor :variables
+
+    # Optional method to skip this describe block altogether. If `skip` is
+    # defined it takes precendence and will print the skip statement instead
+    # of adding other tests.
+    attr_accessor :skip
+
+    include RubyHelper
+
+    def initialize
+      @qualifier = []
+      @tests = []
+      @variables = []
+    end
+
+    def add_test(its, matcher, expectation)
+      test = Inspec::Describe::Test.new(its, matcher, expectation, false)
+      tests.push(test)
+      test
+    end
+
+    def to_ruby
+      return rb_skip if !skip.nil?
+      rb_describe
+    end
+
+    def to_hash
+      { qualifier: qualifier, tests: tests.map(&:to_h), variables: variables, skip: skip }
+    end
+
+    def resource
+      return nil if qualifier.empty? || qualifier[0].empty? || qualifier[0][0].empty?
+      qualifier[0][0]
+    end
+
+    private
+
+    def rb_describe
+      vars = variables.map(&:to_ruby).join("\n")
+      vars += "\n" unless vars.empty?
+
+      objarr = @qualifier
+      objarr = [['unknown object'.inspect]] if objarr.nil? || objarr.empty?
+      obj = objarr.map { |q| ruby_qualifier(q) }.join('.')
+
+      rbtests = tests.map(&:to_ruby).join("\n  ")
+      format("%sdescribe %s do\n  %s\nend", vars, obj, rbtests)
+    end
+
+    def rb_skip
+      obj = @qualifier || skip.inspect
+      format("describe %s do\n  skip %s\nend", obj, skip.inspect)
+    end
+  end
+end

data/lib/inspec/reporters.rb

@@ -0,0 +1,33 @@
+require 'inspec/reporters/base'
+require 'inspec/reporters/cli'
+require 'inspec/reporters/json'
+require 'inspec/reporters/json_min'
+require 'inspec/reporters/junit'
+
+module Inspec::Reporters
+  def self.render(reporter, run_data)
+    name, config = reporter
+    config[:run_data] = run_data
+    case name
+    when 'cli'
+      reporter = Inspec::Reporters::CLI.new(config)
+    when 'json'
+      reporter = Inspec::Reporters::Json.new(config)
+    when 'json-min'
+      reporter = Inspec::Reporters::JsonMin.new(config)
+    when 'junit'
+      reporter = Inspec::Reporters::Junit.new(config)
+    else
+      raise NotImplementedError, "'#{name}' is not a valid reporter type."
+    end
+
+    reporter.render
+    output = reporter.rendered_output
+
+    if config['file']
+      File.write(config['file'], output)
+    elsif config['stdout'] == true
+      puts output
+    end
+  end
+end

data/lib/inspec/reporters/base.rb

@@ -0,0 +1,23 @@
+module Inspec::Reporters
+  class Base
+    attr_reader :run_data
+
+    def initialize(config)
+      @run_data = config[:run_data]
+      @output = ''
+    end
+
+    def output(str)
+      @output << "#{str}\n"
+    end
+
+    def rendered_output
+      @output
+    end
+
+    # each reporter must implement #render
+    def render
+      raise NotImplementedError, "#{self.class} must implement a `#render` method to format its output."
+    end
+  end
+end

data/lib/inspec/reporters/cli.rb

@@ -0,0 +1,395 @@
+# encoding: utf-8
+
+module Inspec::Reporters
+  class CLI < Base
+    case RUBY_PLATFORM
+    when /windows|mswin|msys|mingw|cygwin/
+      # Most currently available Windows terminals have poor support
+      # for ANSI extended colors
+      COLORS = {
+        'critical' => "\033[0;1;31m",
+        'major'    => "\033[0;1;31m",
+        'minor'    => "\033[0;36m",
+        'failed'   => "\033[0;1;31m",
+        'passed'   => "\033[0;1;32m",
+        'skipped'  => "\033[0;37m",
+        'reset'    => "\033[0m",
+      }.freeze
+
+      # Most currently available Windows terminals have poor support
+      # for UTF-8 characters so use these boring indicators
+      INDICATORS = {
+        'critical' => '[CRIT]',
+        'major'    => '[MAJR]',
+        'minor'    => '[MINR]',
+        'failed'   => '[FAIL]',
+        'skipped'  => '[SKIP]',
+        'passed'   => '[PASS]',
+        'unknown'  => '[UNKN]',
+      }.freeze
+    else
+      # Extended colors for everyone else
+      COLORS = {
+        'critical' => "\033[38;5;9m",
+        'major'    => "\033[38;5;208m",
+        'minor'    => "\033[0;36m",
+        'failed'   => "\033[38;5;9m",
+        'passed'   => "\033[38;5;41m",
+        'skipped'  => "\033[38;5;247m",
+        'reset'    => "\033[0m",
+      }.freeze
+
+      # Groovy UTF-8 characters for everyone else...
+      # ...even though they probably only work on Mac
+      INDICATORS = {
+        'critical' => '×',
+        'major'    => '∅',
+        'minor'    => '⊚',
+        'failed'   => '×',
+        'skipped'  => '↺',
+        'passed'   => '✔',
+        'unknown'  => '?',
+      }.freeze
+    end
+
+    MULTI_TEST_CONTROL_SUMMARY_MAX_LEN = 60
+
+    def render
+      run_data[:profiles].each do |profile|
+        @control_count = 0
+        output('')
+        print_profile_header(profile)
+        print_standard_control_results(profile)
+        print_anonymous_control_results(profile)
+        output(format_message(
+                 indentation: 5,
+                 message: 'No tests executed.',
+        )) if @control_count == 0
+      end
+
+      output('')
+      print_profile_summary
+      print_tests_summary
+    end
+
+    private
+
+    def print_profile_header(profile)
+      output("Profile: #{format_profile_name(profile)}")
+      output("Version: #{profile[:version] || '(not specified)'}")
+      output("Target:  #{run_data[:platform][:target]}") unless run_data[:platform][:target].nil?
+      output('')
+    end
+
+    def print_standard_control_results(profile)
+      standard_controls_from_profile(profile).each do |control_from_profile|
+        control = Control.new(control_from_profile)
+        next if control.results.nil?
+        output(format_control_header(control))
+        control.results.each do |result|
+          output(format_result(control, result, :standard))
+          @control_count += 1
+        end
+      end
+      output('') if @control_count > 0
+    end
+
+    def print_anonymous_control_results(profile)
+      anonymous_controls_from_profile(profile).each do |control_from_profile|
+        control = Control.new(control_from_profile)
+        next if control.results.nil?
+        output(format_control_header(control))
+        control.results.each do |result|
+          output(format_result(control, result, :anonymous))
+          @control_count += 1
+        end
+      end
+    end
+
+    def format_profile_name(profile)
+      if profile[:title].nil?
+        (profile[:name] || 'unknown').to_s
+      else
+        "#{profile[:title]} (#{profile[:name] || 'unknown'})"
+      end
+    end
+
+    def format_control_header(control)
+      impact = control.impact_string
+      format_message(
+        color: impact,
+        indicator: impact,
+        message: control.title_for_report,
+      )
+    end
+
+    def format_result(control, result, type)
+      impact = control.impact_string_for_result(result)
+
+      message = if result[:status] == 'skipped'
+                  result[:skip_message]
+                elsif type == :anonymous
+                  result[:expectation_message]
+                else
+                  result[:code_desc]
+                end
+
+      # append any failure details to the message if they exist
+      message += "\n#{result[:message]}" if result[:message]
+
+      format_message(
+        color: impact,
+        indicator: impact,
+        indentation: 5,
+        message: message,
+      )
+    end
+
+    def format_message(message_info)
+      indicator = message_info[:indicator]
+      color = message_info[:color]
+      indentation = message_info.fetch(:indentation, 2)
+      message = message_info[:message]
+
+      message_to_format = ''
+      message_to_format += "#{INDICATORS[indicator]}  " unless indicator.nil?
+      message_to_format += message.to_s.lstrip
+
+      format_with_color(color, indent_lines(message_to_format, indentation))
+    end
+
+    def format_with_color(color_name, text)
+      return text if defined?(RSpec.configuration) && !RSpec.configuration.color
+      return text unless COLORS.key?(color_name)
+
+      "#{COLORS[color_name]}#{text}#{COLORS['reset']}"
+    end
+
+    def all_unique_controls
+      return @unique_controls unless @unique_controls.nil?
+
+      @unique_controls = Set.new
+      run_data[:profiles].each do |profile|
+        profile[:controls].map { |control| @unique_controls.add(control) }
+      end
+
+      @unique_controls
+    end
+
+    def profile_summary
+      return @profile_summary unless @profile_summary.nil?
+
+      failed = 0
+      skipped = 0
+      passed = 0
+      critical = 0
+      major = 0
+      minor = 0
+
+      all_unique_controls.each do |control|
+        next if control[:id].start_with? '(generated from '
+        next unless control[:results]
+        if control[:results].any? { |r| r[:status] == 'failed' }
+          failed += 1
+          if control[:impact] >= 0.7
+            critical += 1
+          elsif control[:impact] >= 0.4
+            major += 1
+          else
+            minor += 1
+          end
+        elsif control[:results].any? { |r| r[:status] == 'skipped' }
+          skipped += 1
+        else
+          passed += 1
+        end
+      end
+
+      total = failed + passed + skipped
+
+      @profile_summary = {
+        'total' => total,
+        'failed' => {
+          'total' => failed,
+          'critical' => critical,
+          'major' => major,
+          'minor' => minor,
+        },
+        'skipped' => skipped,
+        'passed' => passed,
+      }
+    end
+
+    def tests_summary
+      return @tests_summary unless @tests_summary.nil?
+
+      total = 0
+      failed = 0
+      skipped = 0
+      passed = 0
+
+      all_unique_controls.each do |control|
+        next unless control[:results]
+        control[:results].each do |result|
+          if result[:status] == 'failed'
+            failed += 1
+          elsif result[:status] == 'skipped'
+            skipped += 1
+          else
+            passed += 1
+          end
+        end
+      end
+
+      @tests_summary = { 'total' => total, 'failed' => failed, 'skipped' => skipped, 'passed' => passed }
+    end
+
+    def print_profile_summary
+      summary = profile_summary
+      return unless summary['total'] > 0
+
+      success_str = summary['passed'] == 1 ? '1 successful control' : "#{summary['passed']} successful controls"
+      failed_str  = summary['failed']['total'] == 1 ? '1 control failure' : "#{summary['failed']['total']} control failures"
+      skipped_str = summary['skipped'] == 1 ? '1 control skipped' : "#{summary['skipped']} controls skipped"
+
+      success_color = summary['passed'] > 0 ? 'passed' : 'no_color'
+      failed_color = summary['failed']['total'] > 0 ? 'failed' : 'no_color'
+      skipped_color = summary['skipped'] > 0 ? 'skipped' : 'no_color'
+
+      s = format(
+        'Profile Summary: %s, %s, %s',
+        format_with_color(success_color, success_str),
+        format_with_color(failed_color, failed_str),
+        format_with_color(skipped_color, skipped_str),
+      )
+      output(s) if summary['total'] > 0
+    end
+
+    def print_tests_summary
+      summary = tests_summary
+
+      failed_str = summary['failed'] == 1 ? '1 failure' : "#{summary['failed']} failures"
+
+      success_color = summary['passed'] > 0 ? 'passed' : 'no_color'
+      failed_color = summary['failed'] > 0 ? 'failed' : 'no_color'
+      skipped_color = summary['skipped'] > 0 ? 'skipped' : 'no_color'
+
+      s = format(
+        'Test Summary: %s, %s, %s',
+        format_with_color(success_color, "#{summary['passed']} successful"),
+        format_with_color(failed_color, failed_str),
+        format_with_color(skipped_color, "#{summary['skipped']} skipped"),
+      )
+
+      output(s)
+    end
+
+    def standard_controls_from_profile(profile)
+      profile[:controls].reject { |c| is_anonymous_control?(c) }
+    end
+
+    def anonymous_controls_from_profile(profile)
+      profile[:controls].select { |c| is_anonymous_control?(c) && !c[:results].nil? }
+    end
+
+    def is_anonymous_control?(control)
+      control[:id].start_with?('(generated from ')
+    end
+
+    def indent_lines(message, indentation)
+      message.lines.map { |line| ' ' * indentation + line }.join
+    end
+
+    class Control
+      IMPACT_SCORES = {
+        critical: 0.7,
+        major: 0.4,
+      }.freeze
+
+      attr_reader :data
+
+      def initialize(control_hash)
+        @data = control_hash
+      end
+
+      def id
+        data[:id]
+      end
+
+      def title
+        data[:title]
+      end
+
+      def results
+        data[:results]
+      end
+
+      def impact
+        data[:impact]
+      end
+
+      def anonymous?
+        id.start_with?('(generated from ')
+      end
+
+      def title_for_report
+        # if this is an anonymous control, just grab the resource title from any result entry
+        return results.first[:resource_title] if anonymous?
+
+        title_for_report = "#{id}: #{title || results.first[:resource_title]}"
+
+        # we will not add any additional data to the title if there's only
+        # zero or one test for this control.
+        return title_for_report if results.nil? || results.size <= 1
+
+        # append a failure summary if appropriate.
+        title_for_report += " (#{failure_count} failed)" if failure_count > 0
+        title_for_report += " (#{skipped_count} skipped)" if skipped_count > 0
+
+        title_for_report
+      end
+
+      def impact_string
+        if anonymous?
+          nil
+        elsif impact.nil?
+          'unknown'
+        elsif results&.find { |r| r[:status] == 'skipped' }
+          'skipped'
+        elsif results.nil? || results.empty? || results.all? { |r| r[:status] == 'passed' }
+          'passed'
+        elsif impact >= IMPACT_SCORES[:critical]
+          'critical'
+        elsif impact >= IMPACT_SCORES[:major]
+          'major'
+        else
+          'minor'
+        end
+      end
+
+      def impact_string_for_result(result)
+        if result[:status] == 'skipped'
+          'skipped'
+        elsif result[:status] == 'passed'
+          'passed'
+        elsif impact.nil?
+          'unknown'
+        elsif impact >= IMPACT_SCORES[:critical]
+          'critical'
+        elsif impact >= IMPACT_SCORES[:major]
+          'major'
+        else
+          'minor'
+        end
+      end
+
+      def failure_count
+        results.select { |r| r[:status] == 'failed' }.size
+      end
+
+      def skipped_count
+        results.select { |r| r[:status] == 'skipped' }.size
+      end
+    end
+  end
+end