inspec-core 5.22.50 → 6.8.1

data/lib/plugins/inspec-license/lib/inspec-license.rb

@@ -0,0 +1,14 @@
+module InspecPlugins
+  module License
+    class Plugin < ::Inspec.plugin(2)
+      plugin_name :"inspec-license"
+
+      if Inspec::Dist::EXEC_NAME == "inspec"
+        cli_command :license do
+          require_relative "inspec-license/cli"
+          InspecPlugins::License::CLI
+        end
+      end
+    end
+  end
+end

data/lib/plugins/inspec-parallel/README.md

@@ -0,0 +1,27 @@
+# Parallel Plugin
+
+Plugin to handle parallel InSpec scan operations over multiple targets.
+
+## parallel cli_command
+
+Implements the `inspec parallel exec` CLI command.
+
+## child-status Plugin
+
+This reporter is an InSpec Streaming Reporter. It is used internally by inspec parallel to provide status updates on child processes.
+
+### What This Plugin Does
+
+For each control executed, after it is complete, the plugin emits a line to STDOUT like:
+```
+12/P/24/Control Title Here
+```
+When the run is complete, the single line 'EOF_MARKER' is emitted.
+
+Where:
+
+ - 12 is the number of the control (12th seen out of all controls in all profiles)
+ - P indicates that it Passed (Also F = Failed, S = Skipped, E = Errored)
+ - 24 is the total number of controls in the run
+ - "Control Title Here" is the title (or if title is missing, id) of the last executed control
+

data/lib/plugins/inspec-parallel/inspec-parallel.gemspec

@@ -0,0 +1,6 @@
+Gem::Specification.new do |spec|
+  spec.name          = "inspec-parallel"
+  spec.summary       = "Plugin to handle parallel InSpec scan operations over multiple targets"
+  spec.description   = ""
+  spec.license       = "Apache-2.0"
+end

data/lib/plugins/inspec-parallel/lib/inspec-parallel/child_status_reporter.rb

@@ -0,0 +1,61 @@
+module InspecPlugins::Parallelism
+  class StreamingReporter < Inspec.plugin(2, :streaming_reporter)
+    # Registering these methods with RSpec::Core::Formatters class is mandatory
+    RSpec::Core::Formatters.register self, :example_passed, :example_failed, :example_pending, :close
+
+    def initialize(output)
+      @status_mapping = {}
+      @control_counter = 0
+      initialize_streaming_reporter
+    end
+
+    def example_passed(notification)
+      set_example(notification, "passed")
+    end
+
+    def example_failed(notification)
+      set_example(notification, "failed")
+    end
+
+    def example_pending(notification)
+      set_example(notification, "skipped")
+    end
+
+    def close(notification)
+      # HACK: if we've reached the end of the execution, send a special marker, to ease EOF detection on Windows
+      puts "EOF_MARKER"
+    end
+
+    private
+
+    def set_example(notification, status)
+      control_id = notification.example.metadata[:id]
+      title = notification.example.metadata[:title]
+      set_status_mapping(control_id, status)
+      output_status(control_id, title) if control_ended?(notification, control_id)
+    end
+
+    def output_status(control_id, title)
+      @control_counter += 1
+      stat = @status_mapping[control_id]
+      stat = if stat.include?("failed")
+               "F"
+             else
+               if stat.include?("skipped")
+                 "S"
+               else
+                 stat.include?("passed") ? "P" : "E"
+               end
+             end
+      display_name = title.gsub(/\n*\s+/, " ").to_s.force_encoding(Encoding::UTF_8) if title
+      display_name = control_id.to_s.lstrip.force_encoding(Encoding::UTF_8) unless title
+
+      puts "#{@control_counter}/#{stat}/#{controls_count}/#{display_name}"
+    end
+
+    def set_status_mapping(control_id, status)
+      @status_mapping[control_id] ||= []
+      @status_mapping[control_id].push(status)
+    end
+  end
+end

data/lib/plugins/inspec-parallel/lib/inspec-parallel/cli.rb

@@ -0,0 +1,39 @@
+require_relative "command"
+require "inspec/dist"
+require "inspec/base_cli"
+require "inspec/feature"
+
+module InspecPlugins::Parallelism
+  class CLI < Inspec.plugin(2, :cli_command)
+    include Inspec::Dist
+
+    subcommand_desc "parallel SUBCOMMAND [options]", "Runs #{PRODUCT_NAME} operations parallely"
+
+    desc "exec", "Executes profile parallely"
+    option :option_file, aliases: :o, type: :string, required: true,
+      desc: "File that contains list of option strings"
+    option :dry_run, type: :boolean,
+      desc: "Print commands that will run"
+    option :verbose, type: :boolean,
+      desc: "Prints all thor options on dry run"
+    option :jobs, aliases: :j, type: :numeric,
+      desc: "Number of jobs to run parallely"
+    option :ui, type: :string, default: "status",
+      desc: "Which UI to use: status, text, silent"
+    option :bg, type: :boolean,
+      desc: "Runs parallel processes in background"
+    option :log_path, type: :string,
+      desc: "Path to the runner and error logs"
+    exec_options
+    def exec(default_profile = nil)
+      Inspec.with_feature("inspec-cli-parallel-exec") {
+        parallel_cmd = InspecPlugins::Parallelism::Command.new(options, default_profile)
+        if options[:dry_run]
+          parallel_cmd.dry_run
+        else
+          parallel_cmd.run
+        end
+      }
+    end
+  end
+end

data/lib/plugins/inspec-parallel/lib/inspec-parallel/command.rb

@@ -0,0 +1,219 @@
+require_relative "runner"
+require_relative "validator"
+require "erb" unless defined?(Erb)
+
+module InspecPlugins
+  module Parallelism
+
+    class OptionFileNotReadable < RuntimeError
+    end
+
+    class Command
+      attr_accessor :cli_options_to_parallel_cmd, :default_profile, :sub_cmd, :invocations, :run_in_background
+
+      def initialize(cli_options_to_parallel_cmd, default_profile, sub_cmd = "exec")
+        @default_profile = default_profile
+        @cli_options_to_parallel_cmd = cli_options_to_parallel_cmd
+        @sub_cmd = sub_cmd
+        @logger  = Inspec::Log
+        @invocations = read_options_file
+        @run_in_background = cli_options_to_parallel_cmd["bg"]
+      end
+
+      def run
+        validate_thor_options
+        validate_invocations!
+        runner = Runner.new(invocations, cli_options_to_parallel_cmd, sub_cmd)
+        catch_ctl_c_and_exit(runner) unless run_in_background
+        runner.run
+      end
+
+      def dry_run
+        validate_invocations!
+        dry_run_commands
+      end
+
+      private
+
+      def catch_ctl_c_and_exit(runner)
+        puts "Press CTL+C to stop\n"
+        trap("SIGINT") do
+          puts "\n"
+          puts "Shutting down jobs..."
+          if Inspec.locally_windows?
+            runner.kill_child_processes
+            sleep 1
+            puts "Renaming error log files..."
+            runner.rename_error_log_files
+          end
+          exit Inspec::UI::EXIT_TERMINATED_BY_CTL_C
+        end
+      end
+
+      def validate_thor_options
+        # only log path validation needed for now
+        validate_log_path
+      end
+
+      def validate_log_path
+        error, message = Validator.new(invocations, cli_options_to_parallel_cmd, sub_cmd).validate_log_path
+        if error
+          @logger.error message
+          Inspec::UI.new.exit(:usage_error)
+        end
+      end
+
+      def validate_invocations!
+        # Validation logic stays in Validator class...
+        Validator.new(invocations, cli_options_to_parallel_cmd, sub_cmd).validate
+        # UI logic stays in Command class.
+        valid = true
+        invocations.each do |invocation_data|
+          invocation_data[:validation_errors].each do |error_message|
+            valid = false
+            @logger.error "Line #{invocation_data[:line_no]}: " + error_message
+          end
+        end
+        unless valid
+          @logger.error "Please fix the options to proceed further."
+          Inspec::UI.new.exit(:usage_error)
+        end
+      end
+
+      def dry_run_commands
+        invocations.each do |invocation_data|
+          puts "inspec #{sub_cmd} #{invocation_data[:value]}"
+        end
+      end
+
+      ## Utility functions
+
+      def read_options_file
+        opts = []
+        begin
+          content = content_from_file(cli_options_to_parallel_cmd[:option_file])
+        rescue OptionFileNotReadable => e
+          @logger.error "Cannot read options file: #{e.message}"
+          Inspec::UI.new.exit(:usage_error)
+        end
+        content.each.with_index(1) do |str, index|
+          data_hash = { line_no: index }
+          str = ERB.new(str).result_with_hash(pid: "CHILD_PID").strip
+          str_has_comment = str.start_with?("#")
+          next if str.empty? || str_has_comment
+
+          default_options = fetch_default_options(str.split(" ")).lstrip
+          if str.start_with?("-")
+            data_hash[:value] = "#{default_profile} #{str} #{default_options}"
+          else
+            data_hash[:value] = "#{str} #{default_options}"
+          end
+          opts << data_hash
+        end
+        opts
+      end
+
+      def content_from_file(option_file)
+        if File.exist?(option_file)
+          unless [".sh", ".csh", ".ps1"].include? File.extname(option_file)
+            File.readlines(option_file)
+          else
+            if Inspec.locally_windows? && (File.extname(option_file) == ".ps1")
+              begin
+                output = `powershell -File "#{option_file}"`
+                output.split("\n")
+              rescue StandardError => e
+                raise OptionFileNotReadable.new("Error reading powershell file #{option_file}: #{e.message}")
+              end
+            elsif [".sh", ".csh"].include? File.extname(option_file)
+              begin
+                output = `bash "#{option_file}"`
+                output.split("\n")
+              rescue StandardError => e
+                raise OptionFileNotReadable.new("Error reading shell file #{option_file}: #{e.message}")
+              end
+            else
+              raise OptionFileNotReadable.new("Powershell not supported in your system.")
+            end
+          end
+        else
+          raise OptionFileNotReadable.new("Option file not found.")
+        end
+      end
+
+      # this must return empty string or default option string which are not part of option file
+      def fetch_default_options(option_line)
+        option_line = option_line.select { |word| word.start_with?("-") }
+
+        # remove prefixes from the options to compare with default options
+        option_line.map! do |option_key|
+          option_key.gsub(options_prefix(option_key), "").gsub("-", "_")
+        end
+
+        default_opts = ""
+        # iterate through the parallel cli default options and append the option and value which are not present in option file
+        parallel_cmd_default_cli_options.each do |cmd|
+          if cmd.is_a? String
+            append_default_value(default_opts, cmd) unless option_line.include?(cmd)
+          elsif cmd.is_a? Array
+            if !option_line.include?(cmd[0]) && !option_line.include?(cmd[1])
+              append_default_value(default_opts, cmd[0])
+            end
+          end
+        end
+        default_opts
+      end
+
+      # returns array of default options of the subcommand
+      def parallel_cmd_default_cli_options
+        sub_cmd_opts = Inspec::InspecCLI.commands[sub_cmd].options
+        parallel_cmd_default_opts = cli_options_to_parallel_cmd.keys & sub_cmd_opts.keys.map(&:to_s)
+        options_to_append = parallel_cmd_default_opts
+
+        if cli_options_to_parallel_cmd["dry_run"] && !cli_options_to_parallel_cmd["verbose"]
+          # to not show thor default options of inspec commands in dry run
+          sub_cmd_opts_with_defaults = fetch_sub_cmd_default_options(sub_cmd_opts)
+          options_to_append -= sub_cmd_opts_with_defaults
+        end
+
+        default_opts_to_append = []
+
+        # append the options and its aliases if available.
+        options_to_append.each do |option_name|
+          opt_alias = sub_cmd_opts[option_name.to_sym].aliases
+          if opt_alias.empty?
+            default_opts_to_append << option_name
+          else
+            default_opts_to_append << [option_name, opt_alias[0].to_s]
+          end
+        end
+        default_opts_to_append
+      end
+
+      def append_default_value(default_opts, command_name)
+        default_value = cli_options_to_parallel_cmd[command_name.to_sym]
+        default_value = default_value.join(" ") if default_value.is_a? Array
+        default_opts << " --#{command_name.gsub("_", "-")} #{default_value}"
+      end
+
+      def options_prefix(option_name)
+        if option_name.start_with?("--")
+          option_name.start_with?("--no-") ? "--no-" : "--"
+        else
+          "-"
+        end
+      end
+
+      def fetch_sub_cmd_default_options(sub_cmd_opts)
+        default_options_to_remove = []
+        sub_cmd_opts_with_defaults = sub_cmd_opts.select { |_, c| !c.default.nil? }.keys.map(&:to_s)
+        sub_cmd_opts_with_defaults.each do |default_opt_name|
+          if sub_cmd_opts[default_opt_name.to_sym].default == cli_options_to_parallel_cmd[default_opt_name]
+            default_options_to_remove << default_opt_name
+          end
+        end
+        default_options_to_remove
+      end
+    end
+  end
+end

data/lib/plugins/inspec-parallel/lib/inspec-parallel/runner.rb

@@ -0,0 +1,270 @@
+require "inspec/cli"
+require "concurrent"
+require_relative "super_reporter/base"
+
+module InspecPlugins
+  module Parallelism
+    class Runner
+      attr_accessor :invocations, :sub_cmd, :total_jobs, :run_in_background, :log_path
+
+      def initialize(invocations, cli_options, sub_cmd = "exec")
+        @invocations = invocations
+        @sub_cmd = sub_cmd
+        @total_jobs = cli_options["jobs"] || Concurrent.physical_processor_count
+        @child_tracker = {}
+        @child_tracker_persisted = {}
+        @run_in_background = cli_options["bg"]
+        unless run_in_background
+          @ui = InspecPlugins::Parallelism::SuperReporter.make(cli_options["ui"], total_jobs, invocations)
+        end
+        @log_path = cli_options["log_path"]
+      end
+
+      def run
+        initiate_background_run if run_in_background # running a process as daemon changes parent process pid
+        original_stdout_stream = ChefLicensing::Config.output
+        until invocations.empty? && @child_tracker.empty?
+          # Changing output to STDERR to avoid the output interruption between runs
+          ChefLicensing::Config.output = STDERR
+          while should_start_more_jobs?
+            if Inspec.locally_windows?
+              spawn_another_process
+            else
+              fork_another_process
+            end
+          end
+
+          update_ui_poll_select
+          cleanup_child_processes
+          sleep 0.1
+        end
+        # Reset output to the original STDOUT stream as a safe measure.
+        ChefLicensing::Config.output = original_stdout_stream
+
+        # Requires renaming operations on windows only
+        # Do Rename and delete operations after all child processes have exited successfully
+        rename_error_log_files if Inspec.locally_windows?
+        cleanup_empty_error_log_files
+        cleanup_daemon_process if run_in_background
+      end
+
+      def initiate_background_run
+        if Inspec.locally_windows?
+          Inspec::UI.new.exit(:usage_error)
+        else
+          Process.daemon(true, true)
+        end
+      end
+
+      def cleanup_daemon_process
+        current_process_id = Process.pid
+        Process.kill(9, current_process_id)
+        # DO NOT TRY TO REFACTOR IT THIS WAY
+        # Calling Process.kill(9,Process.pid) kills the "stopper" process itself, rather than the one it's trying to stop.
+      end
+
+      def cleanup_empty_error_log_files
+        logs_dir_path = log_path || Dir.pwd
+        error_files = Dir.glob("#{logs_dir_path}/logs/*.err")
+        error_files.each do |error_file|
+          if File.exist?(error_file) && !File.size?(error_file)
+            File.delete(error_file)
+          end
+        end
+      end
+
+      def kill_child_processes
+        @child_tracker.each do |pid, info|
+          Process.kill("SIGKILL", pid)
+        rescue Exception => e
+          $stderr.puts "Error while shutting down process #{pid}: #{e.message}"
+        end
+        # Waiting for child processes to die after they have been killed
+        wait_for_child_processes_to_die
+      end
+
+      def wait_for_child_processes_to_die
+        until @child_tracker.empty?
+          begin
+            exited_pid = Process.waitpid(-1, Process::WNOHANG)
+            @child_tracker.delete exited_pid if exited_pid && exited_pid > 0
+            sleep 1
+          rescue Errno::ECHILD
+            Inspec::Log.info "Processes shutdown complete!"
+          rescue Exception => e
+            Inspec::Log.debug "Error while waiting for child processes to shutdown: #{e.message}"
+          end
+        end
+      end
+
+      def rename_error_log_files
+        @child_tracker_persisted.each do |pid, info|
+          rename_error_log(info[:error_log_file], pid)
+        end
+      end
+
+      def should_start_more_jobs?
+        @child_tracker.length < total_jobs && !invocations.empty?
+      end
+
+      def spawn_another_process
+        invocation = invocations.shift[:value]
+
+        child_reader, parent_writer = IO.pipe
+        begin
+          logs_dir_path = log_path || Dir.pwd
+          log_dir = File.join(logs_dir_path, "logs")
+          FileUtils.mkdir_p(log_dir)
+          error_log_file = File.open("#{log_dir}/#{Time.now.nsec}.err", "a+")
+          cmd = "#{$0} #{sub_cmd} #{invocation}"
+          log_msg = "#{Time.now.iso8601} Start Time: #{Time.now}\n#{Time.now.iso8601} Arguments: #{invocation}\n"
+          child_pid = Process.spawn(cmd, out: parent_writer, err: error_log_file.path)
+
+          # Logging
+          create_logs(child_pid, log_msg)
+          @child_tracker[child_pid] = { io: child_reader }
+
+          # This is used to rename error log files after all child processes are exited
+          @child_tracker_persisted[child_pid] = { error_log_file: error_log_file }
+          @ui.child_spawned(child_pid, invocation)
+
+          # Close the file to unlock the error log files opened by processes
+          error_log_file.close
+        rescue StandardError => e
+          $stderr.puts "#{Time.now.iso8601} Error Message: #{e.message}"
+          $stderr.puts "#{Time.now.iso8601} Error Backtrace: #{e.backtrace}"
+        end
+      end
+
+      def fork_another_process
+        invocation = invocations.shift[:value] # Be sure to do this shift() in parent process
+        # thing_that_reads_from_the_child, thing_that_writes_to_the_parent = IO.pipe
+        child_reader, parent_writer = IO.pipe
+        if (child_pid = Process.fork)
+          # In parent with newly forked child
+          parent_writer.close
+          @child_tracker[child_pid] = { io: child_reader }
+          @ui.child_forked(child_pid, invocation) unless run_in_background
+        else
+          # In child
+          child_reader.close
+          # replace stdout with writer
+          $stdout = parent_writer
+          create_logs(Process.pid, nil, $stderr)
+
+          begin
+            create_logs(
+              Process.pid,
+              "#{Time.now.iso8601} Start Time: #{Time.now}\n#{Time.now.iso8601} Arguments: #{invocation}\n"
+            )
+            runner_invocation(invocation)
+          rescue StandardError => e
+            $stderr.puts "#{Time.now.iso8601} Error Message: #{e.message}"
+            $stderr.puts "#{Time.now.iso8601} Error Backtrace: #{e.backtrace}"
+          end
+
+          # should be unreachable but child MUST exit
+          exit(42)
+        end
+      end
+
+      # Still in parent
+      # Loop over children and check for finished processes
+      def cleanup_child_processes
+        @child_tracker.each do |pid, info|
+          if Process.wait(pid, Process::WNOHANG)
+            # Expect to (probably) find EOF marker on the pipe, and close it if so
+            update_ui_poll_select(pid)
+
+            create_logs(pid, "#{Time.now.iso8601} Exit code: #{$?}\n")
+
+            # child exited - status in $?
+            @ui.child_exited(pid) unless run_in_background
+            @child_tracker.delete pid
+          end
+        end
+      end
+
+      def update_ui_poll_select(target_pid = nil)
+        # Focus on one pid's pipe if specified, otherwise poll all pipes
+        pipes_for_reading = target_pid ? [ @child_tracker[target_pid][:io] ] : @child_tracker.values.map { |i| i[:io] }
+        # Next line is due to a race between the close() and the wait()... shouldn't need it, but it fixes the race.
+        pipes_for_reading.reject!(&:closed?)
+        ready_pipes = IO.select(pipes_for_reading, [], [], 0.1)
+        return unless ready_pipes
+
+        ready_pipes[0].each do |pipe_ready_for_reading|
+          # If we weren't provided a PID, hackishly look up the pid from the matching IO.
+          pid = target_pid || @child_tracker.keys.detect { |p| @child_tracker[p][:io] == pipe_ready_for_reading }
+          begin
+            while (update_line = pipe_ready_for_reading.readline) && !pipe_ready_for_reading.closed?
+              if update_line =~ /EOF_MARKER/
+                pipe_ready_for_reading.close
+                break
+              elsif update_line =~ /WARN/ || update_line =~ /ERROR/ || update_line =~ /INFO/
+                create_logs(
+                  pid,
+                  "#{Time.now.iso8601} Extra log: #{update_line}\n"
+                )
+                break
+              end
+              update_ui_with_line(pid, update_line) unless run_in_background
+              # Only pull one line if we are doing normal updates; slurp the whole file
+              # if we are doing a final pull on a targeted PID
+              break unless target_pid
+            end
+          rescue EOFError
+            # On unix, readline throws an EOFError when we hit the end. On Windows, nothing apparently happens.
+            pipe_ready_for_reading.close
+            next
+          end
+        end
+        # TODO: loop over ready_pipes[2] and handle errors?
+      end
+
+      def update_ui_with_line(pid, update_line)
+        @ui.child_status_update_line(pid, update_line)
+      end
+
+      private
+
+      def runner_invocation(runner_option)
+        splitted_result = runner_option.split(" ")
+        profile_to_run = splitted_result[0]
+        splitted_result.delete_at(0)
+
+        # thor invocation
+        arguments = [sub_cmd, profile_to_run, splitted_result].flatten
+        Inspec::InspecCLI.start(arguments, enforce_license: true)
+      end
+
+      def create_logs(child_pid, run_log , stderr = nil)
+        logs_dir_path = log_path || Dir.pwd
+        log_dir = File.join(logs_dir_path, "logs")
+        FileUtils.mkdir_p(log_dir)
+
+        if stderr
+          log_file = File.join(log_dir, "#{child_pid}.err") unless File.exist?("#{child_pid}.err")
+          stderr.reopen(log_file, "a")
+        else
+          log_file = File.join(log_dir, "#{child_pid}.log") unless File.exist?("#{child_pid}.log")
+          File.write(log_file, run_log, mode: "a")
+        end
+      end
+
+      def rename_error_log(error_log_file, child_pid)
+        logs_dir_path = log_path || Dir.pwd
+        log_dir = File.join(logs_dir_path, "logs")
+        FileUtils.mkdir_p(log_dir)
+
+        if error_log_file.closed? && File.exist?(error_log_file.path)
+          begin
+            File.rename("#{error_log_file.path}", "#{log_dir}/#{child_pid}.err")
+          rescue
+            $stderr.puts "Cannot rename error log file #{error_log_file.path} for child pid #{child_pid}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/plugins/inspec-parallel/lib/inspec-parallel/super_reporter/base.rb

@@ -0,0 +1,24 @@
+module InspecPlugins::Parallelism
+  class SuperReporter
+
+    def self.make(type, job_count, invocations)
+      Object.const_get("InspecPlugins::Parallelism::SuperReporter::" + (type[0].upcase + type[1..-1])).new(job_count, invocations)
+    end
+
+    class Base
+      def initialize(job_count, invocations); end
+
+      def child_spawned(pid, invocation); end
+
+      def child_forked(pid, invocation); end
+
+      def child_exited(pid); end
+
+      def child_status_update_line(pid, update_line); end
+    end
+
+    require_relative "text"
+    require_relative "status"
+    require_relative "silent"
+  end
+end

data/lib/plugins/inspec-parallel/lib/inspec-parallel/super_reporter/silent.rb

@@ -0,0 +1,7 @@
+module InspecPlugins::Parallelism
+  class SuperReporter
+    class Silent < InspecPlugins::Parallelism::SuperReporter::Base
+      # This is a silent super reporter with no reporting functionality.
+    end
+  end
+end