inspec-core 5.22.29 → 6.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Chef-EULA +9 -0
- data/Gemfile +10 -1
- data/etc/features.sig +6 -0
- data/etc/features.yaml +94 -0
- data/inspec-core.gemspec +14 -5
- data/lib/inspec/backend.rb +2 -0
- data/lib/inspec/base_cli.rb +80 -4
- data/lib/inspec/cached_fetcher.rb +24 -3
- data/lib/inspec/cli.rb +300 -230
- data/lib/inspec/config.rb +24 -2
- data/lib/inspec/dependencies/cache.rb +33 -0
- data/lib/inspec/enhanced_outcomes.rb +1 -0
- data/lib/inspec/errors.rb +5 -0
- data/lib/inspec/exceptions.rb +2 -0
- data/lib/inspec/feature/config.rb +75 -0
- data/lib/inspec/feature/runner.rb +26 -0
- data/lib/inspec/feature.rb +34 -0
- data/lib/inspec/fetcher/git.rb +5 -0
- data/lib/inspec/globals.rb +6 -0
- data/lib/inspec/plugin/v1/plugin_types/fetcher.rb +7 -0
- data/lib/inspec/plugin/v2/plugin_types/streaming_reporter.rb +30 -2
- data/lib/inspec/profile.rb +373 -12
- data/lib/inspec/reporters/cli.rb +1 -1
- data/lib/inspec/reporters.rb +67 -54
- data/lib/inspec/resources/security_policy.rb +7 -2
- data/lib/inspec/run_data.rb +7 -5
- data/lib/inspec/runner.rb +34 -5
- data/lib/inspec/runner_rspec.rb +12 -9
- data/lib/inspec/secrets/yaml.rb +9 -3
- data/lib/inspec/shell.rb +10 -0
- data/lib/inspec/ui.rb +4 -0
- data/lib/inspec/utils/licensing_config.rb +9 -0
- data/lib/inspec/utils/profile_ast_helpers.rb +372 -0
- data/lib/inspec/version.rb +1 -1
- data/lib/inspec/waiver_file_reader.rb +68 -27
- data/lib/inspec.rb +2 -1
- data/lib/plugins/inspec-compliance/lib/inspec-compliance/cli.rb +189 -168
- data/lib/plugins/inspec-habitat/lib/inspec-habitat/cli.rb +10 -3
- data/lib/plugins/inspec-init/lib/inspec-init/cli.rb +1 -0
- data/lib/plugins/inspec-init/lib/inspec-init/cli_plugin.rb +23 -21
- data/lib/plugins/inspec-init/lib/inspec-init/cli_profile.rb +15 -13
- data/lib/plugins/inspec-init/lib/inspec-init/cli_resource.rb +15 -13
- data/lib/plugins/inspec-license/README.md +16 -0
- data/lib/plugins/inspec-license/inspec-license.gemspec +6 -0
- data/lib/plugins/inspec-license/lib/inspec-license/cli.rb +26 -0
- data/lib/plugins/inspec-license/lib/inspec-license.rb +14 -0
- data/lib/plugins/inspec-parallel/README.md +27 -0
- data/lib/plugins/inspec-parallel/inspec-parallel.gemspec +6 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/child_status_reporter.rb +61 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/cli.rb +39 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/command.rb +219 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/runner.rb +265 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/super_reporter/base.rb +24 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/super_reporter/silent.rb +7 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/super_reporter/status.rb +124 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/super_reporter/text.rb +23 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel/validator.rb +170 -0
- data/lib/plugins/inspec-parallel/lib/inspec-parallel.rb +18 -0
- data/lib/plugins/inspec-reporter-html2/templates/control.html.erb +7 -6
- data/lib/plugins/inspec-reporter-html2/templates/default.js +6 -6
- data/lib/plugins/inspec-sign/lib/inspec-sign/base.rb +6 -2
- data/lib/plugins/inspec-sign/lib/inspec-sign/cli.rb +11 -4
- data/lib/plugins/inspec-streaming-reporter-progress-bar/lib/inspec-streaming-reporter-progress-bar/streaming_reporter.rb +6 -13
- metadata +54 -13
@@ -0,0 +1,26 @@
|
|
1
|
+
require "chef-licensing"
|
2
|
+
module InspecPlugins::License
|
3
|
+
class CLI < Inspec.plugin(2, :cli_command)
|
4
|
+
include Inspec::Dist
|
5
|
+
|
6
|
+
subcommand_desc "license SUBCOMMAND [options]", "Manage #{PRODUCT_NAME} license"
|
7
|
+
desc "list", "List licenses (not applicable to local licensing service)"
|
8
|
+
def list
|
9
|
+
ChefLicensing.list_license_keys_info
|
10
|
+
rescue ChefLicensing::Error => e
|
11
|
+
Inspec::Log.error e.message
|
12
|
+
Inspec::UI.new.exit(Inspec::UI::EXIT_LICENSE_NOT_SET)
|
13
|
+
end
|
14
|
+
|
15
|
+
desc "add", "Add a new license (not applicable to local licensing service)"
|
16
|
+
def add
|
17
|
+
ChefLicensing.add_license
|
18
|
+
rescue ChefLicensing::LicenseKeyFetcher::LicenseKeyAddNotAllowed => e
|
19
|
+
Inspec::Log.error e.message
|
20
|
+
Inspec::UI.new.exit(Inspec::UI::EXIT_LICENSE_NOT_SET)
|
21
|
+
rescue ChefLicensing::Error => e
|
22
|
+
Inspec::Log.error e.message
|
23
|
+
Inspec::UI.new.exit(Inspec::UI::EXIT_LICENSE_NOT_SET)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module InspecPlugins
|
2
|
+
module License
|
3
|
+
class Plugin < ::Inspec.plugin(2)
|
4
|
+
plugin_name :"inspec-license"
|
5
|
+
|
6
|
+
if Inspec::Dist::EXEC_NAME == "inspec"
|
7
|
+
cli_command :license do
|
8
|
+
require_relative "inspec-license/cli"
|
9
|
+
InspecPlugins::License::CLI
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# Parallel Plugin
|
2
|
+
|
3
|
+
Plugin to handle parallel InSpec scan operations over multiple targets.
|
4
|
+
|
5
|
+
## parallel cli_command
|
6
|
+
|
7
|
+
Implements the `inspec parallel exec` CLI command.
|
8
|
+
|
9
|
+
## child-status Plugin
|
10
|
+
|
11
|
+
This reporter is an InSpec Streaming Reporter. It is used internally by inspec parallel to provide status updates on child processes.
|
12
|
+
|
13
|
+
### What This Plugin Does
|
14
|
+
|
15
|
+
For each control executed, after it is complete, the plugin emits a line to STDOUT like:
|
16
|
+
```
|
17
|
+
12/P/24/Control Title Here
|
18
|
+
```
|
19
|
+
When the run is complete, the single line 'EOF_MARKER' is emitted.
|
20
|
+
|
21
|
+
Where:
|
22
|
+
|
23
|
+
- 12 is the number of the control (12th seen out of all controls in all profiles)
|
24
|
+
- P indicates that it Passed (Also F = Failed, S = Skipped, E = Errored)
|
25
|
+
- 24 is the total number of controls in the run
|
26
|
+
- "Control Title Here" is the title (or if title is missing, id) of the last executed control
|
27
|
+
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module InspecPlugins::Parallelism
|
2
|
+
class StreamingReporter < Inspec.plugin(2, :streaming_reporter)
|
3
|
+
# Registering these methods with RSpec::Core::Formatters class is mandatory
|
4
|
+
RSpec::Core::Formatters.register self, :example_passed, :example_failed, :example_pending, :close
|
5
|
+
|
6
|
+
def initialize(output)
|
7
|
+
@status_mapping = {}
|
8
|
+
@control_counter = 0
|
9
|
+
initialize_streaming_reporter
|
10
|
+
end
|
11
|
+
|
12
|
+
def example_passed(notification)
|
13
|
+
set_example(notification, "passed")
|
14
|
+
end
|
15
|
+
|
16
|
+
def example_failed(notification)
|
17
|
+
set_example(notification, "failed")
|
18
|
+
end
|
19
|
+
|
20
|
+
def example_pending(notification)
|
21
|
+
set_example(notification, "skipped")
|
22
|
+
end
|
23
|
+
|
24
|
+
def close(notification)
|
25
|
+
# HACK: if we've reached the end of the execution, send a special marker, to ease EOF detection on Windows
|
26
|
+
puts "EOF_MARKER"
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def set_example(notification, status)
|
32
|
+
control_id = notification.example.metadata[:id]
|
33
|
+
title = notification.example.metadata[:title]
|
34
|
+
set_status_mapping(control_id, status)
|
35
|
+
output_status(control_id, title) if control_ended?(notification, control_id)
|
36
|
+
end
|
37
|
+
|
38
|
+
def output_status(control_id, title)
|
39
|
+
@control_counter += 1
|
40
|
+
stat = @status_mapping[control_id]
|
41
|
+
stat = if stat.include?("failed")
|
42
|
+
"F"
|
43
|
+
else
|
44
|
+
if stat.include?("skipped")
|
45
|
+
"S"
|
46
|
+
else
|
47
|
+
stat.include?("passed") ? "P" : "E"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
display_name = title.gsub(/\n*\s+/, " ").to_s.force_encoding(Encoding::UTF_8) if title
|
51
|
+
display_name = control_id.to_s.lstrip.force_encoding(Encoding::UTF_8) unless title
|
52
|
+
|
53
|
+
puts "#{@control_counter}/#{stat}/#{controls_count}/#{display_name}"
|
54
|
+
end
|
55
|
+
|
56
|
+
def set_status_mapping(control_id, status)
|
57
|
+
@status_mapping[control_id] ||= []
|
58
|
+
@status_mapping[control_id].push(status)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require_relative "command"
|
2
|
+
require "inspec/dist"
|
3
|
+
require "inspec/base_cli"
|
4
|
+
require "inspec/feature"
|
5
|
+
|
6
|
+
module InspecPlugins::Parallelism
|
7
|
+
class CLI < Inspec.plugin(2, :cli_command)
|
8
|
+
include Inspec::Dist
|
9
|
+
|
10
|
+
subcommand_desc "parallel SUBCOMMAND [options]", "Runs #{PRODUCT_NAME} operations parallely"
|
11
|
+
|
12
|
+
desc "exec", "Executes profile parallely"
|
13
|
+
option :option_file, aliases: :o, type: :string, required: true,
|
14
|
+
desc: "File that contains list of option strings"
|
15
|
+
option :dry_run, type: :boolean,
|
16
|
+
desc: "Print commands that will run"
|
17
|
+
option :verbose, type: :boolean,
|
18
|
+
desc: "Prints all thor options on dry run"
|
19
|
+
option :jobs, aliases: :j, type: :numeric,
|
20
|
+
desc: "Number of jobs to run parallely"
|
21
|
+
option :ui, type: :string, default: "status",
|
22
|
+
desc: "Which UI to use: status, text, silent"
|
23
|
+
option :bg, type: :boolean,
|
24
|
+
desc: "Runs parallel processes in background"
|
25
|
+
option :log_path, type: :string,
|
26
|
+
desc: "Path to the runner and error logs"
|
27
|
+
exec_options
|
28
|
+
def exec(default_profile = nil)
|
29
|
+
Inspec.with_feature("inspec-cli-parallel-exec") {
|
30
|
+
parallel_cmd = InspecPlugins::Parallelism::Command.new(options, default_profile)
|
31
|
+
if options[:dry_run]
|
32
|
+
parallel_cmd.dry_run
|
33
|
+
else
|
34
|
+
parallel_cmd.run
|
35
|
+
end
|
36
|
+
}
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,219 @@
|
|
1
|
+
require_relative "runner"
|
2
|
+
require_relative "validator"
|
3
|
+
require "erb" unless defined?(Erb)
|
4
|
+
|
5
|
+
module InspecPlugins
|
6
|
+
module Parallelism
|
7
|
+
|
8
|
+
class OptionFileNotReadable < RuntimeError
|
9
|
+
end
|
10
|
+
|
11
|
+
class Command
|
12
|
+
attr_accessor :cli_options_to_parallel_cmd, :default_profile, :sub_cmd, :invocations, :run_in_background
|
13
|
+
|
14
|
+
def initialize(cli_options_to_parallel_cmd, default_profile, sub_cmd = "exec")
|
15
|
+
@default_profile = default_profile
|
16
|
+
@cli_options_to_parallel_cmd = cli_options_to_parallel_cmd
|
17
|
+
@sub_cmd = sub_cmd
|
18
|
+
@logger = Inspec::Log
|
19
|
+
@invocations = read_options_file
|
20
|
+
@run_in_background = cli_options_to_parallel_cmd["bg"]
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
validate_thor_options
|
25
|
+
validate_invocations!
|
26
|
+
runner = Runner.new(invocations, cli_options_to_parallel_cmd, sub_cmd)
|
27
|
+
catch_ctl_c_and_exit(runner) unless run_in_background
|
28
|
+
runner.run
|
29
|
+
end
|
30
|
+
|
31
|
+
def dry_run
|
32
|
+
validate_invocations!
|
33
|
+
dry_run_commands
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def catch_ctl_c_and_exit(runner)
|
39
|
+
puts "Press CTL+C to stop\n"
|
40
|
+
trap("SIGINT") do
|
41
|
+
puts "\n"
|
42
|
+
puts "Shutting down jobs..."
|
43
|
+
if Inspec.locally_windows?
|
44
|
+
runner.kill_child_processes
|
45
|
+
sleep 1
|
46
|
+
puts "Renaming error log files..."
|
47
|
+
runner.rename_error_log_files
|
48
|
+
end
|
49
|
+
exit Inspec::UI::EXIT_TERMINATED_BY_CTL_C
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def validate_thor_options
|
54
|
+
# only log path validation needed for now
|
55
|
+
validate_log_path
|
56
|
+
end
|
57
|
+
|
58
|
+
def validate_log_path
|
59
|
+
error, message = Validator.new(invocations, cli_options_to_parallel_cmd, sub_cmd).validate_log_path
|
60
|
+
if error
|
61
|
+
@logger.error message
|
62
|
+
Inspec::UI.new.exit(:usage_error)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def validate_invocations!
|
67
|
+
# Validation logic stays in Validator class...
|
68
|
+
Validator.new(invocations, cli_options_to_parallel_cmd, sub_cmd).validate
|
69
|
+
# UI logic stays in Command class.
|
70
|
+
valid = true
|
71
|
+
invocations.each do |invocation_data|
|
72
|
+
invocation_data[:validation_errors].each do |error_message|
|
73
|
+
valid = false
|
74
|
+
@logger.error "Line #{invocation_data[:line_no]}: " + error_message
|
75
|
+
end
|
76
|
+
end
|
77
|
+
unless valid
|
78
|
+
@logger.error "Please fix the options to proceed further."
|
79
|
+
Inspec::UI.new.exit(:usage_error)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def dry_run_commands
|
84
|
+
invocations.each do |invocation_data|
|
85
|
+
puts "inspec #{sub_cmd} #{invocation_data[:value]}"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
## Utility functions
|
90
|
+
|
91
|
+
def read_options_file
|
92
|
+
opts = []
|
93
|
+
begin
|
94
|
+
content = content_from_file(cli_options_to_parallel_cmd[:option_file])
|
95
|
+
rescue OptionFileNotReadable => e
|
96
|
+
@logger.error "Cannot read options file: #{e.message}"
|
97
|
+
Inspec::UI.new.exit(:usage_error)
|
98
|
+
end
|
99
|
+
content.each.with_index(1) do |str, index|
|
100
|
+
data_hash = { line_no: index }
|
101
|
+
str = ERB.new(str).result_with_hash(pid: "CHILD_PID").strip
|
102
|
+
str_has_comment = str.start_with?("#")
|
103
|
+
next if str.empty? || str_has_comment
|
104
|
+
|
105
|
+
default_options = fetch_default_options(str.split(" ")).lstrip
|
106
|
+
if str.start_with?("-")
|
107
|
+
data_hash[:value] = "#{default_profile} #{str} #{default_options}"
|
108
|
+
else
|
109
|
+
data_hash[:value] = "#{str} #{default_options}"
|
110
|
+
end
|
111
|
+
opts << data_hash
|
112
|
+
end
|
113
|
+
opts
|
114
|
+
end
|
115
|
+
|
116
|
+
def content_from_file(option_file)
|
117
|
+
if File.exist?(option_file)
|
118
|
+
unless [".sh", ".csh", ".ps1"].include? File.extname(option_file)
|
119
|
+
File.readlines(option_file)
|
120
|
+
else
|
121
|
+
if Inspec.locally_windows? && (File.extname(option_file) == ".ps1")
|
122
|
+
begin
|
123
|
+
output = `powershell -File "#{option_file}"`
|
124
|
+
output.split("\n")
|
125
|
+
rescue StandardError => e
|
126
|
+
raise OptionFileNotReadable.new("Error reading powershell file #{option_file}: #{e.message}")
|
127
|
+
end
|
128
|
+
elsif [".sh", ".csh"].include? File.extname(option_file)
|
129
|
+
begin
|
130
|
+
output = `bash "#{option_file}"`
|
131
|
+
output.split("\n")
|
132
|
+
rescue StandardError => e
|
133
|
+
raise OptionFileNotReadable.new("Error reading shell file #{option_file}: #{e.message}")
|
134
|
+
end
|
135
|
+
else
|
136
|
+
raise OptionFileNotReadable.new("Powershell not supported in your system.")
|
137
|
+
end
|
138
|
+
end
|
139
|
+
else
|
140
|
+
raise OptionFileNotReadable.new("Option file not found.")
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
# this must return empty string or default option string which are not part of option file
|
145
|
+
def fetch_default_options(option_line)
|
146
|
+
option_line = option_line.select { |word| word.start_with?("-") }
|
147
|
+
|
148
|
+
# remove prefixes from the options to compare with default options
|
149
|
+
option_line.map! do |option_key|
|
150
|
+
option_key.gsub(options_prefix(option_key), "").gsub("-", "_")
|
151
|
+
end
|
152
|
+
|
153
|
+
default_opts = ""
|
154
|
+
# iterate through the parallel cli default options and append the option and value which are not present in option file
|
155
|
+
parallel_cmd_default_cli_options.each do |cmd|
|
156
|
+
if cmd.is_a? String
|
157
|
+
append_default_value(default_opts, cmd) unless option_line.include?(cmd)
|
158
|
+
elsif cmd.is_a? Array
|
159
|
+
if !option_line.include?(cmd[0]) && !option_line.include?(cmd[1])
|
160
|
+
append_default_value(default_opts, cmd[0])
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
default_opts
|
165
|
+
end
|
166
|
+
|
167
|
+
# returns array of default options of the subcommand
|
168
|
+
def parallel_cmd_default_cli_options
|
169
|
+
sub_cmd_opts = Inspec::InspecCLI.commands[sub_cmd].options
|
170
|
+
parallel_cmd_default_opts = cli_options_to_parallel_cmd.keys & sub_cmd_opts.keys.map(&:to_s)
|
171
|
+
options_to_append = parallel_cmd_default_opts
|
172
|
+
|
173
|
+
if cli_options_to_parallel_cmd["dry_run"] && !cli_options_to_parallel_cmd["verbose"]
|
174
|
+
# to not show thor default options of inspec commands in dry run
|
175
|
+
sub_cmd_opts_with_defaults = fetch_sub_cmd_default_options(sub_cmd_opts)
|
176
|
+
options_to_append -= sub_cmd_opts_with_defaults
|
177
|
+
end
|
178
|
+
|
179
|
+
default_opts_to_append = []
|
180
|
+
|
181
|
+
# append the options and its aliases if available.
|
182
|
+
options_to_append.each do |option_name|
|
183
|
+
opt_alias = sub_cmd_opts[option_name.to_sym].aliases
|
184
|
+
if opt_alias.empty?
|
185
|
+
default_opts_to_append << option_name
|
186
|
+
else
|
187
|
+
default_opts_to_append << [option_name, opt_alias[0].to_s]
|
188
|
+
end
|
189
|
+
end
|
190
|
+
default_opts_to_append
|
191
|
+
end
|
192
|
+
|
193
|
+
def append_default_value(default_opts, command_name)
|
194
|
+
default_value = cli_options_to_parallel_cmd[command_name.to_sym]
|
195
|
+
default_value = default_value.join(" ") if default_value.is_a? Array
|
196
|
+
default_opts << " --#{command_name.gsub("_", "-")} #{default_value}"
|
197
|
+
end
|
198
|
+
|
199
|
+
def options_prefix(option_name)
|
200
|
+
if option_name.start_with?("--")
|
201
|
+
option_name.start_with?("--no-") ? "--no-" : "--"
|
202
|
+
else
|
203
|
+
"-"
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
def fetch_sub_cmd_default_options(sub_cmd_opts)
|
208
|
+
default_options_to_remove = []
|
209
|
+
sub_cmd_opts_with_defaults = sub_cmd_opts.select { |_, c| !c.default.nil? }.keys.map(&:to_s)
|
210
|
+
sub_cmd_opts_with_defaults.each do |default_opt_name|
|
211
|
+
if sub_cmd_opts[default_opt_name.to_sym].default == cli_options_to_parallel_cmd[default_opt_name]
|
212
|
+
default_options_to_remove << default_opt_name
|
213
|
+
end
|
214
|
+
end
|
215
|
+
default_options_to_remove
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
@@ -0,0 +1,265 @@
|
|
1
|
+
require "inspec/cli"
|
2
|
+
require "concurrent"
|
3
|
+
require_relative "super_reporter/base"
|
4
|
+
|
5
|
+
module InspecPlugins
|
6
|
+
module Parallelism
|
7
|
+
class Runner
|
8
|
+
attr_accessor :invocations, :sub_cmd, :total_jobs, :run_in_background, :log_path
|
9
|
+
|
10
|
+
def initialize(invocations, cli_options, sub_cmd = "exec")
|
11
|
+
@invocations = invocations
|
12
|
+
@sub_cmd = sub_cmd
|
13
|
+
@total_jobs = cli_options["jobs"] || Concurrent.physical_processor_count
|
14
|
+
@child_tracker = {}
|
15
|
+
@child_tracker_persisted = {}
|
16
|
+
@run_in_background = cli_options["bg"]
|
17
|
+
unless run_in_background
|
18
|
+
@ui = InspecPlugins::Parallelism::SuperReporter.make(cli_options["ui"], total_jobs, invocations)
|
19
|
+
end
|
20
|
+
@log_path = cli_options["log_path"]
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
initiate_background_run if run_in_background # running a process as daemon changes parent process pid
|
25
|
+
until invocations.empty? && @child_tracker.empty?
|
26
|
+
while should_start_more_jobs?
|
27
|
+
if Inspec.locally_windows?
|
28
|
+
spawn_another_process
|
29
|
+
else
|
30
|
+
fork_another_process
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
update_ui_poll_select
|
35
|
+
cleanup_child_processes
|
36
|
+
sleep 0.1
|
37
|
+
end
|
38
|
+
|
39
|
+
# Requires renaming operations on windows only
|
40
|
+
# Do Rename and delete operations after all child processes have exited successfully
|
41
|
+
rename_error_log_files if Inspec.locally_windows?
|
42
|
+
cleanup_empty_error_log_files
|
43
|
+
cleanup_daemon_process if run_in_background
|
44
|
+
end
|
45
|
+
|
46
|
+
def initiate_background_run
|
47
|
+
if Inspec.locally_windows?
|
48
|
+
Inspec::UI.new.exit(:usage_error)
|
49
|
+
else
|
50
|
+
Process.daemon(true, true)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def cleanup_daemon_process
|
55
|
+
current_process_id = Process.pid
|
56
|
+
Process.kill(9, current_process_id)
|
57
|
+
# DO NOT TRY TO REFACTOR IT THIS WAY
|
58
|
+
# Calling Process.kill(9,Process.pid) kills the "stopper" process itself, rather than the one it's trying to stop.
|
59
|
+
end
|
60
|
+
|
61
|
+
def cleanup_empty_error_log_files
|
62
|
+
logs_dir_path = log_path || Dir.pwd
|
63
|
+
error_files = Dir.glob("#{logs_dir_path}/logs/*.err")
|
64
|
+
error_files.each do |error_file|
|
65
|
+
if File.exist?(error_file) && !File.size?(error_file)
|
66
|
+
File.delete(error_file)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def kill_child_processes
|
72
|
+
@child_tracker.each do |pid, info|
|
73
|
+
Process.kill("SIGKILL", pid)
|
74
|
+
rescue Exception => e
|
75
|
+
$stderr.puts "Error while shutting down process #{pid}: #{e.message}"
|
76
|
+
end
|
77
|
+
# Waiting for child processes to die after they have been killed
|
78
|
+
wait_for_child_processes_to_die
|
79
|
+
end
|
80
|
+
|
81
|
+
def wait_for_child_processes_to_die
|
82
|
+
until @child_tracker.empty?
|
83
|
+
begin
|
84
|
+
exited_pid = Process.waitpid(-1, Process::WNOHANG)
|
85
|
+
@child_tracker.delete exited_pid if exited_pid && exited_pid > 0
|
86
|
+
sleep 1
|
87
|
+
rescue Errno::ECHILD
|
88
|
+
Inspec::Log.info "Processes shutdown complete!"
|
89
|
+
rescue Exception => e
|
90
|
+
Inspec::Log.debug "Error while waiting for child processes to shutdown: #{e.message}"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def rename_error_log_files
|
96
|
+
@child_tracker_persisted.each do |pid, info|
|
97
|
+
rename_error_log(info[:error_log_file], pid)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def should_start_more_jobs?
|
102
|
+
@child_tracker.length < total_jobs && !invocations.empty?
|
103
|
+
end
|
104
|
+
|
105
|
+
def spawn_another_process
|
106
|
+
invocation = invocations.shift[:value]
|
107
|
+
|
108
|
+
child_reader, parent_writer = IO.pipe
|
109
|
+
begin
|
110
|
+
logs_dir_path = log_path || Dir.pwd
|
111
|
+
log_dir = File.join(logs_dir_path, "logs")
|
112
|
+
FileUtils.mkdir_p(log_dir)
|
113
|
+
error_log_file = File.open("#{log_dir}/#{Time.now.nsec}.err", "a+")
|
114
|
+
cmd = "#{$0} #{sub_cmd} #{invocation}"
|
115
|
+
log_msg = "#{Time.now.iso8601} Start Time: #{Time.now}\n#{Time.now.iso8601} Arguments: #{invocation}\n"
|
116
|
+
child_pid = Process.spawn(cmd, out: parent_writer, err: error_log_file.path)
|
117
|
+
|
118
|
+
# Logging
|
119
|
+
create_logs(child_pid, log_msg)
|
120
|
+
@child_tracker[child_pid] = { io: child_reader }
|
121
|
+
|
122
|
+
# This is used to rename error log files after all child processes are exited
|
123
|
+
@child_tracker_persisted[child_pid] = { error_log_file: error_log_file }
|
124
|
+
@ui.child_spawned(child_pid, invocation)
|
125
|
+
|
126
|
+
# Close the file to unlock the error log files opened by processes
|
127
|
+
error_log_file.close
|
128
|
+
rescue StandardError => e
|
129
|
+
$stderr.puts "#{Time.now.iso8601} Error Message: #{e.message}"
|
130
|
+
$stderr.puts "#{Time.now.iso8601} Error Backtrace: #{e.backtrace}"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def fork_another_process
|
135
|
+
invocation = invocations.shift[:value] # Be sure to do this shift() in parent process
|
136
|
+
# thing_that_reads_from_the_child, thing_that_writes_to_the_parent = IO.pipe
|
137
|
+
child_reader, parent_writer = IO.pipe
|
138
|
+
if (child_pid = Process.fork)
|
139
|
+
# In parent with newly forked child
|
140
|
+
parent_writer.close
|
141
|
+
@child_tracker[child_pid] = { io: child_reader }
|
142
|
+
@ui.child_forked(child_pid, invocation) unless run_in_background
|
143
|
+
else
|
144
|
+
# In child
|
145
|
+
child_reader.close
|
146
|
+
# replace stdout with writer
|
147
|
+
$stdout = parent_writer
|
148
|
+
create_logs(Process.pid, nil, $stderr)
|
149
|
+
|
150
|
+
begin
|
151
|
+
create_logs(
|
152
|
+
Process.pid,
|
153
|
+
"#{Time.now.iso8601} Start Time: #{Time.now}\n#{Time.now.iso8601} Arguments: #{invocation}\n"
|
154
|
+
)
|
155
|
+
runner_invocation(invocation)
|
156
|
+
rescue StandardError => e
|
157
|
+
$stderr.puts "#{Time.now.iso8601} Error Message: #{e.message}"
|
158
|
+
$stderr.puts "#{Time.now.iso8601} Error Backtrace: #{e.backtrace}"
|
159
|
+
end
|
160
|
+
|
161
|
+
# should be unreachable but child MUST exit
|
162
|
+
exit(42)
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
# Still in parent
|
167
|
+
# Loop over children and check for finished processes
|
168
|
+
def cleanup_child_processes
|
169
|
+
@child_tracker.each do |pid, info|
|
170
|
+
if Process.wait(pid, Process::WNOHANG)
|
171
|
+
# Expect to (probably) find EOF marker on the pipe, and close it if so
|
172
|
+
update_ui_poll_select(pid)
|
173
|
+
|
174
|
+
create_logs(pid, "#{Time.now.iso8601} Exit code: #{$?}\n")
|
175
|
+
|
176
|
+
# child exited - status in $?
|
177
|
+
@ui.child_exited(pid) unless run_in_background
|
178
|
+
@child_tracker.delete pid
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def update_ui_poll_select(target_pid = nil)
|
184
|
+
# Focus on one pid's pipe if specified, otherwise poll all pipes
|
185
|
+
pipes_for_reading = target_pid ? [ @child_tracker[target_pid][:io] ] : @child_tracker.values.map { |i| i[:io] }
|
186
|
+
# Next line is due to a race between the close() and the wait()... shouldn't need it, but it fixes the race.
|
187
|
+
pipes_for_reading.reject!(&:closed?)
|
188
|
+
ready_pipes = IO.select(pipes_for_reading, [], [], 0.1)
|
189
|
+
return unless ready_pipes
|
190
|
+
|
191
|
+
ready_pipes[0].each do |pipe_ready_for_reading|
|
192
|
+
# If we weren't provided a PID, hackishly look up the pid from the matching IO.
|
193
|
+
pid = target_pid || @child_tracker.keys.detect { |p| @child_tracker[p][:io] == pipe_ready_for_reading }
|
194
|
+
begin
|
195
|
+
while (update_line = pipe_ready_for_reading.readline) && !pipe_ready_for_reading.closed?
|
196
|
+
if update_line =~ /EOF_MARKER/
|
197
|
+
pipe_ready_for_reading.close
|
198
|
+
break
|
199
|
+
elsif update_line =~ /WARN/ || update_line =~ /ERROR/ || update_line =~ /INFO/
|
200
|
+
create_logs(
|
201
|
+
pid,
|
202
|
+
"#{Time.now.iso8601} Extra log: #{update_line}\n"
|
203
|
+
)
|
204
|
+
break
|
205
|
+
end
|
206
|
+
update_ui_with_line(pid, update_line) unless run_in_background
|
207
|
+
# Only pull one line if we are doing normal updates; slurp the whole file
|
208
|
+
# if we are doing a final pull on a targeted PID
|
209
|
+
break unless target_pid
|
210
|
+
end
|
211
|
+
rescue EOFError
|
212
|
+
# On unix, readline throws an EOFError when we hit the end. On Windows, nothing apparently happens.
|
213
|
+
pipe_ready_for_reading.close
|
214
|
+
next
|
215
|
+
end
|
216
|
+
end
|
217
|
+
# TODO: loop over ready_pipes[2] and handle errors?
|
218
|
+
end
|
219
|
+
|
220
|
+
def update_ui_with_line(pid, update_line)
|
221
|
+
@ui.child_status_update_line(pid, update_line)
|
222
|
+
end
|
223
|
+
|
224
|
+
private
|
225
|
+
|
226
|
+
def runner_invocation(runner_option)
|
227
|
+
splitted_result = runner_option.split(" ")
|
228
|
+
profile_to_run = splitted_result[0]
|
229
|
+
splitted_result.delete_at(0)
|
230
|
+
|
231
|
+
# thor invocation
|
232
|
+
arguments = [sub_cmd, profile_to_run, splitted_result].flatten
|
233
|
+
Inspec::InspecCLI.start(arguments, enforce_license: true)
|
234
|
+
end
|
235
|
+
|
236
|
+
def create_logs(child_pid, run_log , stderr = nil)
|
237
|
+
logs_dir_path = log_path || Dir.pwd
|
238
|
+
log_dir = File.join(logs_dir_path, "logs")
|
239
|
+
FileUtils.mkdir_p(log_dir)
|
240
|
+
|
241
|
+
if stderr
|
242
|
+
log_file = File.join(log_dir, "#{child_pid}.err") unless File.exist?("#{child_pid}.err")
|
243
|
+
stderr.reopen(log_file, "a")
|
244
|
+
else
|
245
|
+
log_file = File.join(log_dir, "#{child_pid}.log") unless File.exist?("#{child_pid}.log")
|
246
|
+
File.write(log_file, run_log, mode: "a")
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
def rename_error_log(error_log_file, child_pid)
|
251
|
+
logs_dir_path = log_path || Dir.pwd
|
252
|
+
log_dir = File.join(logs_dir_path, "logs")
|
253
|
+
FileUtils.mkdir_p(log_dir)
|
254
|
+
|
255
|
+
if error_log_file.closed? && File.exist?(error_log_file.path)
|
256
|
+
begin
|
257
|
+
File.rename("#{error_log_file.path}", "#{log_dir}/#{child_pid}.err")
|
258
|
+
rescue
|
259
|
+
$stderr.puts "Cannot rename error log file #{error_log_file.path} for child pid #{child_pid}"
|
260
|
+
end
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module InspecPlugins::Parallelism
|
2
|
+
class SuperReporter
|
3
|
+
|
4
|
+
def self.make(type, job_count, invocations)
|
5
|
+
Object.const_get("InspecPlugins::Parallelism::SuperReporter::" + (type[0].upcase + type[1..-1])).new(job_count, invocations)
|
6
|
+
end
|
7
|
+
|
8
|
+
class Base
|
9
|
+
def initialize(job_count, invocations); end
|
10
|
+
|
11
|
+
def child_spawned(pid, invocation); end
|
12
|
+
|
13
|
+
def child_forked(pid, invocation); end
|
14
|
+
|
15
|
+
def child_exited(pid); end
|
16
|
+
|
17
|
+
def child_status_update_line(pid, update_line); end
|
18
|
+
end
|
19
|
+
|
20
|
+
require_relative "text"
|
21
|
+
require_relative "status"
|
22
|
+
require_relative "silent"
|
23
|
+
end
|
24
|
+
end
|