smart_proxy_remote_execution_ssh 0.1.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +1 -1
- data/lib/smart_proxy_remote_execution_ssh.rb +28 -0
- data/lib/smart_proxy_remote_execution_ssh/actions/run_script.rb +20 -0
- data/lib/smart_proxy_remote_execution_ssh/api.rb +33 -0
- data/lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh +110 -0
- data/lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh +151 -0
- data/lib/smart_proxy_remote_execution_ssh/cockpit.rb +269 -0
- data/lib/smart_proxy_remote_execution_ssh/dispatcher.rb +10 -0
- data/lib/smart_proxy_remote_execution_ssh/log_filter.rb +14 -0
- data/lib/smart_proxy_remote_execution_ssh/plugin.rb +27 -10
- data/lib/smart_proxy_remote_execution_ssh/runners.rb +7 -0
- data/lib/smart_proxy_remote_execution_ssh/runners/fake_script_runner.rb +87 -0
- data/lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb +140 -0
- data/lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb +469 -0
- data/lib/smart_proxy_remote_execution_ssh/version.rb +1 -1
- data/lib/smart_proxy_remote_execution_ssh/webrick_ext.rb +17 -0
- data/settings.d/remote_execution_ssh.yml.example +8 -0
- metadata +61 -24
@@ -0,0 +1,10 @@
|
|
1
|
+
require 'foreman_tasks_core/runner/dispatcher'
|
2
|
+
|
3
|
+
module Proxy::RemoteExecution::Ssh
|
4
|
+
class Dispatcher < ::ForemanTasksCore::Runner::Dispatcher
|
5
|
+
def refresh_interval
|
6
|
+
@refresh_interval ||= Plugin.settings[:runner_refresh_interval] ||
|
7
|
+
Plugin.runner_class::DEFAULT_REFRESH_INTERVAL
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Proxy::RemoteExecution::Ssh
|
2
|
+
class LogFilter < ::Logger
|
3
|
+
def initialize(base_logger)
|
4
|
+
@base_logger = base_logger
|
5
|
+
end
|
6
|
+
|
7
|
+
def add(severity, *args, &block)
|
8
|
+
severity ||= ::Logger::UNKNOWN
|
9
|
+
return true if @base_logger.nil? || severity < @level
|
10
|
+
|
11
|
+
@base_logger.add(severity, *args, &block)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
module Proxy::RemoteExecution::Ssh
|
2
2
|
class Plugin < Proxy::Plugin
|
3
|
+
SSH_LOG_LEVELS = %w[debug info warn error fatal].freeze
|
4
|
+
|
3
5
|
http_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
|
4
6
|
https_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
|
5
7
|
|
@@ -9,24 +11,39 @@ module Proxy::RemoteExecution::Ssh
|
|
9
11
|
:remote_working_dir => '/var/tmp',
|
10
12
|
:local_working_dir => '/var/tmp',
|
11
13
|
:kerberos_auth => false,
|
12
|
-
:async_ssh => false
|
14
|
+
:async_ssh => false,
|
15
|
+
# When set to nil, makes REX use the runner's default interval
|
16
|
+
# :runner_refresh_interval => nil,
|
17
|
+
:ssh_log_level => :fatal,
|
18
|
+
:cleanup_working_dirs => true
|
13
19
|
|
14
20
|
plugin :ssh, Proxy::RemoteExecution::Ssh::VERSION
|
15
21
|
after_activation do
|
16
22
|
require 'smart_proxy_dynflow'
|
17
23
|
require 'smart_proxy_remote_execution_ssh/version'
|
24
|
+
require 'smart_proxy_remote_execution_ssh/cockpit'
|
18
25
|
require 'smart_proxy_remote_execution_ssh/api'
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
rescue LoadError # rubocop:disable Lint/HandleExceptions
|
25
|
-
# Dynflow core is not available in the proxy, will be handled
|
26
|
-
# by standalone Dynflow core
|
27
|
-
end
|
26
|
+
require 'smart_proxy_remote_execution_ssh/actions/run_script'
|
27
|
+
require 'smart_proxy_remote_execution_ssh/dispatcher'
|
28
|
+
require 'smart_proxy_remote_execution_ssh/log_filter'
|
29
|
+
require 'smart_proxy_remote_execution_ssh/runners'
|
30
|
+
require 'smart_proxy_dynflow_core'
|
28
31
|
|
29
32
|
Proxy::RemoteExecution::Ssh.validate!
|
30
33
|
end
|
34
|
+
|
35
|
+
def self.simulate?
|
36
|
+
@simulate ||= %w[yes true 1].include? ENV.fetch('REX_SIMULATE', '').downcase
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.runner_class
|
40
|
+
@runner_class ||= if simulate?
|
41
|
+
Runners::FakeScriptRunner
|
42
|
+
elsif settings[:async_ssh]
|
43
|
+
Runners::PollingScriptRunner
|
44
|
+
else
|
45
|
+
Runners::ScriptRunner
|
46
|
+
end
|
47
|
+
end
|
31
48
|
end
|
32
49
|
end
|
@@ -0,0 +1,7 @@
|
|
1
|
+
module Proxy::RemoteExecution::Ssh
|
2
|
+
module Runners
|
3
|
+
require 'smart_proxy_remote_execution_ssh/runners/script_runner'
|
4
|
+
require 'smart_proxy_remote_execution_ssh/runners/polling_script_runner'
|
5
|
+
require 'smart_proxy_remote_execution_ssh/runners/fake_script_runner'
|
6
|
+
end
|
7
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module Proxy::RemoteExecution::Ssh::Runners
|
2
|
+
class FakeScriptRunner < ForemanTasksCore::Runner::Base
|
3
|
+
DEFAULT_REFRESH_INTERVAL = 1
|
4
|
+
|
5
|
+
@data = []
|
6
|
+
|
7
|
+
class << self
|
8
|
+
attr_accessor :data
|
9
|
+
|
10
|
+
def load_data(path = nil)
|
11
|
+
if path.nil?
|
12
|
+
@data = <<-BANNER.gsub(/^\s+\| ?/, '').lines
|
13
|
+
| ====== Simulated Remote Execution ======
|
14
|
+
|
|
15
|
+
| This is an output of a simulated remote
|
16
|
+
| execution run. It should run for about
|
17
|
+
| 5 seconds and finish successfully.
|
18
|
+
BANNER
|
19
|
+
else
|
20
|
+
File.open(File.expand_path(path), 'r') do |f|
|
21
|
+
@data = f.readlines.map(&:chomp)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
@data.freeze
|
25
|
+
end
|
26
|
+
|
27
|
+
def build(options, suspended_action:)
|
28
|
+
new(options, suspended_action: suspended_action)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def initialize(*args)
|
33
|
+
super
|
34
|
+
# Load the fake output the first time its needed
|
35
|
+
self.class.load_data(ENV['REX_SIMULATE_PATH']) unless self.class.data.frozen?
|
36
|
+
@position = 0
|
37
|
+
end
|
38
|
+
|
39
|
+
def start
|
40
|
+
refresh
|
41
|
+
end
|
42
|
+
|
43
|
+
# Do one step
|
44
|
+
def refresh
|
45
|
+
if done?
|
46
|
+
finish
|
47
|
+
else
|
48
|
+
step
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def kill
|
53
|
+
finish
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def finish
|
59
|
+
publish_exit_status exit_code
|
60
|
+
end
|
61
|
+
|
62
|
+
def step
|
63
|
+
publish_data(next_chunk, 'stdout')
|
64
|
+
end
|
65
|
+
|
66
|
+
def done?
|
67
|
+
@position == self.class.data.count
|
68
|
+
end
|
69
|
+
|
70
|
+
def next_chunk
|
71
|
+
output = self.class.data[@position]
|
72
|
+
@position += 1
|
73
|
+
output
|
74
|
+
end
|
75
|
+
|
76
|
+
# Decide if the execution should fail or not
|
77
|
+
def exit_code
|
78
|
+
fail_chance = ENV.fetch('REX_SIMULATE_FAIL_CHANCE', 0).to_i
|
79
|
+
fail_exitcode = ENV.fetch('REX_SIMULATE_EXIT', 0).to_i
|
80
|
+
if fail_exitcode.zero? || fail_chance < (Random.rand * 100).round
|
81
|
+
0
|
82
|
+
else
|
83
|
+
fail_exitcode
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,140 @@
|
|
1
|
+
require 'base64'
|
2
|
+
|
3
|
+
module Proxy::RemoteExecution::Ssh::Runners
|
4
|
+
class PollingScriptRunner < ScriptRunner
|
5
|
+
DEFAULT_REFRESH_INTERVAL = 60
|
6
|
+
|
7
|
+
def self.load_script(name)
|
8
|
+
script_dir = File.expand_path('../async_scripts', __dir__)
|
9
|
+
File.read(File.join(script_dir, name))
|
10
|
+
end
|
11
|
+
|
12
|
+
# The script that controls the flow of the job, able to initiate update or
|
13
|
+
# finish on the task, or take over the control over script lifecycle
|
14
|
+
CONTROL_SCRIPT = load_script('control.sh')
|
15
|
+
|
16
|
+
# The script always outputs at least one line
|
17
|
+
# First line of the output either has to begin with
|
18
|
+
# "RUNNING" or "DONE $EXITCODE"
|
19
|
+
# The following lines are treated as regular output
|
20
|
+
RETRIEVE_SCRIPT = load_script('retrieve.sh')
|
21
|
+
|
22
|
+
def initialize(options, user_method, suspended_action: nil)
|
23
|
+
super(options, user_method, suspended_action: suspended_action)
|
24
|
+
@callback_host = options[:callback_host]
|
25
|
+
@task_id = options[:uuid]
|
26
|
+
@step_id = options[:step_id]
|
27
|
+
@otp = ForemanTasksCore::OtpManager.generate_otp(@task_id)
|
28
|
+
end
|
29
|
+
|
30
|
+
def prepare_start
|
31
|
+
super
|
32
|
+
@base_dir = File.dirname @remote_script
|
33
|
+
upload_control_scripts
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialization_script
|
37
|
+
close_stdin = '</dev/null'
|
38
|
+
close_fds = close_stdin + ' >/dev/null 2>/dev/null'
|
39
|
+
main_script = "(#{@remote_script} #{close_stdin} 2>&1; echo $?>#{@base_dir}/init_exit_code) >#{@base_dir}/output"
|
40
|
+
control_script_finish = "#{@control_script_path} init-script-finish"
|
41
|
+
<<-SCRIPT.gsub(/^ +\| /, '')
|
42
|
+
| export CONTROL_SCRIPT="#{@control_script_path}"
|
43
|
+
| sh -c '#{main_script}; #{control_script_finish}' #{close_fds} &
|
44
|
+
| echo $! > '#{@base_dir}/pid'
|
45
|
+
SCRIPT
|
46
|
+
end
|
47
|
+
|
48
|
+
def trigger(*args)
|
49
|
+
run_sync(*args)
|
50
|
+
end
|
51
|
+
|
52
|
+
def refresh
|
53
|
+
err = output = nil
|
54
|
+
begin
|
55
|
+
_, output, err = run_sync("#{@user_method.cli_command_prefix} #{@retrieval_script}")
|
56
|
+
rescue StandardError => e
|
57
|
+
@logger.info("Error while connecting to the remote host on refresh: #{e.message}")
|
58
|
+
end
|
59
|
+
|
60
|
+
process_retrieved_data(output, err)
|
61
|
+
ensure
|
62
|
+
destroy_session
|
63
|
+
end
|
64
|
+
|
65
|
+
def process_retrieved_data(output, err)
|
66
|
+
return if output.nil? || output.empty?
|
67
|
+
|
68
|
+
lines = output.lines
|
69
|
+
result = lines.shift.match(/^DONE (\d+)?/)
|
70
|
+
publish_data(lines.join, 'stdout') unless lines.empty?
|
71
|
+
publish_data(err, 'stderr') unless err.empty?
|
72
|
+
if result
|
73
|
+
exitcode = result[1] || 0
|
74
|
+
publish_exit_status(exitcode.to_i)
|
75
|
+
cleanup
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def external_event(event)
|
80
|
+
data = event.data
|
81
|
+
if data['manual_mode']
|
82
|
+
load_event_updates(data)
|
83
|
+
else
|
84
|
+
# getting the update from automatic mode - reaching to the host to get the latest update
|
85
|
+
return run_refresh
|
86
|
+
end
|
87
|
+
ensure
|
88
|
+
destroy_session
|
89
|
+
end
|
90
|
+
|
91
|
+
def close
|
92
|
+
super
|
93
|
+
ForemanTasksCore::OtpManager.drop_otp(@task_id, @otp) if @otp
|
94
|
+
end
|
95
|
+
|
96
|
+
def upload_control_scripts
|
97
|
+
return if @control_scripts_uploaded
|
98
|
+
|
99
|
+
cp_script_to_remote(env_script, 'env.sh')
|
100
|
+
@control_script_path = cp_script_to_remote(CONTROL_SCRIPT, 'control.sh')
|
101
|
+
@retrieval_script = cp_script_to_remote(RETRIEVE_SCRIPT, 'retrieve.sh')
|
102
|
+
@control_scripts_uploaded = true
|
103
|
+
end
|
104
|
+
|
105
|
+
# Script setting the dynamic values to env variables: it's sourced from other control scripts
|
106
|
+
def env_script
|
107
|
+
<<-SCRIPT.gsub(/^ +\| /, '')
|
108
|
+
| CALLBACK_HOST="#{@callback_host}"
|
109
|
+
| TASK_ID="#{@task_id}"
|
110
|
+
| STEP_ID="#{@step_id}"
|
111
|
+
| OTP="#{@otp}"
|
112
|
+
SCRIPT
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
# Generates updates based on the callback data from the manual mode
|
118
|
+
def load_event_updates(event_data)
|
119
|
+
continuous_output = ForemanTasksCore::ContinuousOutput.new
|
120
|
+
if event_data.key?('output')
|
121
|
+
lines = Base64.decode64(event_data['output']).sub(/\A(RUNNING|DONE).*\n/, '')
|
122
|
+
continuous_output.add_output(lines, 'stdout')
|
123
|
+
end
|
124
|
+
cleanup if event_data['exit_code']
|
125
|
+
new_update(continuous_output, event_data['exit_code'])
|
126
|
+
end
|
127
|
+
|
128
|
+
def cleanup
|
129
|
+
run_sync("rm -rf \"#{remote_command_dir}\"") if @cleanup_working_dirs
|
130
|
+
end
|
131
|
+
|
132
|
+
def destroy_session
|
133
|
+
if @session
|
134
|
+
@logger.debug("Closing session with #{@ssh_user}@#{@host}")
|
135
|
+
@session.close
|
136
|
+
@session = nil
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
@@ -0,0 +1,469 @@
|
|
1
|
+
require 'net/ssh'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
# Rubocop can't make up its mind what it wants
|
5
|
+
# rubocop:disable Lint/SuppressedException, Lint/RedundantCopDisableDirective
|
6
|
+
begin
|
7
|
+
require 'net/ssh/krb'
|
8
|
+
rescue LoadError; end
|
9
|
+
# rubocop:enable Lint/SuppressedException, Lint/RedundantCopDisableDirective
|
10
|
+
|
11
|
+
module Proxy::RemoteExecution::Ssh::Runners
|
12
|
+
class EffectiveUserMethod
|
13
|
+
attr_reader :effective_user, :ssh_user, :effective_user_password, :password_sent
|
14
|
+
|
15
|
+
def initialize(effective_user, ssh_user, effective_user_password)
|
16
|
+
@effective_user = effective_user
|
17
|
+
@ssh_user = ssh_user
|
18
|
+
@effective_user_password = effective_user_password.to_s
|
19
|
+
@password_sent = false
|
20
|
+
end
|
21
|
+
|
22
|
+
def on_data(received_data, ssh_channel)
|
23
|
+
if received_data.match(login_prompt)
|
24
|
+
ssh_channel.send_data(effective_user_password + "\n")
|
25
|
+
@password_sent = true
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def filter_password?(received_data)
|
30
|
+
!@effective_user_password.empty? && @password_sent && received_data.match(Regexp.escape(@effective_user_password))
|
31
|
+
end
|
32
|
+
|
33
|
+
def sent_all_data?
|
34
|
+
effective_user_password.empty? || password_sent
|
35
|
+
end
|
36
|
+
|
37
|
+
def reset
|
38
|
+
@password_sent = false
|
39
|
+
end
|
40
|
+
|
41
|
+
def cli_command_prefix; end
|
42
|
+
|
43
|
+
def login_prompt; end
|
44
|
+
end
|
45
|
+
|
46
|
+
class SudoUserMethod < EffectiveUserMethod
|
47
|
+
LOGIN_PROMPT = 'rex login: '.freeze
|
48
|
+
|
49
|
+
def login_prompt
|
50
|
+
LOGIN_PROMPT
|
51
|
+
end
|
52
|
+
|
53
|
+
def cli_command_prefix
|
54
|
+
"sudo -p '#{LOGIN_PROMPT}' -u #{effective_user} "
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
class DzdoUserMethod < EffectiveUserMethod
|
59
|
+
LOGIN_PROMPT = /password/i.freeze
|
60
|
+
|
61
|
+
def login_prompt
|
62
|
+
LOGIN_PROMPT
|
63
|
+
end
|
64
|
+
|
65
|
+
def cli_command_prefix
|
66
|
+
"dzdo -u #{effective_user} "
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
class SuUserMethod < EffectiveUserMethod
|
71
|
+
LOGIN_PROMPT = /Password: /i.freeze
|
72
|
+
|
73
|
+
def login_prompt
|
74
|
+
LOGIN_PROMPT
|
75
|
+
end
|
76
|
+
|
77
|
+
def cli_command_prefix
|
78
|
+
"su - #{effective_user} -c "
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
class NoopUserMethod
|
83
|
+
def on_data(_, _); end
|
84
|
+
|
85
|
+
def filter_password?(received_data)
|
86
|
+
false
|
87
|
+
end
|
88
|
+
|
89
|
+
def sent_all_data?
|
90
|
+
true
|
91
|
+
end
|
92
|
+
|
93
|
+
def cli_command_prefix; end
|
94
|
+
|
95
|
+
def reset; end
|
96
|
+
end
|
97
|
+
|
98
|
+
# rubocop:disable Metrics/ClassLength
|
99
|
+
class ScriptRunner < ForemanTasksCore::Runner::Base
|
100
|
+
attr_reader :execution_timeout_interval
|
101
|
+
|
102
|
+
EXPECTED_POWER_ACTION_MESSAGES = ['restart host', 'shutdown host'].freeze
|
103
|
+
DEFAULT_REFRESH_INTERVAL = 1
|
104
|
+
MAX_PROCESS_RETRIES = 4
|
105
|
+
|
106
|
+
def initialize(options, user_method, suspended_action: nil)
|
107
|
+
super suspended_action: suspended_action
|
108
|
+
@host = options.fetch(:hostname)
|
109
|
+
@script = options.fetch(:script)
|
110
|
+
@ssh_user = options.fetch(:ssh_user, 'root')
|
111
|
+
@ssh_port = options.fetch(:ssh_port, 22)
|
112
|
+
@ssh_password = options.fetch(:secrets, {}).fetch(:ssh_password, nil)
|
113
|
+
@key_passphrase = options.fetch(:secrets, {}).fetch(:key_passphrase, nil)
|
114
|
+
@host_public_key = options.fetch(:host_public_key, nil)
|
115
|
+
@verify_host = options.fetch(:verify_host, nil)
|
116
|
+
@execution_timeout_interval = options.fetch(:execution_timeout_interval, nil)
|
117
|
+
|
118
|
+
@client_private_key_file = settings.ssh_identity_key_file
|
119
|
+
@local_working_dir = options.fetch(:local_working_dir, settings.local_working_dir)
|
120
|
+
@remote_working_dir = options.fetch(:remote_working_dir, settings.remote_working_dir)
|
121
|
+
@cleanup_working_dirs = options.fetch(:cleanup_working_dirs, settings.cleanup_working_dirs)
|
122
|
+
@user_method = user_method
|
123
|
+
end
|
124
|
+
|
125
|
+
def self.build(options, suspended_action:)
|
126
|
+
effective_user = options.fetch(:effective_user, nil)
|
127
|
+
ssh_user = options.fetch(:ssh_user, 'root')
|
128
|
+
effective_user_method = options.fetch(:effective_user_method, 'sudo')
|
129
|
+
|
130
|
+
user_method = if effective_user.nil? || effective_user == ssh_user
|
131
|
+
NoopUserMethod.new
|
132
|
+
elsif effective_user_method == 'sudo'
|
133
|
+
SudoUserMethod.new(effective_user, ssh_user,
|
134
|
+
options.fetch(:secrets, {}).fetch(:effective_user_password, nil))
|
135
|
+
elsif effective_user_method == 'dzdo'
|
136
|
+
DzdoUserMethod.new(effective_user, ssh_user,
|
137
|
+
options.fetch(:secrets, {}).fetch(:effective_user_password, nil))
|
138
|
+
elsif effective_user_method == 'su'
|
139
|
+
SuUserMethod.new(effective_user, ssh_user,
|
140
|
+
options.fetch(:secrets, {}).fetch(:effective_user_password, nil))
|
141
|
+
else
|
142
|
+
raise "effective_user_method '#{effective_user_method}' not supported"
|
143
|
+
end
|
144
|
+
|
145
|
+
new(options, user_method, suspended_action: suspended_action)
|
146
|
+
end
|
147
|
+
|
148
|
+
def start
|
149
|
+
prepare_start
|
150
|
+
script = initialization_script
|
151
|
+
logger.debug("executing script:\n#{indent_multiline(script)}")
|
152
|
+
trigger(script)
|
153
|
+
rescue StandardError => e
|
154
|
+
logger.error("error while initalizing command #{e.class} #{e.message}:\n #{e.backtrace.join("\n")}")
|
155
|
+
publish_exception('Error initializing command', e)
|
156
|
+
end
|
157
|
+
|
158
|
+
def trigger(*args)
|
159
|
+
run_async(*args)
|
160
|
+
end
|
161
|
+
|
162
|
+
def prepare_start
|
163
|
+
@remote_script = cp_script_to_remote
|
164
|
+
@output_path = File.join(File.dirname(@remote_script), 'output')
|
165
|
+
@exit_code_path = File.join(File.dirname(@remote_script), 'exit_code')
|
166
|
+
end
|
167
|
+
|
168
|
+
# the script that initiates the execution
|
169
|
+
def initialization_script
|
170
|
+
su_method = @user_method.instance_of?(SuUserMethod)
|
171
|
+
# pipe the output to tee while capturing the exit code in a file
|
172
|
+
<<-SCRIPT.gsub(/^\s+\| /, '')
|
173
|
+
| sh -c "(#{@user_method.cli_command_prefix}#{su_method ? "'#{@remote_script} < /dev/null '" : "#{@remote_script} < /dev/null"}; echo \\$?>#{@exit_code_path}) | /usr/bin/tee #{@output_path}
|
174
|
+
| exit \\$(cat #{@exit_code_path})"
|
175
|
+
SCRIPT
|
176
|
+
end
|
177
|
+
|
178
|
+
def refresh
|
179
|
+
return if @session.nil?
|
180
|
+
|
181
|
+
with_retries do
|
182
|
+
with_disconnect_handling do
|
183
|
+
@session.process(0)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
ensure
|
187
|
+
check_expecting_disconnect
|
188
|
+
end
|
189
|
+
|
190
|
+
def kill
|
191
|
+
if @session
|
192
|
+
run_sync("pkill -f #{remote_command_file('script')}")
|
193
|
+
else
|
194
|
+
logger.debug('connection closed')
|
195
|
+
end
|
196
|
+
rescue StandardError => e
|
197
|
+
publish_exception('Unexpected error', e, false)
|
198
|
+
end
|
199
|
+
|
200
|
+
def timeout
|
201
|
+
@logger.debug('job timed out')
|
202
|
+
super
|
203
|
+
end
|
204
|
+
|
205
|
+
def timeout_interval
|
206
|
+
execution_timeout_interval
|
207
|
+
end
|
208
|
+
|
209
|
+
def with_retries
|
210
|
+
tries = 0
|
211
|
+
begin
|
212
|
+
yield
|
213
|
+
rescue StandardError => e
|
214
|
+
logger.error("Unexpected error: #{e.class} #{e.message}\n #{e.backtrace.join("\n")}")
|
215
|
+
tries += 1
|
216
|
+
if tries <= MAX_PROCESS_RETRIES
|
217
|
+
logger.error('Retrying')
|
218
|
+
retry
|
219
|
+
else
|
220
|
+
publish_exception('Unexpected error', e)
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
def with_disconnect_handling
|
226
|
+
yield
|
227
|
+
rescue IOError, Net::SSH::Disconnect => e
|
228
|
+
@session.shutdown!
|
229
|
+
check_expecting_disconnect
|
230
|
+
if @expecting_disconnect
|
231
|
+
publish_exit_status(0)
|
232
|
+
else
|
233
|
+
publish_exception('Unexpected disconnect', e)
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
def close
|
238
|
+
run_sync("rm -rf \"#{remote_command_dir}\"") if should_cleanup?
|
239
|
+
rescue StandardError => e
|
240
|
+
publish_exception('Error when removing remote working dir', e, false)
|
241
|
+
ensure
|
242
|
+
@session.close if @session && !@session.closed?
|
243
|
+
FileUtils.rm_rf(local_command_dir) if Dir.exist?(local_command_dir) && @cleanup_working_dirs
|
244
|
+
end
|
245
|
+
|
246
|
+
def publish_data(data, type)
|
247
|
+
super(data.force_encoding('UTF-8'), type)
|
248
|
+
end
|
249
|
+
|
250
|
+
private
|
251
|
+
|
252
|
+
def indent_multiline(string)
|
253
|
+
string.lines.map { |line| " | #{line}" }.join
|
254
|
+
end
|
255
|
+
|
256
|
+
def should_cleanup?
|
257
|
+
@session && !@session.closed? && @cleanup_working_dirs
|
258
|
+
end
|
259
|
+
|
260
|
+
def session
|
261
|
+
@session ||= begin
|
262
|
+
@logger.debug("opening session to #{@ssh_user}@#{@host}")
|
263
|
+
Net::SSH.start(@host, @ssh_user, ssh_options)
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
def ssh_options
|
268
|
+
ssh_options = {}
|
269
|
+
ssh_options[:port] = @ssh_port if @ssh_port
|
270
|
+
ssh_options[:keys] = [@client_private_key_file] if @client_private_key_file
|
271
|
+
ssh_options[:password] = @ssh_password if @ssh_password
|
272
|
+
ssh_options[:passphrase] = @key_passphrase if @key_passphrase
|
273
|
+
ssh_options[:keys_only] = true
|
274
|
+
# if the host public key is contained in the known_hosts_file,
|
275
|
+
# verify it, otherwise, if missing, import it and continue
|
276
|
+
ssh_options[:paranoid] = true
|
277
|
+
ssh_options[:auth_methods] = available_authentication_methods
|
278
|
+
ssh_options[:user_known_hosts_file] = prepare_known_hosts if @host_public_key
|
279
|
+
ssh_options[:number_of_password_prompts] = 1
|
280
|
+
ssh_options[:verbose] = settings[:ssh_log_level]
|
281
|
+
ssh_options[:logger] = Proxy::RemoteExecution::Ssh::LogFilter.new(SmartProxyDynflowCore::Log.instance)
|
282
|
+
return ssh_options
|
283
|
+
end
|
284
|
+
|
285
|
+
def settings
|
286
|
+
Proxy::RemoteExecution::Ssh::Plugin.settings
|
287
|
+
end
|
288
|
+
|
289
|
+
# Initiates run of the remote command and yields the data when
|
290
|
+
# available. The yielding doesn't happen automatically, but as
|
291
|
+
# part of calling the `refresh` method.
|
292
|
+
def run_async(command)
|
293
|
+
raise 'Async command already in progress' if @started
|
294
|
+
|
295
|
+
@started = false
|
296
|
+
@user_method.reset
|
297
|
+
|
298
|
+
session.open_channel do |channel|
|
299
|
+
channel.request_pty
|
300
|
+
channel.on_data do |ch, data|
|
301
|
+
publish_data(data, 'stdout') unless @user_method.filter_password?(data)
|
302
|
+
@user_method.on_data(data, ch)
|
303
|
+
end
|
304
|
+
channel.on_extended_data { |ch, type, data| publish_data(data, 'stderr') }
|
305
|
+
# standard exit of the command
|
306
|
+
channel.on_request('exit-status') { |ch, data| publish_exit_status(data.read_long) }
|
307
|
+
# on signal: sending the signal value (such as 'TERM')
|
308
|
+
channel.on_request('exit-signal') do |ch, data|
|
309
|
+
publish_exit_status(data.read_string)
|
310
|
+
ch.close
|
311
|
+
# wait for the channel to finish so that we know at the end
|
312
|
+
# that the session is inactive
|
313
|
+
ch.wait
|
314
|
+
end
|
315
|
+
channel.exec(command) do |_, success|
|
316
|
+
@started = true
|
317
|
+
raise('Error initializing command') unless success
|
318
|
+
end
|
319
|
+
end
|
320
|
+
session.process(0) { !run_started? }
|
321
|
+
return true
|
322
|
+
end
|
323
|
+
|
324
|
+
def run_started?
|
325
|
+
@started && @user_method.sent_all_data?
|
326
|
+
end
|
327
|
+
|
328
|
+
def run_sync(command, stdin = nil)
|
329
|
+
stdout = ''
|
330
|
+
stderr = ''
|
331
|
+
exit_status = nil
|
332
|
+
started = false
|
333
|
+
|
334
|
+
channel = session.open_channel do |ch|
|
335
|
+
ch.on_data do |c, data|
|
336
|
+
stdout.concat(data)
|
337
|
+
end
|
338
|
+
ch.on_extended_data { |_, _, data| stderr.concat(data) }
|
339
|
+
ch.on_request('exit-status') { |_, data| exit_status = data.read_long }
|
340
|
+
# Send data to stdin if we have some
|
341
|
+
ch.send_data(stdin) unless stdin.nil?
|
342
|
+
# on signal: sending the signal value (such as 'TERM')
|
343
|
+
ch.on_request('exit-signal') do |_, data|
|
344
|
+
exit_status = data.read_string
|
345
|
+
ch.close
|
346
|
+
ch.wait
|
347
|
+
end
|
348
|
+
ch.exec command do |_, success|
|
349
|
+
raise 'could not execute command' unless success
|
350
|
+
|
351
|
+
started = true
|
352
|
+
end
|
353
|
+
end
|
354
|
+
session.process(0) { !started }
|
355
|
+
# Closing the channel without sending any data gives us SIGPIPE
|
356
|
+
channel.close unless stdin.nil?
|
357
|
+
channel.wait
|
358
|
+
return exit_status, stdout, stderr
|
359
|
+
end
|
360
|
+
|
361
|
+
def prepare_known_hosts
|
362
|
+
path = local_command_file('known_hosts')
|
363
|
+
if @host_public_key
|
364
|
+
write_command_file_locally('known_hosts', "#{@host} #{@host_public_key}")
|
365
|
+
end
|
366
|
+
return path
|
367
|
+
end
|
368
|
+
|
369
|
+
def local_command_dir
|
370
|
+
File.join(@local_working_dir, 'foreman-proxy', "foreman-ssh-cmd-#{@id}")
|
371
|
+
end
|
372
|
+
|
373
|
+
def local_command_file(filename)
|
374
|
+
File.join(local_command_dir, filename)
|
375
|
+
end
|
376
|
+
|
377
|
+
def remote_command_dir
|
378
|
+
File.join(@remote_working_dir, "foreman-ssh-cmd-#{id}")
|
379
|
+
end
|
380
|
+
|
381
|
+
def remote_command_file(filename)
|
382
|
+
File.join(remote_command_dir, filename)
|
383
|
+
end
|
384
|
+
|
385
|
+
def ensure_local_directory(path)
|
386
|
+
if File.exist?(path)
|
387
|
+
raise "#{path} expected to be a directory" unless File.directory?(path)
|
388
|
+
else
|
389
|
+
FileUtils.mkdir_p(path)
|
390
|
+
end
|
391
|
+
return path
|
392
|
+
end
|
393
|
+
|
394
|
+
def cp_script_to_remote(script = @script, name = 'script')
|
395
|
+
path = remote_command_file(name)
|
396
|
+
@logger.debug("copying script to #{path}:\n#{indent_multiline(script)}")
|
397
|
+
upload_data(sanitize_script(script), path, 555)
|
398
|
+
end
|
399
|
+
|
400
|
+
def upload_data(data, path, permissions = 555)
|
401
|
+
ensure_remote_directory File.dirname(path)
|
402
|
+
# We use tee here to pipe stdin coming from ssh to a file at $path, while silencing its output
|
403
|
+
# This is used to write to $path with elevated permissions, solutions using cat and output redirection
|
404
|
+
# would not work, because the redirection would happen in the non-elevated shell.
|
405
|
+
command = "tee '#{path}' >/dev/null && chmod '#{permissions}' '#{path}'"
|
406
|
+
|
407
|
+
@logger.debug("Sending data to #{path} on remote host:\n#{data}")
|
408
|
+
status, _out, err = run_sync(command, data)
|
409
|
+
|
410
|
+
@logger.warn("Output on stderr while uploading #{path}:\n#{err}") unless err.empty?
|
411
|
+
if status != 0
|
412
|
+
raise "Unable to upload file to #{path} on remote system: exit code: #{status}"
|
413
|
+
end
|
414
|
+
|
415
|
+
path
|
416
|
+
end
|
417
|
+
|
418
|
+
def upload_file(local_path, remote_path)
|
419
|
+
mode = File.stat(local_path).mode.to_s(8)[-3..-1]
|
420
|
+
@logger.debug("Uploading local file: #{local_path} as #{remote_path} with #{mode} permissions")
|
421
|
+
upload_data(File.read(local_path), remote_path, mode)
|
422
|
+
end
|
423
|
+
|
424
|
+
def ensure_remote_directory(path)
|
425
|
+
exit_code, _output, err = run_sync("mkdir -p #{path}")
|
426
|
+
if exit_code != 0
|
427
|
+
raise "Unable to create directory on remote system #{path}: exit code: #{exit_code}\n #{err}"
|
428
|
+
end
|
429
|
+
end
|
430
|
+
|
431
|
+
def sanitize_script(script)
|
432
|
+
script.tr("\r", '')
|
433
|
+
end
|
434
|
+
|
435
|
+
def write_command_file_locally(filename, content)
|
436
|
+
path = local_command_file(filename)
|
437
|
+
ensure_local_directory(File.dirname(path))
|
438
|
+
File.write(path, content)
|
439
|
+
return path
|
440
|
+
end
|
441
|
+
|
442
|
+
# when a remote server disconnects, it's hard to tell if it was on purpose (when calling reboot)
|
443
|
+
# or it's an error. When it's expected, we expect the script to produce 'restart host' as
|
444
|
+
# its last command output
|
445
|
+
def check_expecting_disconnect
|
446
|
+
last_output = @continuous_output.raw_outputs.find { |d| d['output_type'] == 'stdout' }
|
447
|
+
return unless last_output
|
448
|
+
|
449
|
+
if EXPECTED_POWER_ACTION_MESSAGES.any? { |message| last_output['output'] =~ /^#{message}/ }
|
450
|
+
@expecting_disconnect = true
|
451
|
+
end
|
452
|
+
end
|
453
|
+
|
454
|
+
def available_authentication_methods
|
455
|
+
methods = %w[publickey] # Always use pubkey auth as fallback
|
456
|
+
if settings[:kerberos_auth]
|
457
|
+
if defined? Net::SSH::Kerberos
|
458
|
+
methods << 'gssapi-with-mic'
|
459
|
+
else
|
460
|
+
@logger.warn('Kerberos authentication requested but not available')
|
461
|
+
end
|
462
|
+
end
|
463
|
+
methods.unshift('password') if @ssh_password
|
464
|
+
|
465
|
+
methods
|
466
|
+
end
|
467
|
+
end
|
468
|
+
# rubocop:enable Metrics/ClassLength
|
469
|
+
end
|