smart_proxy_remote_execution_ssh 0.11.7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/smart_proxy_remote_execution_ssh/actions/run_script.rb +1 -1
- data/lib/smart_proxy_remote_execution_ssh/api.rb +6 -0
- data/lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb +9 -2
- data/lib/smart_proxy_remote_execution_ssh/plugin.rb +3 -3
- data/lib/smart_proxy_remote_execution_ssh/runners.rb +0 -1
- data/lib/smart_proxy_remote_execution_ssh/version.rb +1 -1
- data/lib/smart_proxy_remote_execution_ssh.rb +27 -17
- data/settings.d/remote_execution_ssh.yml.example +11 -3
- metadata +2 -5
- data/lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh +0 -110
- data/lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh +0 -151
- data/lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb +0 -147
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b91b743f9c5192c2a59236ca56c9194886d16fb71952f0140fae23653bed2264
|
|
4
|
+
data.tar.gz: 5bead6e1167ccc427f401de408c82c8375ee3c6888effebb862f5e5655d20894
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3c7fe36e5c40484a2a56d9c1145c82256606be3f875e4f055f45912b5c563a10ebbc8a5c5957e63313e437830d7a44b72f2e7c69e6524316e601f2cfacc15e53
|
|
7
|
+
data.tar.gz: 50b7ddfe77b7c191a436b5300aed2e18e00d63558ee24731489b19334330b7017f282bc11a7a05795ca6145a15254c4f735d34fa7416ffde345fead071fda6a3
|
|
@@ -13,6 +13,12 @@ module Proxy::RemoteExecution
|
|
|
13
13
|
File.read(Ssh.public_key_file)
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
+
get "/ca_pubkey" do
|
|
17
|
+
if Ssh.ca_public_key_file
|
|
18
|
+
File.read(Ssh.ca_public_key_file)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
16
22
|
if Proxy::RemoteExecution::Ssh::Plugin.settings.cockpit_integration
|
|
17
23
|
post "/session" do
|
|
18
24
|
do_authorize_any
|
|
@@ -60,6 +60,8 @@ module Proxy::RemoteExecution::Ssh::Runners
|
|
|
60
60
|
@host_public_key = options.fetch(:host_public_key, nil)
|
|
61
61
|
@verify_host = options.fetch(:verify_host, nil)
|
|
62
62
|
@client_private_key_file = settings.ssh_identity_key_file
|
|
63
|
+
@client_ca_known_hosts_file = settings.ssh_ca_known_hosts_file
|
|
64
|
+
@client_cert_file = Proxy::RemoteExecution::Ssh.cert_file if File.exist?(Proxy::RemoteExecution::Ssh.cert_file)
|
|
63
65
|
|
|
64
66
|
@local_working_dir = options.fetch(:local_working_dir, settings.local_working_dir)
|
|
65
67
|
@socket_working_dir = options.fetch(:socket_working_dir, settings.socket_working_dir)
|
|
@@ -154,9 +156,14 @@ module Proxy::RemoteExecution::Ssh::Runners
|
|
|
154
156
|
ssh_options << "-o User=#{@ssh_user}"
|
|
155
157
|
ssh_options << "-o Port=#{@ssh_port}" if @ssh_port
|
|
156
158
|
ssh_options << "-o IdentityFile=#{@client_private_key_file}" if @client_private_key_file
|
|
159
|
+
ssh_options << "-o CertificateFile=#{@client_cert_file}" if @client_cert_file
|
|
157
160
|
ssh_options << "-o IdentitiesOnly=yes"
|
|
158
|
-
ssh_options << "-o StrictHostKeyChecking
|
|
159
|
-
|
|
161
|
+
ssh_options << "-o StrictHostKeyChecking=#{@client_ca_known_hosts_file ? 'yes' : 'accept-new'}"
|
|
162
|
+
if @host_public_key
|
|
163
|
+
ssh_options << "-o UserKnownHostsFile=#{prepare_known_hosts}"
|
|
164
|
+
elsif @client_ca_known_hosts_file
|
|
165
|
+
ssh_options << "-o UserKnownHostsFile=#{@client_ca_known_hosts_file}"
|
|
166
|
+
end
|
|
160
167
|
ssh_options << "-o LogLevel=#{ssh_log_level(true)}"
|
|
161
168
|
ssh_options << "-o ControlMaster=auto"
|
|
162
169
|
ssh_options << "-o ControlPath=#{socket_file}"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
module Proxy::RemoteExecution::Ssh
|
|
2
2
|
class Plugin < Proxy::Plugin
|
|
3
3
|
SSH_LOG_LEVELS = %w[debug info error fatal].freeze
|
|
4
|
-
MODES = %i[ssh
|
|
4
|
+
MODES = %i[ssh pull pull-mqtt].freeze
|
|
5
5
|
# Unix domain socket path length is limited to 104 (on some platforms) characters
|
|
6
6
|
# Socket path is composed of custom path (max 49 characters) + job id (37 characters)
|
|
7
7
|
# + offset(17 characters) + null terminator
|
|
@@ -12,6 +12,8 @@ module Proxy::RemoteExecution::Ssh
|
|
|
12
12
|
|
|
13
13
|
settings_file "remote_execution_ssh.yml"
|
|
14
14
|
default_settings :ssh_identity_key_file => '~/.ssh/id_rsa_foreman_proxy',
|
|
15
|
+
# :ssh_ca_known_hosts_file => nil,
|
|
16
|
+
# :ssh_user_ca_public_key_file => nil,
|
|
15
17
|
:ssh_user => 'root',
|
|
16
18
|
:remote_working_dir => '/var/tmp',
|
|
17
19
|
:local_working_dir => '/var/tmp',
|
|
@@ -61,8 +63,6 @@ module Proxy::RemoteExecution::Ssh
|
|
|
61
63
|
def self.runner_class
|
|
62
64
|
@runner_class ||= if simulate?
|
|
63
65
|
Runners::FakeScriptRunner
|
|
64
|
-
elsif settings.mode == :'ssh-async'
|
|
65
|
-
Runners::PollingScriptRunner
|
|
66
66
|
else
|
|
67
67
|
Runners::ScriptRunner
|
|
68
68
|
end
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
module Proxy::RemoteExecution::Ssh
|
|
2
2
|
module Runners
|
|
3
3
|
require 'smart_proxy_remote_execution_ssh/runners/script_runner'
|
|
4
|
-
require 'smart_proxy_remote_execution_ssh/runners/polling_script_runner'
|
|
5
4
|
require 'smart_proxy_remote_execution_ssh/runners/fake_script_runner'
|
|
6
5
|
end
|
|
7
6
|
end
|
|
@@ -21,26 +21,21 @@ module Proxy::RemoteExecution
|
|
|
21
21
|
File.expand_path("#{private_key_file}.pub")
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
def cert_file
|
|
25
|
+
File.expand_path("#{private_key_file}-cert.pub")
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def ca_public_key_file
|
|
29
|
+
path = Plugin.settings.ssh_user_ca_public_key_file
|
|
30
|
+
File.expand_path(path) if present?(path)
|
|
31
|
+
end
|
|
32
|
+
|
|
24
33
|
def validate_mode!
|
|
25
34
|
Plugin.settings.mode = Plugin.settings.mode.to_sym
|
|
26
35
|
|
|
27
36
|
unless Plugin::MODES.include? Plugin.settings.mode
|
|
28
37
|
raise "Mode has to be one of #{Plugin::MODES.join(', ')}, given #{Plugin.settings.mode}"
|
|
29
38
|
end
|
|
30
|
-
|
|
31
|
-
if Plugin.settings.async_ssh
|
|
32
|
-
Plugin.logger.warn('Option async_ssh is deprecated, use ssh-async mode instead.')
|
|
33
|
-
|
|
34
|
-
case Plugin.settings.mode
|
|
35
|
-
when :ssh
|
|
36
|
-
Plugin.logger.warn('Deprecated option async_ssh used together with ssh mode, switching mode to ssh-async.')
|
|
37
|
-
Plugin.settings.mode = :'ssh-async'
|
|
38
|
-
when :'ssh-async'
|
|
39
|
-
# This is a noop
|
|
40
|
-
else
|
|
41
|
-
Plugin.logger.warn('Deprecated option async_ssh used together with incompatible mode, ignoring.')
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
39
|
end
|
|
45
40
|
|
|
46
41
|
def validate_mqtt_settings!
|
|
@@ -64,7 +59,7 @@ module Proxy::RemoteExecution
|
|
|
64
59
|
end
|
|
65
60
|
|
|
66
61
|
unless File.exist?(private_key_file)
|
|
67
|
-
raise "SSH
|
|
62
|
+
raise "SSH private key file #{private_key_file} doesn't exist.\n"\
|
|
68
63
|
"You can generate one with `ssh-keygen -t rsa -b 4096 -f #{private_key_file} -N ''`"
|
|
69
64
|
end
|
|
70
65
|
|
|
@@ -72,6 +67,15 @@ module Proxy::RemoteExecution
|
|
|
72
67
|
raise "SSH public key file #{public_key_file} doesn't exist"
|
|
73
68
|
end
|
|
74
69
|
|
|
70
|
+
if present?(Plugin.settings.ssh_user_ca_public_key_file)
|
|
71
|
+
{ ca_public_key_file: 'CA public key', cert_file: 'certificate' }.each do |file, label|
|
|
72
|
+
file_path = public_send(file)
|
|
73
|
+
unless file_path && File.exist?(file_path)
|
|
74
|
+
raise "SSH #{label} file '#{file_path}' doesn't exist"
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
75
79
|
validate_ssh_log_level!
|
|
76
80
|
end
|
|
77
81
|
|
|
@@ -97,11 +101,11 @@ module Proxy::RemoteExecution
|
|
|
97
101
|
end
|
|
98
102
|
|
|
99
103
|
def requires_configured_ssh?
|
|
100
|
-
|
|
104
|
+
Plugin.settings.mode == :ssh || Plugin.settings.cockpit_integration
|
|
101
105
|
end
|
|
102
106
|
|
|
103
107
|
def validate_socket_path!
|
|
104
|
-
return unless Plugin.settings.mode == :'ssh'
|
|
108
|
+
return unless Plugin.settings.mode == :'ssh'
|
|
105
109
|
|
|
106
110
|
socket_path = File.expand_path(Plugin.settings.socket_working_dir)
|
|
107
111
|
raise "Socket path #{socket_path} is too long" if socket_path.length > Plugin::SOCKET_PATH_MAX_LENGTH
|
|
@@ -114,6 +118,12 @@ module Proxy::RemoteExecution
|
|
|
114
118
|
def with_mqtt?
|
|
115
119
|
Proxy::RemoteExecution::Ssh::Plugin.settings.mode == :'pull-mqtt'
|
|
116
120
|
end
|
|
121
|
+
|
|
122
|
+
private
|
|
123
|
+
|
|
124
|
+
def present?(value)
|
|
125
|
+
value && !value.empty?
|
|
126
|
+
end
|
|
117
127
|
end
|
|
118
128
|
end
|
|
119
129
|
end
|
|
@@ -8,13 +8,21 @@
|
|
|
8
8
|
|
|
9
9
|
# :cockpit_integration: true
|
|
10
10
|
|
|
11
|
-
# Mode of operation, one of ssh,
|
|
11
|
+
# Mode of operation, one of ssh, pull, pull-mqtt
|
|
12
12
|
:mode: ssh
|
|
13
13
|
|
|
14
|
+
# Enables the use of SSH certificate for smart proxy authentication
|
|
15
|
+
# The file should contain an SSH CA public key that the SSH public key of smart proxy is signed by
|
|
16
|
+
# :ssh_user_ca_public_key_file:
|
|
17
|
+
|
|
18
|
+
# Enables the use of SSH host certificates for host authentication
|
|
19
|
+
# The file should contain a list of trusted SSH CA authorities that the host certs can be signed by
|
|
20
|
+
# Example file content: @cert-authority * <SSH CA public key>
|
|
21
|
+
# :ssh_ca_known_hosts_file:
|
|
22
|
+
|
|
14
23
|
# Defines how often (in seconds) should the runner check
|
|
15
24
|
# for new data leave empty to use the runner's default
|
|
16
|
-
#
|
|
17
|
-
# :runner_refresh_interval:
|
|
25
|
+
# :runner_refresh_interval: 1
|
|
18
26
|
|
|
19
27
|
# Defines the verbosity of logging coming from ssh command
|
|
20
28
|
# one of :debug, :info, :error, :fatal
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: smart_proxy_remote_execution_ssh
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ivan Nečas
|
|
@@ -198,8 +198,6 @@ files:
|
|
|
198
198
|
- lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb
|
|
199
199
|
- lib/smart_proxy_remote_execution_ssh/actions/run_script.rb
|
|
200
200
|
- lib/smart_proxy_remote_execution_ssh/api.rb
|
|
201
|
-
- lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh
|
|
202
|
-
- lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh
|
|
203
201
|
- lib/smart_proxy_remote_execution_ssh/cockpit.rb
|
|
204
202
|
- lib/smart_proxy_remote_execution_ssh/command_logging.rb
|
|
205
203
|
- lib/smart_proxy_remote_execution_ssh/dispatcher.rb
|
|
@@ -212,7 +210,6 @@ files:
|
|
|
212
210
|
- lib/smart_proxy_remote_execution_ssh/plugin.rb
|
|
213
211
|
- lib/smart_proxy_remote_execution_ssh/runners.rb
|
|
214
212
|
- lib/smart_proxy_remote_execution_ssh/runners/fake_script_runner.rb
|
|
215
|
-
- lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb
|
|
216
213
|
- lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb
|
|
217
214
|
- lib/smart_proxy_remote_execution_ssh/utils.rb
|
|
218
215
|
- lib/smart_proxy_remote_execution_ssh/version.rb
|
|
@@ -229,7 +226,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
229
226
|
requirements:
|
|
230
227
|
- - ">="
|
|
231
228
|
- !ruby/object:Gem::Version
|
|
232
|
-
version:
|
|
229
|
+
version: '3.0'
|
|
233
230
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
234
231
|
requirements:
|
|
235
232
|
- - ">="
|
|
@@ -1,110 +0,0 @@
|
|
|
1
|
-
#!/bin/sh
|
|
2
|
-
#
|
|
3
|
-
# Control script for the remote execution jobs.
|
|
4
|
-
#
|
|
5
|
-
# The initial script calls `$CONTROL_SCRIPT init-script-finish` once the original script exits.
|
|
6
|
-
# In automatic mode, the exit code is sent back to the proxy on `init-script-finish`.
|
|
7
|
-
#
|
|
8
|
-
# What the script provides is also a manual mode, where the author of the rex script can take
|
|
9
|
-
# full control of the job lifecycle. This allows keeping the marked as running even when
|
|
10
|
-
# the initial script finishes.
|
|
11
|
-
#
|
|
12
|
-
# The manual mode is turned on by calling `$CONTROL_SCRIPT manual-control`. After calling this,
|
|
13
|
-
# one can call `echo message | $CONTROL_SCRIPT update` to send output to the remote execution jobs
|
|
14
|
-
# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code) to send output to the remote execution jobs
|
|
15
|
-
# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code)
|
|
16
|
-
BASE_DIR="$(dirname "$(readlink -f "$0")")"
|
|
17
|
-
|
|
18
|
-
if ! command -v curl >/dev/null; then
|
|
19
|
-
echo 'curl is required' >&2
|
|
20
|
-
exit 1
|
|
21
|
-
fi
|
|
22
|
-
|
|
23
|
-
# send the callback data to proxy
|
|
24
|
-
update() {
|
|
25
|
-
"$BASE_DIR/retrieve.sh" push_update
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
# wait for named pipe $1 to retrieve data. If $2 is provided, it serves as timeout
|
|
29
|
-
# in seconds on how long to wait when reading.
|
|
30
|
-
wait_for_pipe() {
|
|
31
|
-
pipe_path=$1
|
|
32
|
-
if [ -n "$2" ]; then
|
|
33
|
-
timeout="-t $2"
|
|
34
|
-
fi
|
|
35
|
-
if read $timeout <>"$pipe_path"; then
|
|
36
|
-
rm "$pipe_path"
|
|
37
|
-
return 0
|
|
38
|
-
else
|
|
39
|
-
return 1
|
|
40
|
-
fi
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
# function run in background, when receiving update data via STDIN.
|
|
44
|
-
periodic_update() {
|
|
45
|
-
interval=1
|
|
46
|
-
# reading some data from periodic_update_control signals we're done
|
|
47
|
-
while ! wait_for_pipe "$BASE_DIR/periodic_update_control" "$interval"; do
|
|
48
|
-
update
|
|
49
|
-
done
|
|
50
|
-
# one more update before we finish
|
|
51
|
-
update
|
|
52
|
-
# signal the main process that we are finished
|
|
53
|
-
echo > "$BASE_DIR/periodic_update_finished"
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
# signal the periodic_update process that the main process is finishing
|
|
57
|
-
periodic_update_finish() {
|
|
58
|
-
if [ -e "$BASE_DIR/periodic_update_control" ]; then
|
|
59
|
-
echo > "$BASE_DIR/periodic_update_control"
|
|
60
|
-
fi
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
ACTION=${1:-finish}
|
|
64
|
-
|
|
65
|
-
case "$ACTION" in
|
|
66
|
-
init-script-finish)
|
|
67
|
-
if ! [ -e "$BASE_DIR/manual_mode" ]; then
|
|
68
|
-
# make the exit code of initialization script the exit code of the whole job
|
|
69
|
-
cp init_exit_code exit_code
|
|
70
|
-
update
|
|
71
|
-
fi
|
|
72
|
-
;;
|
|
73
|
-
finish)
|
|
74
|
-
# take exit code passed via the command line, with fallback
|
|
75
|
-
# to the exit code of the initialization script
|
|
76
|
-
exit_code=${2:-$(cat "$BASE_DIR/init_exit_code")}
|
|
77
|
-
echo $exit_code > "$BASE_DIR/exit_code"
|
|
78
|
-
update
|
|
79
|
-
if [ -e "$BASE_DIR/manual_mode" ]; then
|
|
80
|
-
rm "$BASE_DIR/manual_mode"
|
|
81
|
-
fi
|
|
82
|
-
;;
|
|
83
|
-
update)
|
|
84
|
-
# read data from input when redirected though a pipe
|
|
85
|
-
if ! [ -t 0 ]; then
|
|
86
|
-
# couple of named pipes to coordinate the main process with the periodic_update
|
|
87
|
-
mkfifo "$BASE_DIR/periodic_update_control"
|
|
88
|
-
mkfifo "$BASE_DIR/periodic_update_finished"
|
|
89
|
-
trap "periodic_update_finish" EXIT
|
|
90
|
-
# run periodic update as separate process to keep sending updates in output to server
|
|
91
|
-
periodic_update &
|
|
92
|
-
# redirect the input into output
|
|
93
|
-
tee -a "$BASE_DIR/output"
|
|
94
|
-
periodic_update_finish
|
|
95
|
-
# ensure the periodic update finished before we return
|
|
96
|
-
wait_for_pipe "$BASE_DIR/periodic_update_finished"
|
|
97
|
-
else
|
|
98
|
-
update
|
|
99
|
-
fi
|
|
100
|
-
;;
|
|
101
|
-
# mark the script to be in manual mode: this means the script author needs to use `update` and `finish`
|
|
102
|
-
# commands to send output to the remote execution job or mark it as finished.
|
|
103
|
-
manual-mode)
|
|
104
|
-
touch "$BASE_DIR/manual_mode"
|
|
105
|
-
;;
|
|
106
|
-
*)
|
|
107
|
-
echo "Unknown action $ACTION"
|
|
108
|
-
exit 1
|
|
109
|
-
;;
|
|
110
|
-
esac
|
|
@@ -1,151 +0,0 @@
|
|
|
1
|
-
#!/bin/sh
|
|
2
|
-
|
|
3
|
-
if ! pgrep --help 2>/dev/null >/dev/null; then
|
|
4
|
-
echo DONE 1
|
|
5
|
-
echo "pgrep is required" >&2
|
|
6
|
-
exit 1
|
|
7
|
-
fi
|
|
8
|
-
|
|
9
|
-
BASE_DIR="$(dirname "$(readlink -f "$0")")"
|
|
10
|
-
|
|
11
|
-
# load the data required for generating the callback
|
|
12
|
-
. "$BASE_DIR/env.sh"
|
|
13
|
-
URL_PREFIX="$CALLBACK_HOST/dynflow/tasks/$TASK_ID"
|
|
14
|
-
AUTH="$TASK_ID:$OTP"
|
|
15
|
-
CURL="curl --silent --show-error --fail --max-time 10"
|
|
16
|
-
|
|
17
|
-
MY_LOCK_FILE="$BASE_DIR/retrieve_lock.$$"
|
|
18
|
-
MY_PID=$$
|
|
19
|
-
echo $MY_PID >"$MY_LOCK_FILE"
|
|
20
|
-
LOCK_FILE="$BASE_DIR/retrieve_lock"
|
|
21
|
-
TMP_OUTPUT_FILE="$BASE_DIR/tmp_output"
|
|
22
|
-
|
|
23
|
-
RUN_TIMEOUT=30 # for how long can the script hold the lock
|
|
24
|
-
WAIT_TIMEOUT=60 # for how long the script is trying to acquire the lock
|
|
25
|
-
START_TIME=$(date +%s)
|
|
26
|
-
|
|
27
|
-
fail() {
|
|
28
|
-
echo RUNNING
|
|
29
|
-
echo "$1"
|
|
30
|
-
exit 1
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
acquire_lock() {
|
|
34
|
-
# try to acquire lock by creating the file (ln should be atomic an fail in case
|
|
35
|
-
# another process succeeded first). We also check the content of the lock file,
|
|
36
|
-
# in case our process won when competing over the lock while invalidating
|
|
37
|
-
# the lock on timeout.
|
|
38
|
-
ln "$MY_LOCK_FILE" "$LOCK_FILE" 2>/dev/null || [ "$(head -n1 "$LOCK_FILE")" = "$MY_PID" ]
|
|
39
|
-
return $?
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
# acquiring the lock before proceeding, to ensure only one instance of the script is running
|
|
43
|
-
while ! acquire_lock; do
|
|
44
|
-
# we failed to create retrieve_lock - assuming there is already another retrieve script running
|
|
45
|
-
current_pid=$(head -n1 "$LOCK_FILE")
|
|
46
|
-
if [ -z "$current_pid" ]; then
|
|
47
|
-
continue
|
|
48
|
-
fi
|
|
49
|
-
# check whether the lock is not too old (compared to $RUN_TIMEOUT) and try to kill
|
|
50
|
-
# if it is, so that we don't have a stalled processes here
|
|
51
|
-
lock_lines_count=$(wc -l < "$LOCK_FILE")
|
|
52
|
-
current_lock_time=$(stat --format "%Y" "$LOCK_FILE")
|
|
53
|
-
current_time=$(date +%s)
|
|
54
|
-
|
|
55
|
-
if [ "$(( current_time - START_TIME ))" -gt "$WAIT_TIMEOUT" ]; then
|
|
56
|
-
# We were waiting for the lock for too long - just give up
|
|
57
|
-
fail "Wait time exceeded $WAIT_TIMEOUT"
|
|
58
|
-
elif [ "$(( current_time - current_lock_time ))" -gt "$RUN_TIMEOUT" ]; then
|
|
59
|
-
# The previous lock it hold for too long - re-acquiring procedure
|
|
60
|
-
if [ "$lock_lines_count" -gt 1 ]; then
|
|
61
|
-
# there were multiple processes waiting for lock without resolution
|
|
62
|
-
# longer than the $RUN_TIMEOUT - we reset the lock file and let processes
|
|
63
|
-
# to compete
|
|
64
|
-
echo "RETRY" > "$LOCK_FILE"
|
|
65
|
-
fi
|
|
66
|
-
if [ "$current_pid" != "RETRY" ]; then
|
|
67
|
-
# try to kill the currently stalled process
|
|
68
|
-
kill -9 "$current_pid" 2>/dev/null
|
|
69
|
-
fi
|
|
70
|
-
# try to add our process as one candidate
|
|
71
|
-
echo $MY_PID >> "$LOCK_FILE"
|
|
72
|
-
if [ "$( head -n2 "$LOCK_FILE" | tail -n1 )" = "$MY_PID" ]; then
|
|
73
|
-
# our process won the competition for the new lock: it is the first pid
|
|
74
|
-
# after the original one in the lock file - take ownership of the lock
|
|
75
|
-
# next iteration only this process will get through
|
|
76
|
-
echo $MY_PID >"$LOCK_FILE"
|
|
77
|
-
fi
|
|
78
|
-
else
|
|
79
|
-
# still waiting for the original owner to finish
|
|
80
|
-
sleep 1
|
|
81
|
-
fi
|
|
82
|
-
done
|
|
83
|
-
|
|
84
|
-
release_lock() {
|
|
85
|
-
rm "$MY_LOCK_FILE"
|
|
86
|
-
rm "$LOCK_FILE"
|
|
87
|
-
}
|
|
88
|
-
# ensure the release the lock at exit
|
|
89
|
-
trap "release_lock" EXIT
|
|
90
|
-
|
|
91
|
-
# make sure we clear previous tmp output file
|
|
92
|
-
if [ -e "$TMP_OUTPUT_FILE" ]; then
|
|
93
|
-
rm "$TMP_OUTPUT_FILE"
|
|
94
|
-
fi
|
|
95
|
-
|
|
96
|
-
pid=$(cat "$BASE_DIR/pid")
|
|
97
|
-
[ -f "$BASE_DIR/position" ] || echo 1 > "$BASE_DIR/position"
|
|
98
|
-
position=$(cat "$BASE_DIR/position")
|
|
99
|
-
|
|
100
|
-
prepare_output() {
|
|
101
|
-
if [ -e "$BASE_DIR/manual_mode" ] || ([ -n "$pid" ] && pgrep -P "$pid" >/dev/null 2>&1); then
|
|
102
|
-
echo RUNNING
|
|
103
|
-
else
|
|
104
|
-
echo "DONE $(cat "$BASE_DIR/exit_code" 2>/dev/null)"
|
|
105
|
-
fi
|
|
106
|
-
[ -f "$BASE_DIR/output" ] || exit 0
|
|
107
|
-
tail --bytes "+${position}" "$BASE_DIR/output" > "$TMP_OUTPUT_FILE"
|
|
108
|
-
cat "$TMP_OUTPUT_FILE"
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
# prepare the callback payload
|
|
112
|
-
payload() {
|
|
113
|
-
if [ -n "$1" ]; then
|
|
114
|
-
exit_code="$1"
|
|
115
|
-
else
|
|
116
|
-
exit_code=null
|
|
117
|
-
fi
|
|
118
|
-
|
|
119
|
-
if [ -e "$BASE_DIR/manual_mode" ]; then
|
|
120
|
-
manual_mode=true
|
|
121
|
-
output=$(prepare_output | base64 -w0)
|
|
122
|
-
else
|
|
123
|
-
manual_mode=false
|
|
124
|
-
fi
|
|
125
|
-
|
|
126
|
-
echo "{ \"exit_code\": $exit_code,"\
|
|
127
|
-
" \"step_id\": \"$STEP_ID\","\
|
|
128
|
-
" \"manual_mode\": $manual_mode,"\
|
|
129
|
-
" \"output\": \"$output\" }"
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
if [ "$1" = "push_update" ]; then
|
|
133
|
-
if [ -e "$BASE_DIR/exit_code" ]; then
|
|
134
|
-
exit_code="$(cat "$BASE_DIR/exit_code")"
|
|
135
|
-
action="done"
|
|
136
|
-
else
|
|
137
|
-
exit_code=""
|
|
138
|
-
action="update"
|
|
139
|
-
fi
|
|
140
|
-
$CURL -X POST -d "$(payload $exit_code)" -u "$AUTH" "$URL_PREFIX"/$action 2>>"$BASE_DIR/curl_stderr"
|
|
141
|
-
success=$?
|
|
142
|
-
else
|
|
143
|
-
prepare_output
|
|
144
|
-
success=$?
|
|
145
|
-
fi
|
|
146
|
-
|
|
147
|
-
if [ "$success" = 0 ] && [ -e "$TMP_OUTPUT_FILE" ]; then
|
|
148
|
-
# in case the retrieval was successful, move the position of the cursor to be read next time
|
|
149
|
-
bytes=$(wc --bytes < "$TMP_OUTPUT_FILE")
|
|
150
|
-
expr "${position}" + "${bytes}" > "$BASE_DIR/position"
|
|
151
|
-
fi
|
|
@@ -1,147 +0,0 @@
|
|
|
1
|
-
require 'base64'
|
|
2
|
-
|
|
3
|
-
module Proxy::RemoteExecution::Ssh::Runners
|
|
4
|
-
class PollingScriptRunner < ScriptRunner
|
|
5
|
-
DEFAULT_REFRESH_INTERVAL = 60
|
|
6
|
-
|
|
7
|
-
def self.load_script(name)
|
|
8
|
-
script_dir = File.expand_path('../async_scripts', __dir__)
|
|
9
|
-
File.read(File.join(script_dir, name))
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
# The script that controls the flow of the job, able to initiate update or
|
|
13
|
-
# finish on the task, or take over the control over script lifecycle
|
|
14
|
-
CONTROL_SCRIPT = load_script('control.sh')
|
|
15
|
-
|
|
16
|
-
# The script always outputs at least one line
|
|
17
|
-
# First line of the output either has to begin with
|
|
18
|
-
# "RUNNING" or "DONE $EXITCODE"
|
|
19
|
-
# The following lines are treated as regular output
|
|
20
|
-
RETRIEVE_SCRIPT = load_script('retrieve.sh')
|
|
21
|
-
|
|
22
|
-
def initialize(options, user_method, suspended_action: nil)
|
|
23
|
-
super(options, user_method, suspended_action: suspended_action)
|
|
24
|
-
@callback_host = options[:callback_host]
|
|
25
|
-
@task_id = options[:uuid]
|
|
26
|
-
@step_id = options[:step_id]
|
|
27
|
-
@otp = Proxy::Dynflow::OtpManager.generate_otp(@task_id)
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def prepare_start
|
|
31
|
-
super
|
|
32
|
-
@base_dir = File.dirname @remote_script
|
|
33
|
-
upload_control_scripts
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
def initialization_script
|
|
37
|
-
close_stdin = '</dev/null'
|
|
38
|
-
close_fds = close_stdin + ' >/dev/null 2>/dev/null'
|
|
39
|
-
main_script = "(#{@remote_script_wrapper} #{@remote_script} #{close_stdin} 2>&1; echo $?>#{@base_dir}/init_exit_code) >#{@base_dir}/output"
|
|
40
|
-
control_script_finish = "#{@control_script_path} init-script-finish"
|
|
41
|
-
<<-SCRIPT.gsub(/^ +\| /, '')
|
|
42
|
-
| export CONTROL_SCRIPT="#{@control_script_path}"
|
|
43
|
-
| #{"chown #{@user_method.effective_user} #{@base_dir}" if @user_method.cli_command_prefix}
|
|
44
|
-
| #{@user_method.cli_command_prefix} sh -c '#{main_script}; #{control_script_finish}' #{close_fds} &
|
|
45
|
-
SCRIPT
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
def trigger(*args)
|
|
49
|
-
run_sync(*args)
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
def refresh
|
|
53
|
-
@connection.establish! unless @connection.connected?
|
|
54
|
-
begin
|
|
55
|
-
pm = run_sync("#{@user_method.cli_command_prefix} #{@retrieval_script}")
|
|
56
|
-
process_retrieved_data(pm.stdout.to_s.chomp, pm.stderr.to_s.chomp)
|
|
57
|
-
rescue StandardError => e
|
|
58
|
-
@logger.info("Error while connecting to the remote host on refresh: #{e.message}")
|
|
59
|
-
end
|
|
60
|
-
ensure
|
|
61
|
-
destroy_session
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
def kill
|
|
65
|
-
run_sync("pkill -P $(cat #{@pid_path})")
|
|
66
|
-
rescue StandardError => e
|
|
67
|
-
publish_exception('Unexpected error', e, false)
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
def process_retrieved_data(output, err)
|
|
71
|
-
return if output.nil? || output.empty?
|
|
72
|
-
|
|
73
|
-
lines = output.lines
|
|
74
|
-
result = lines.shift.match(/^DONE (\d+)?/)
|
|
75
|
-
publish_data(lines.join, 'stdout') unless lines.empty?
|
|
76
|
-
publish_data(err, 'stderr') unless err.empty?
|
|
77
|
-
if result
|
|
78
|
-
exitcode = result[1] || 0
|
|
79
|
-
publish_exit_status(exitcode.to_i)
|
|
80
|
-
cleanup
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
def external_event(event)
|
|
85
|
-
data = event.data
|
|
86
|
-
if data['manual_mode']
|
|
87
|
-
load_event_updates(data)
|
|
88
|
-
else
|
|
89
|
-
# getting the update from automatic mode - reaching to the host to get the latest update
|
|
90
|
-
return run_refresh
|
|
91
|
-
end
|
|
92
|
-
ensure
|
|
93
|
-
destroy_session
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
def close
|
|
97
|
-
super
|
|
98
|
-
Proxy::Dynflow::OtpManager.drop_otp(@task_id, @otp) if @otp
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
def upload_control_scripts
|
|
102
|
-
return if @control_scripts_uploaded
|
|
103
|
-
|
|
104
|
-
cp_script_to_remote(env_script, 'env.sh')
|
|
105
|
-
@control_script_path = cp_script_to_remote(CONTROL_SCRIPT, 'control.sh')
|
|
106
|
-
@retrieval_script = cp_script_to_remote(RETRIEVE_SCRIPT, 'retrieve.sh')
|
|
107
|
-
@control_scripts_uploaded = true
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
# Script setting the dynamic values to env variables: it's sourced from other control scripts
|
|
111
|
-
def env_script
|
|
112
|
-
<<-SCRIPT.gsub(/^ +\| /, '')
|
|
113
|
-
| CALLBACK_HOST="#{@callback_host}"
|
|
114
|
-
| TASK_ID="#{@task_id}"
|
|
115
|
-
| STEP_ID="#{@step_id}"
|
|
116
|
-
| OTP="#{@otp}"
|
|
117
|
-
SCRIPT
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
private
|
|
121
|
-
|
|
122
|
-
# Generates updates based on the callback data from the manual mode
|
|
123
|
-
def load_event_updates(event_data)
|
|
124
|
-
continuous_output = Proxy::Dynflow::ContinuousOutput.new
|
|
125
|
-
if event_data.key?('output')
|
|
126
|
-
lines = Base64.decode64(event_data['output']).sub(/\A(RUNNING|DONE).*\n/, '')
|
|
127
|
-
continuous_output.add_output(lines, 'stdout')
|
|
128
|
-
end
|
|
129
|
-
cleanup if event_data['exit_code']
|
|
130
|
-
new_update(continuous_output, event_data['exit_code'])
|
|
131
|
-
end
|
|
132
|
-
|
|
133
|
-
def cleanup
|
|
134
|
-
if @cleanup_working_dirs
|
|
135
|
-
ensure_remote_command("rm -rf #{remote_command_dir}",
|
|
136
|
-
error: "Unable to remove working directory #{remote_command_dir} on remote system, exit code: %{exit_code}")
|
|
137
|
-
end
|
|
138
|
-
end
|
|
139
|
-
|
|
140
|
-
def destroy_session
|
|
141
|
-
if @connection.connected?
|
|
142
|
-
@logger.debug("Closing session with #{@ssh_user}@#{@host}")
|
|
143
|
-
close_session
|
|
144
|
-
end
|
|
145
|
-
end
|
|
146
|
-
end
|
|
147
|
-
end
|