smart_proxy_remote_execution_ssh 0.9.0 → 0.10.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb +51 -46
- data/lib/smart_proxy_remote_execution_ssh/api.rb +9 -0
- data/lib/smart_proxy_remote_execution_ssh/mqtt/dispatcher.rb +169 -0
- data/lib/smart_proxy_remote_execution_ssh/mqtt.rb +23 -0
- data/lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb +2 -0
- data/lib/smart_proxy_remote_execution_ssh/plugin.rb +8 -1
- data/lib/smart_proxy_remote_execution_ssh/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: debc9989aade4d23bda1b90e9fa8e85c99948f5b9ca37d6cecca7573bb994e5e
|
4
|
+
data.tar.gz: 38a2731c7f28643daaa33ddd29f93172b7467877f36b12c750b82f86e921c04d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a3f5776ce0f6ad361f2c344d0ff5d5cef9b6b032e23dc42ae0e877298abd9e95c0156c77334b106036f376e62c60c01ab13344d6bda53baa52c93c7caba79b71
|
7
|
+
data.tar.gz: d1de077038e5fce86df1871e4702e2abb4c7de4937f66aa8730803bb4542aa8f26c1c848d24bb74db1ce5594514d3a552b92b7ad315d346ead5e7586cb20da20
|
@@ -4,9 +4,19 @@ require 'time'
|
|
4
4
|
|
5
5
|
module Proxy::RemoteExecution::Ssh::Actions
|
6
6
|
class PullScript < Proxy::Dynflow::Action::Runner
|
7
|
+
include ::Dynflow::Action::Timeouts
|
8
|
+
|
7
9
|
JobDelivered = Class.new
|
8
10
|
PickupTimeout = Class.new
|
9
|
-
|
11
|
+
|
12
|
+
# The proxy has the job stored in its job storage
|
13
|
+
READY_FOR_PICKUP = 'ready_for_pickup'
|
14
|
+
# The host was notified over MQTT at least once
|
15
|
+
NOTIFIED = 'notified'
|
16
|
+
# The host has picked up the job
|
17
|
+
DELIVERED = 'delivered'
|
18
|
+
# We received at least one output from the host
|
19
|
+
RUNNING = 'running'
|
10
20
|
|
11
21
|
execution_plan_hooks.use :cleanup, :on => :stopped
|
12
22
|
|
@@ -17,20 +27,22 @@ module Proxy::RemoteExecution::Ssh::Actions
|
|
17
27
|
|
18
28
|
def run(event = nil)
|
19
29
|
if event == JobDelivered
|
20
|
-
output[:state] =
|
30
|
+
output[:state] = DELIVERED
|
21
31
|
suspend
|
22
32
|
elsif event == PickupTimeout
|
23
33
|
process_pickup_timeout
|
24
|
-
elsif event ==
|
25
|
-
|
26
|
-
|
27
|
-
|
34
|
+
elsif event == ::Dynflow::Action::Timeouts::Timeout
|
35
|
+
process_timeout
|
36
|
+
elsif event.nil?
|
37
|
+
if (timeout = input['execution_timeout_interval'])
|
38
|
+
schedule_timeout(timeout, optional: true)
|
28
39
|
end
|
29
|
-
|
40
|
+
super
|
30
41
|
else
|
31
42
|
super
|
32
43
|
end
|
33
44
|
rescue => e
|
45
|
+
cleanup
|
34
46
|
action_logger.error(e)
|
35
47
|
process_update(Proxy::Dynflow::Runner::Update.encode_exception('Proxy error', e))
|
36
48
|
end
|
@@ -43,22 +55,21 @@ module Proxy::RemoteExecution::Ssh::Actions
|
|
43
55
|
plan_event(PickupTimeout, input[:time_to_pickup], optional: true) if input[:time_to_pickup]
|
44
56
|
|
45
57
|
input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script].tr("\r", ''), effective_user: input[:effective_user])
|
46
|
-
output[:state] =
|
58
|
+
output[:state] = READY_FOR_PICKUP
|
47
59
|
output[:result] = []
|
48
|
-
|
49
|
-
|
50
|
-
mqtt_start(otp_password)
|
51
|
-
end
|
60
|
+
|
61
|
+
mqtt_start(otp_password) if input[:with_mqtt]
|
52
62
|
suspend
|
53
63
|
end
|
54
64
|
|
55
65
|
def cleanup(_plan = nil)
|
56
66
|
job_storage.drop_job(execution_plan_id, run_step_id)
|
57
67
|
Proxy::Dynflow::OtpManager.passwords.delete(execution_plan_id)
|
68
|
+
Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.done(input[:job_uuid])
|
58
69
|
end
|
59
70
|
|
60
71
|
def process_external_event(event)
|
61
|
-
output[:state] =
|
72
|
+
output[:state] = RUNNING
|
62
73
|
data = event.data
|
63
74
|
case data['version']
|
64
75
|
when nil
|
@@ -97,19 +108,34 @@ module Proxy::RemoteExecution::Ssh::Actions
|
|
97
108
|
process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
|
98
109
|
end
|
99
110
|
|
100
|
-
def
|
111
|
+
def process_timeout
|
112
|
+
kill_run "Execution timeout exceeded"
|
113
|
+
end
|
114
|
+
|
115
|
+
def kill_run(fail_msg = 'The job was cancelled by the user')
|
116
|
+
continuous_output = Proxy::Dynflow::ContinuousOutput.new
|
117
|
+
exit_code = nil
|
118
|
+
|
101
119
|
case output[:state]
|
102
|
-
when
|
120
|
+
when READY_FOR_PICKUP, NOTIFIED
|
103
121
|
# If the job is not running yet on the client, wipe it from storage
|
104
122
|
cleanup
|
105
|
-
|
106
|
-
when
|
123
|
+
exit_code = 'EXCEPTION'
|
124
|
+
when DELIVERED, RUNNING
|
107
125
|
# Client was notified or is already running, dealing with this situation
|
108
126
|
# is only supported if mqtt is available
|
109
127
|
# Otherwise we have to wait it out
|
110
|
-
|
128
|
+
if input[:with_mqtt]
|
129
|
+
mqtt_cancel
|
130
|
+
fail_msg += ', notifying the host over MQTT'
|
131
|
+
else
|
132
|
+
fail_msg += ', however the job was triggered without MQTT and cannot be stopped'
|
133
|
+
end
|
111
134
|
end
|
112
|
-
|
135
|
+
continuous_output.add_output(fail_msg + "\n")
|
136
|
+
process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
|
137
|
+
|
138
|
+
suspend unless exit_code
|
113
139
|
end
|
114
140
|
|
115
141
|
def mqtt_start(otp_password)
|
@@ -125,13 +151,13 @@ module Proxy::RemoteExecution::Ssh::Actions
|
|
125
151
|
'effective_user': input[:effective_user]
|
126
152
|
},
|
127
153
|
)
|
128
|
-
|
129
|
-
output[:state] =
|
154
|
+
Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.new(input[:job_uuid], mqtt_topic, payload)
|
155
|
+
output[:state] = NOTIFIED
|
130
156
|
end
|
131
157
|
|
132
158
|
def mqtt_cancel
|
133
|
-
cleanup
|
134
159
|
payload = mqtt_payload_base.merge(
|
160
|
+
content: "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}/cancel",
|
135
161
|
metadata: {
|
136
162
|
'event': 'cancel',
|
137
163
|
'job_uuid': input[:job_uuid]
|
@@ -141,18 +167,7 @@ module Proxy::RemoteExecution::Ssh::Actions
|
|
141
167
|
end
|
142
168
|
|
143
169
|
def mqtt_notify(payload)
|
144
|
-
|
145
|
-
c.publish(mqtt_topic, JSON.dump(payload), false, 1)
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
def with_mqtt_client(&block)
|
150
|
-
MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port,
|
151
|
-
:ssl => settings.mqtt_tls,
|
152
|
-
:cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
|
153
|
-
:key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
|
154
|
-
:ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
|
155
|
-
&block)
|
170
|
+
Proxy::RemoteExecution::Ssh::MQTT.publish(mqtt_topic, JSON.dump(payload))
|
156
171
|
end
|
157
172
|
|
158
173
|
def host_name
|
@@ -167,10 +182,6 @@ module Proxy::RemoteExecution::Ssh::Actions
|
|
167
182
|
"yggdrasil/#{host_name}/data/in"
|
168
183
|
end
|
169
184
|
|
170
|
-
def settings
|
171
|
-
Proxy::RemoteExecution::Ssh::Plugin.settings
|
172
|
-
end
|
173
|
-
|
174
185
|
def job_storage
|
175
186
|
Proxy::RemoteExecution::Ssh.job_storage
|
176
187
|
end
|
@@ -186,15 +197,9 @@ module Proxy::RemoteExecution::Ssh::Actions
|
|
186
197
|
end
|
187
198
|
|
188
199
|
def process_pickup_timeout
|
189
|
-
|
190
|
-
raise "The job was not picked up in time"
|
191
|
-
else
|
192
|
-
suspend
|
193
|
-
end
|
194
|
-
end
|
200
|
+
suspend unless [READY_FOR_PICKUP, NOTIFIED].include? output[:state]
|
195
201
|
|
196
|
-
|
197
|
-
plan_event(ResendNotification, settings[:mqtt_resend_interval], optional: true)
|
202
|
+
kill_run 'The job was not picked up in time'
|
198
203
|
end
|
199
204
|
end
|
200
205
|
end
|
@@ -64,12 +64,21 @@ module Proxy::RemoteExecution
|
|
64
64
|
do_authorize_with_ssl_client
|
65
65
|
|
66
66
|
with_authorized_job(job_uuid) do |job_record|
|
67
|
+
Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.running(job_record[:uuid])
|
67
68
|
notify_job(job_record, Actions::PullScript::JobDelivered)
|
68
69
|
response.headers['X-Foreman-Effective-User'] = job_record[:effective_user]
|
69
70
|
job_record[:job]
|
70
71
|
end
|
71
72
|
end
|
72
73
|
|
74
|
+
get "/jobs/:job_uuid/cancel" do |job_uuid|
|
75
|
+
do_authorize_with_ssl_client
|
76
|
+
|
77
|
+
with_authorized_job(job_uuid) do |job_record|
|
78
|
+
{}
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
73
82
|
private
|
74
83
|
|
75
84
|
def notify_job(job_record, event)
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require 'concurrent'
|
2
|
+
require 'mqtt'
|
3
|
+
|
4
|
+
class Proxy::RemoteExecution::Ssh::MQTT
|
5
|
+
class Dispatcher
|
6
|
+
include Singleton
|
7
|
+
|
8
|
+
attr_reader :reference
|
9
|
+
def initialize
|
10
|
+
limit = Proxy::RemoteExecution::Ssh::Plugin.settings[:mqtt_rate_limit]
|
11
|
+
@reference = DispatcherActor.spawn('MQTT dispatcher',
|
12
|
+
Proxy::Dynflow::Core.world.clock,
|
13
|
+
limit)
|
14
|
+
end
|
15
|
+
|
16
|
+
def new(uuid, topic, payload)
|
17
|
+
reference.tell([:new, uuid, topic, payload])
|
18
|
+
end
|
19
|
+
|
20
|
+
def running(uuid)
|
21
|
+
reference.tell([:running, uuid])
|
22
|
+
end
|
23
|
+
|
24
|
+
def resend(uuid)
|
25
|
+
reference.tell([:resend, uuid])
|
26
|
+
end
|
27
|
+
|
28
|
+
def done(uuid)
|
29
|
+
reference.tell([:done, uuid])
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class DispatcherActor < Concurrent::Actor::RestartingContext
|
34
|
+
JobDefinition = Struct.new :uuid, :topic, :payload
|
35
|
+
|
36
|
+
class Tracker
|
37
|
+
def initialize(limit, clock)
|
38
|
+
@clock = clock
|
39
|
+
@limit = limit
|
40
|
+
@jobs = {}
|
41
|
+
@pending = []
|
42
|
+
@running = Set.new
|
43
|
+
@hot = Set.new
|
44
|
+
@cold = Set.new
|
45
|
+
end
|
46
|
+
|
47
|
+
def new(uuid, topic, payload)
|
48
|
+
@jobs[uuid] = JobDefinition.new(uuid, topic, payload)
|
49
|
+
@pending << uuid
|
50
|
+
dispatch_pending
|
51
|
+
end
|
52
|
+
|
53
|
+
def running(uuid)
|
54
|
+
[@pending, @hot, @cold].each { |source| source.delete(uuid) }
|
55
|
+
@running << uuid
|
56
|
+
end
|
57
|
+
|
58
|
+
def resend(uuid)
|
59
|
+
return unless @jobs[uuid]
|
60
|
+
|
61
|
+
@pending << uuid
|
62
|
+
dispatch_pending
|
63
|
+
end
|
64
|
+
|
65
|
+
def done(uuid)
|
66
|
+
@jobs.delete(uuid)
|
67
|
+
[@pending, @running, @hot, @cold].each do |source|
|
68
|
+
source.delete(uuid)
|
69
|
+
end
|
70
|
+
dispatch_pending
|
71
|
+
end
|
72
|
+
|
73
|
+
def needs_processing?
|
74
|
+
pending_count.positive? || @hot.any? || @cold.any?
|
75
|
+
end
|
76
|
+
|
77
|
+
def pending_count
|
78
|
+
pending = @pending.count
|
79
|
+
return pending if @limit.nil?
|
80
|
+
|
81
|
+
running = [@running, @hot, @cold].map(&:count).sum
|
82
|
+
capacity = @limit - running
|
83
|
+
pending > capacity ? capacity : pending
|
84
|
+
end
|
85
|
+
|
86
|
+
def dispatch_pending
|
87
|
+
pending_count.times do
|
88
|
+
mqtt_notify(@pending.first)
|
89
|
+
@hot << @pending.shift
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def process
|
94
|
+
@cold.each { |uuid| schedule_resend(uuid) }
|
95
|
+
@cold = @hot
|
96
|
+
@hot = Set.new
|
97
|
+
|
98
|
+
dispatch_pending
|
99
|
+
end
|
100
|
+
|
101
|
+
def mqtt_notify(uuid)
|
102
|
+
job = @jobs[uuid]
|
103
|
+
return if job.nil?
|
104
|
+
|
105
|
+
Proxy::RemoteExecution::Ssh::MQTT.publish(job.topic, JSON.dump(job.payload))
|
106
|
+
end
|
107
|
+
|
108
|
+
def settings
|
109
|
+
Proxy::RemoteExecution::Ssh::Plugin.settings
|
110
|
+
end
|
111
|
+
|
112
|
+
def schedule_resend(uuid)
|
113
|
+
@clock.ping(Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance, resend_interval, uuid, :resend)
|
114
|
+
end
|
115
|
+
|
116
|
+
def resend_interval
|
117
|
+
settings[:mqtt_resend_interval]
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def initialize(clock, limit = nil)
|
122
|
+
@tracker = Tracker.new(limit, clock)
|
123
|
+
|
124
|
+
interval = Proxy::RemoteExecution::Ssh::Plugin.settings[:mqtt_ttl]
|
125
|
+
@timer = Concurrent::TimerTask.new(execution_interval: interval) do
|
126
|
+
reference.tell(:tick)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def on_message(message)
|
131
|
+
action, arg = message
|
132
|
+
# Enable the timer just in case anything in tracker raises an exception so
|
133
|
+
# we can continue
|
134
|
+
timer_on
|
135
|
+
case action
|
136
|
+
when :new
|
137
|
+
_, uuid, topic, payload = message
|
138
|
+
@tracker.new(uuid, topic, payload)
|
139
|
+
when :resend
|
140
|
+
@tracker.resend(arg)
|
141
|
+
when :running
|
142
|
+
@tracker.running(arg)
|
143
|
+
when :done
|
144
|
+
@tracker.done(arg)
|
145
|
+
when :tick
|
146
|
+
@tracker.process
|
147
|
+
end
|
148
|
+
timer_set(@tracker.needs_processing?)
|
149
|
+
end
|
150
|
+
|
151
|
+
def timer_set(on)
|
152
|
+
on ? timer_on : timer_off
|
153
|
+
end
|
154
|
+
|
155
|
+
def timer_on
|
156
|
+
@timer.execute
|
157
|
+
end
|
158
|
+
|
159
|
+
def timer_off
|
160
|
+
@timer.shutdown
|
161
|
+
end
|
162
|
+
|
163
|
+
# In case an exception is raised during processing, instruct concurrent-ruby
|
164
|
+
# to keep going without losing state
|
165
|
+
def behaviour_definition
|
166
|
+
Concurrent::Actor::Behaviour.restarting_behaviour_definition(:resume!)
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Proxy::RemoteExecution::Ssh
|
2
|
+
class MQTT
|
3
|
+
require 'smart_proxy_remote_execution_ssh/mqtt/dispatcher'
|
4
|
+
|
5
|
+
class << self
|
6
|
+
def publish(topic, payload, retain: false, qos: 1)
|
7
|
+
with_mqtt_client do |c|
|
8
|
+
c.publish(topic, payload, retain, qos)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def with_mqtt_client(&block)
|
13
|
+
::MQTT::Client.connect(Plugin.settings.mqtt_broker,
|
14
|
+
Plugin.settings.mqtt_port,
|
15
|
+
:ssl => Plugin.settings.mqtt_tls,
|
16
|
+
:cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
|
17
|
+
:key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
|
18
|
+
:ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
|
19
|
+
&block)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -154,6 +154,8 @@ module Proxy::RemoteExecution::Ssh::Runners
|
|
154
154
|
ssh_options << "-o ControlPath=#{socket_file}"
|
155
155
|
ssh_options << "-o ControlPersist=yes"
|
156
156
|
ssh_options << "-o ProxyCommand=none"
|
157
|
+
ssh_options << "-o ServerAliveInterval=15"
|
158
|
+
ssh_options << "-o ServerAliveCountMax=3" # This is the default, but let's be explicit
|
157
159
|
@establish_ssh_options = ssh_options
|
158
160
|
end
|
159
161
|
|
@@ -25,8 +25,10 @@ module Proxy::RemoteExecution::Ssh
|
|
25
25
|
# :mqtt_broker => nil,
|
26
26
|
# :mqtt_port => nil,
|
27
27
|
# :mqtt_tls => nil,
|
28
|
+
# :mqtt_rate_limit => nil
|
28
29
|
:mode => :ssh,
|
29
|
-
:mqtt_resend_interval => 900
|
30
|
+
:mqtt_resend_interval => 900,
|
31
|
+
:mqtt_ttl => 5
|
30
32
|
|
31
33
|
capability(proc { 'cockpit' if settings.cockpit_integration })
|
32
34
|
|
@@ -46,6 +48,11 @@ module Proxy::RemoteExecution::Ssh
|
|
46
48
|
Proxy::RemoteExecution::Ssh.validate!
|
47
49
|
|
48
50
|
Proxy::Dynflow::TaskLauncherRegistry.register('ssh', Proxy::Dynflow::TaskLauncher::Batch)
|
51
|
+
if settings.mode == :'pull-mqtt'
|
52
|
+
require 'smart_proxy_remote_execution_ssh/mqtt'
|
53
|
+
# Force initialization
|
54
|
+
Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance
|
55
|
+
end
|
49
56
|
end
|
50
57
|
|
51
58
|
def self.simulate?
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: smart_proxy_remote_execution_ssh
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ivan Nečas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-01-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -175,6 +175,8 @@ files:
|
|
175
175
|
- lib/smart_proxy_remote_execution_ssh/http_config.ru
|
176
176
|
- lib/smart_proxy_remote_execution_ssh/job_storage.rb
|
177
177
|
- lib/smart_proxy_remote_execution_ssh/log_filter.rb
|
178
|
+
- lib/smart_proxy_remote_execution_ssh/mqtt.rb
|
179
|
+
- lib/smart_proxy_remote_execution_ssh/mqtt/dispatcher.rb
|
178
180
|
- lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb
|
179
181
|
- lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb
|
180
182
|
- lib/smart_proxy_remote_execution_ssh/plugin.rb
|