dash 4.1.3 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/dash/autoscale/connections/builder.rb +51 -0
- data/lib/dash/autoscale/connections.rb +102 -0
- data/lib/dash/autoscale/controller.rb +296 -0
- data/lib/dash/autoscale/cron.rb +75 -0
- data/lib/dash/autoscale/decision.rb +89 -0
- data/lib/dash/autoscale/duration.rb +19 -0
- data/lib/dash/autoscale/evaluation.rb +52 -0
- data/lib/dash/autoscale/guard.rb +48 -0
- data/lib/dash/autoscale/lease.rb +55 -0
- data/lib/dash/autoscale/lease_held.rb +2 -0
- data/lib/dash/autoscale/lease_lost.rb +2 -0
- data/lib/dash/autoscale/member.rb +28 -0
- data/lib/dash/autoscale/metrics.rb +70 -0
- data/lib/dash/autoscale/metrics_server.rb +71 -0
- data/lib/dash/autoscale/observation.rb +41 -0
- data/lib/dash/autoscale/pause.rb +34 -0
- data/lib/dash/autoscale/policy.rb +198 -0
- data/lib/dash/autoscale/pool.rb +103 -0
- data/lib/dash/autoscale/prometheus_client.rb +76 -0
- data/lib/dash/autoscale/provider/base.rb +43 -0
- data/lib/dash/autoscale/provider/exec.rb +94 -0
- data/lib/dash/autoscale/provider/upcloud.rb +140 -0
- data/lib/dash/autoscale/provider.rb +9 -0
- data/lib/dash/autoscale/provider_error.rb +4 -0
- data/lib/dash/autoscale/read_only_query.rb +38 -0
- data/lib/dash/autoscale/reader.rb +17 -0
- data/lib/dash/autoscale/recorder.rb +13 -0
- data/lib/dash/autoscale/role_state.rb +110 -0
- data/lib/dash/autoscale/sample.rb +13 -0
- data/lib/dash/autoscale/signal/pgbus.rb +19 -0
- data/lib/dash/autoscale/signal/prometheus.rb +8 -0
- data/lib/dash/autoscale/signal/sidekiq.rb +49 -0
- data/lib/dash/autoscale/signal.rb +31 -0
- data/lib/dash/autoscale/simulation.rb +200 -0
- data/lib/dash/autoscale/state_store.rb +132 -0
- data/lib/dash/autoscale/timestamp.rb +15 -0
- data/lib/dash/autoscale/tunnel.rb +48 -0
- data/lib/dash/cli/accessory.rb +16 -0
- data/lib/dash/cli/app/boot.rb +69 -44
- data/lib/dash/cli/app/boot_state.rb +70 -0
- data/lib/dash/cli/app/rollout_boot.rb +28 -9
- data/lib/dash/cli/app/running_targets.rb +39 -0
- data/lib/dash/cli/app/slot_version.rb +29 -0
- data/lib/dash/cli/app.rb +89 -32
- data/lib/dash/cli/autoscale.rb +312 -0
- data/lib/dash/cli/base.rb +15 -3
- data/lib/dash/cli/lock.rb +6 -1
- data/lib/dash/cli/main.rb +77 -8
- data/lib/dash/cli/proxy/inspection.rb +52 -0
- data/lib/dash/cli/proxy/loadbalancer_deploy.rb +20 -0
- data/lib/dash/cli/proxy/loadbalancer_reboot.rb +1 -1
- data/lib/dash/cli/proxy/reboot.rb +4 -9
- data/lib/dash/cli/proxy.rb +12 -7
- data/lib/dash/cli/prune.rb +5 -3
- data/lib/dash/cli/report.rb +4 -1
- data/lib/dash/cli/scale/host_plan.rb +63 -0
- data/lib/dash/cli/scale/join.rb +128 -0
- data/lib/dash/cli/scale/leave.rb +58 -0
- data/lib/dash/cli/scale/placement.rb +54 -0
- data/lib/dash/cli/scale/power_off.rb +26 -0
- data/lib/dash/cli/scale/replica_join.rb +74 -0
- data/lib/dash/cli/scale/replica_leave.rb +121 -0
- data/lib/dash/cli/scale.rb +276 -0
- data/lib/dash/cli/server.rb +41 -0
- data/lib/dash/cli/stats_table.rb +34 -0
- data/lib/dash/cli/templates/sample_hooks/post-scale-in.sample +3 -0
- data/lib/dash/cli/templates/sample_hooks/post-scale-out.sample +3 -0
- data/lib/dash/cli/templates/sample_hooks/pre-scale-in.sample +6 -0
- data/lib/dash/cli/templates/sample_hooks/pre-scale-out.sample +6 -0
- data/lib/dash/commander.rb +17 -3
- data/lib/dash/commands/accessory/proxy.rb +1 -1
- data/lib/dash/commands/accessory.rb +21 -2
- data/lib/dash/commands/app/containers.rb +28 -3
- data/lib/dash/commands/app/logging.rb +13 -2
- data/lib/dash/commands/app/proxy.rb +4 -4
- data/lib/dash/commands/app.rb +90 -10
- data/lib/dash/commands/auditor.rb +2 -2
- data/lib/dash/commands/autoscale.rb +133 -0
- data/lib/dash/commands/base.rb +4 -0
- data/lib/dash/commands/proxy/services.rb +10 -0
- data/lib/dash/commands/prune.rb +10 -2
- data/lib/dash/commands/server.rb +16 -0
- data/lib/dash/configuration/accessory.rb +2 -2
- data/lib/dash/configuration/autoscale.rb +123 -0
- data/lib/dash/configuration/docs/autoscale.yml +141 -0
- data/lib/dash/configuration/docs/configuration.yml +8 -2
- data/lib/dash/configuration/docs/proxy.yml +1 -1
- data/lib/dash/configuration/docs/role.yml +203 -0
- data/lib/dash/configuration/loadbalancer.rb +3 -2
- data/lib/dash/configuration/proxy/run.rb +1 -1
- data/lib/dash/configuration/proxy.rb +19 -7
- data/lib/dash/configuration/role/drain.rb +49 -0
- data/lib/dash/configuration/role/replicas.rb +72 -0
- data/lib/dash/configuration/role/scale/guard.rb +40 -0
- data/lib/dash/configuration/role/scale/signal.rb +93 -0
- data/lib/dash/configuration/role/scale/window.rb +37 -0
- data/lib/dash/configuration/role/scale.rb +199 -0
- data/lib/dash/configuration/role.rb +123 -6
- data/lib/dash/configuration/validator/autoscale.rb +126 -0
- data/lib/dash/configuration/validator/proxy.rb +13 -0
- data/lib/dash/configuration/validator/role.rb +26 -5
- data/lib/dash/configuration.rb +126 -15
- data/lib/dash/diagnostics/audit.rb +28 -0
- data/lib/dash/diagnostics/autoscale_decisions.rb +18 -0
- data/lib/dash/diagnostics/autoscale_explain.rb +30 -0
- data/lib/dash/diagnostics/autoscale_signal.rb +82 -0
- data/lib/dash/diagnostics/base.rb +53 -0
- data/lib/dash/diagnostics/config.rb +32 -0
- data/lib/dash/diagnostics/container_stats.rb +67 -0
- data/lib/dash/diagnostics/containers.rb +20 -0
- data/lib/dash/diagnostics/controller_status.rb +44 -0
- data/lib/dash/diagnostics/docker_ps.rb +40 -0
- data/lib/dash/{cli → diagnostics}/doctor/config_checks.rb +3 -3
- data/lib/dash/diagnostics/doctor/controller_checks.rb +46 -0
- data/lib/dash/{cli → diagnostics}/doctor/endpoint_checks.rb +12 -4
- data/lib/dash/{cli → diagnostics}/doctor/host_checks.rb +5 -4
- data/lib/dash/diagnostics/doctor/pool_checks.rb +65 -0
- data/lib/dash/diagnostics/doctor/signal_checks.rb +54 -0
- data/lib/dash/diagnostics/doctor.rb +182 -0
- data/lib/dash/diagnostics/drift.rb +168 -0
- data/lib/dash/diagnostics/host_stats.rb +59 -0
- data/lib/dash/diagnostics/lines.rb +14 -0
- data/lib/dash/diagnostics/lock.rb +27 -0
- data/lib/dash/diagnostics/logs.rb +76 -0
- data/lib/dash/diagnostics/pool.rb +41 -0
- data/lib/dash/diagnostics/proxy_services.rb +36 -0
- data/lib/dash/diagnostics/redactor.rb +48 -0
- data/lib/dash/diagnostics/reports.rb +17 -0
- data/lib/dash/diagnostics/scale.rb +51 -0
- data/lib/dash/diagnostics/units.rb +31 -0
- data/lib/dash/mcp/base_tool.rb +43 -0
- data/lib/dash/mcp/runner.rb +28 -0
- data/lib/dash/mcp/server.rb +40 -0
- data/lib/dash/mcp/session.rb +51 -0
- data/lib/dash/mcp/tools/audit_tool.rb +11 -0
- data/lib/dash/mcp/tools/autoscale_decisions_tool.rb +22 -0
- data/lib/dash/mcp/tools/autoscale_explain_tool.rb +56 -0
- data/lib/dash/mcp/tools/config_tool.rb +10 -0
- data/lib/dash/mcp/tools/container_stats_tool.rb +15 -0
- data/lib/dash/mcp/tools/containers_tool.rb +10 -0
- data/lib/dash/mcp/tools/controller_status_tool.rb +18 -0
- data/lib/dash/mcp/tools/deploy_reports_tool.rb +10 -0
- data/lib/dash/mcp/tools/doctor_tool.rb +13 -0
- data/lib/dash/mcp/tools/drift_tool.rb +16 -0
- data/lib/dash/mcp/tools/host_stats_tool.rb +14 -0
- data/lib/dash/mcp/tools/lock_status_tool.rb +15 -0
- data/lib/dash/mcp/tools/logs_tool.rb +41 -0
- data/lib/dash/mcp/tools/pool_members_tool.rb +15 -0
- data/lib/dash/mcp/tools/proxy_services_tool.rb +10 -0
- data/lib/dash/mcp/tools/scale_status_tool.rb +10 -0
- data/lib/dash/mcp/tools/signal_sample_tool.rb +19 -0
- data/lib/dash/mcp/transport.rb +14 -0
- data/lib/dash/mcp.rb +56 -0
- data/lib/dash/sshkit_with_ext.rb +4 -4
- data/lib/dash/utils.rb +7 -0
- data/lib/dash/version.rb +1 -1
- data/lib/dash.rb +2 -0
- metadata +119 -5
- data/lib/dash/cli/doctor.rb +0 -113
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
# `dash autoscale`: the controller that holds each scheduled or signalled role at its target
|
|
2
|
+
# (Dash::Autoscale::Controller), and the commands that read what it decided and why. The
|
|
3
|
+
# controller's state lives on one host - `autoscale.controller`, or the primary host
|
|
4
|
+
# without it - so every operator's checkout reads the same heartbeat, decisions and pauses.
|
|
5
|
+
class Dash::Cli::Autoscale < Dash::Cli::Base
|
|
6
|
+
MINIMUM_INTERVAL = 5
|
|
7
|
+
LOCK_WAIT_INTERVAL = 5
|
|
8
|
+
|
|
9
|
+
# `run` is a Thor reserved word, so the method has another name.
|
|
10
|
+
map "run" => :start_controller
|
|
11
|
+
|
|
12
|
+
desc "run", "Run the autoscale controller: every interval, scale each role with a scale/schedule or scale/signal toward its target through dash scale set"
|
|
13
|
+
option :once, type: :boolean, default: false, desc: "Run one tick and exit (for cron)"
|
|
14
|
+
option :dry_run, type: :boolean, default: false, desc: "Decide and print, without acting or writing any state"
|
|
15
|
+
option :interval, type: :numeric, desc: "Seconds between ticks (default autoscale/interval)"
|
|
16
|
+
option :metrics_port, type: :numeric, desc: "Serve Prometheus metrics on this port at /metrics"
|
|
17
|
+
option :metrics_bind, default: "127.0.0.1", desc: "Address the metrics endpoint binds"
|
|
18
|
+
option :record, desc: "Append every tick's decisions, with their inputs, to this local JSONL file"
|
|
19
|
+
option :takeover, type: :boolean, default: false, desc: "Start even while another controller's heartbeat is alive (it stops at its next tick)"
|
|
20
|
+
def start_controller
|
|
21
|
+
raise ArgumentError, "dash autoscale run works on whole roles: narrow it with --roles, not --hosts or --primary" if options[:hosts] || options[:primary]
|
|
22
|
+
|
|
23
|
+
controller = Dash::Autoscale::Controller.new(cli: self, dry_run: options[:dry_run], takeover: options[:takeover], interval: tick_interval,
|
|
24
|
+
mode: options[:once] ? "once" : "loop", recorder: (Dash::Autoscale::Recorder.new(options[:record]) if options[:record]))
|
|
25
|
+
pre_connect_if_required
|
|
26
|
+
|
|
27
|
+
options[:once] ? run_once(controller) : run_loop(controller)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
desc "explain ROLE", "Evaluate ROLE as the controller's next tick would, with every input, without acting"
|
|
31
|
+
option :json, type: :boolean, default: false, desc: "Print the evaluation as JSON"
|
|
32
|
+
def explain(role_name)
|
|
33
|
+
role = scaled_role(role_name)
|
|
34
|
+
explain = -> { Dash::Diagnostics::AutoscaleExplain.new(role: role).to_h }
|
|
35
|
+
|
|
36
|
+
if options[:json]
|
|
37
|
+
puts_json(&explain)
|
|
38
|
+
else
|
|
39
|
+
pre_connect_if_required
|
|
40
|
+
print_explain(DASH.with_verbosity(:error) { explain.call })
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
desc "signal ROLE", "Read one live sample of ROLE's scale/signal and of each scale/hold_when guard, from this machine"
|
|
45
|
+
option :json, type: :boolean, default: false, desc: "Print the sample as JSON"
|
|
46
|
+
def signal(role_name)
|
|
47
|
+
role = scaled_role(role_name)
|
|
48
|
+
sample = -> { Dash::Diagnostics::AutoscaleSignal.new(role: role).to_h }
|
|
49
|
+
|
|
50
|
+
if options[:json]
|
|
51
|
+
puts_json(&sample)
|
|
52
|
+
else
|
|
53
|
+
# A source with `via:` is reached through an SSH tunnel.
|
|
54
|
+
pre_connect_if_required
|
|
55
|
+
print_signal(sample.call)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
desc "simulate", "Replay a dash autoscale run --record file with deploy.yml's thresholds now: every simulated action, and a summary per role"
|
|
60
|
+
option :samples, required: true, desc: "The --record file (JSONL, one tick per line) to replay"
|
|
61
|
+
option :json, type: :boolean, default: false, desc: "Print the simulation as JSON"
|
|
62
|
+
def simulate
|
|
63
|
+
raise ArgumentError, "No such file: #{options[:samples]}" unless File.file?(options[:samples])
|
|
64
|
+
|
|
65
|
+
simulation = Dash::Autoscale::Simulation.new(path: options[:samples], config: DASH.config, roles: DASH.specific_roles&.map(&:name)).run
|
|
66
|
+
options[:json] ? puts(JSON.pretty_generate(simulation)) : print_simulation(simulation)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
desc "history [ROLE]", "Show the controller's last decisions (of ROLE, or of every role)"
|
|
70
|
+
option :lines, type: :numeric, default: Dash::Diagnostics::AutoscaleDecisions::DEFAULT_LINES, desc: "How many decisions"
|
|
71
|
+
option :json, type: :boolean, default: false, desc: "Print the decisions as JSON"
|
|
72
|
+
def history(role_name = nil)
|
|
73
|
+
role = scaled_role(role_name).name if role_name
|
|
74
|
+
history = -> { Dash::Diagnostics::AutoscaleDecisions.new(roles: ([ role ] if role), lines: options[:lines]).to_h }
|
|
75
|
+
|
|
76
|
+
if options[:json]
|
|
77
|
+
puts_json(&history)
|
|
78
|
+
else
|
|
79
|
+
pre_connect_if_required
|
|
80
|
+
print_history(DASH.with_verbosity(:error) { history.call })
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
desc "status", "Show which controller runs, when it last ticked, and the paused roles"
|
|
85
|
+
option :json, type: :boolean, default: false, desc: "Print the status as JSON"
|
|
86
|
+
def status
|
|
87
|
+
status = -> { Dash::Diagnostics::ControllerStatus.new.to_h }
|
|
88
|
+
|
|
89
|
+
if options[:json]
|
|
90
|
+
puts_json(&status)
|
|
91
|
+
else
|
|
92
|
+
pre_connect_if_required
|
|
93
|
+
print_status(DASH.with_verbosity(:error) { status.call })
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
desc "pause ROLE", "Stop the controller from scaling ROLE, for good or --for a while (resume ROLE lifts it)"
|
|
98
|
+
option :for, desc: "How long: seconds, or a number followed by s, m, h or d (30m, 2h, 1d)"
|
|
99
|
+
def pause(role_name)
|
|
100
|
+
role = scaled_role(role_name)
|
|
101
|
+
now = Time.now.utc
|
|
102
|
+
ends_at = options[:for] ? now + pause_duration(options[:for]) : :indefinite
|
|
103
|
+
pause = Dash::Autoscale::Pause.new(ends_at: ends_at, by: performer, at: now)
|
|
104
|
+
until_text = pause.indefinite? ? "until resumed" : "until #{Dash::Autoscale::Timestamp.dump(ends_at)}"
|
|
105
|
+
|
|
106
|
+
on(Dash::Autoscale::StateStore.host) do
|
|
107
|
+
store = Dash::Autoscale::StateStore.new(self)
|
|
108
|
+
store.ensure_directory
|
|
109
|
+
store.write_pause(role.name, pause.to_h)
|
|
110
|
+
end
|
|
111
|
+
audit "Paused autoscaling of #{role} #{until_text}"
|
|
112
|
+
|
|
113
|
+
say "Paused autoscaling of #{role} #{until_text}", :magenta
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
desc "resume ROLE", "Let the controller scale ROLE again"
|
|
117
|
+
def resume(role_name)
|
|
118
|
+
role = scaled_role(role_name)
|
|
119
|
+
|
|
120
|
+
on(Dash::Autoscale::StateStore.host) do
|
|
121
|
+
Dash::Autoscale::StateStore.new(self).remove_pause(role.name)
|
|
122
|
+
end
|
|
123
|
+
audit "Resumed autoscaling of #{role}"
|
|
124
|
+
|
|
125
|
+
say "Resumed autoscaling of #{role}", :magenta
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# What Dash::Autoscale::Controller asks of the CLI.
|
|
129
|
+
no_commands do
|
|
130
|
+
# A fresh deploy.yml every tick, as `dash mcp` reads one per question, with the run's
|
|
131
|
+
# --roles applied again (reconfigure drops them).
|
|
132
|
+
def reconfigure!
|
|
133
|
+
roles = options[:roles]&.split(",")
|
|
134
|
+
DASH.reconfigure(config_file: Pathname.new(File.expand_path(options[:config_file])), destination: options[:destination], explicit_roles: roles)
|
|
135
|
+
DASH.specific_roles = roles
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# `dash scale set ROLE COUNT`, waiting up to autoscale/lock_wait_timeout for the deploy
|
|
139
|
+
# lock. The invocation is reset even when it fails, or Thor would skip the next one.
|
|
140
|
+
def invoke_scale_set(role, count)
|
|
141
|
+
with_lock_wait do
|
|
142
|
+
invoke "dash:cli:scale:set", [ role.name, count.to_s ], scale_options
|
|
143
|
+
ensure
|
|
144
|
+
reset_invocation(Dash::Cli::Scale)
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def replace_member(role, member, count:)
|
|
149
|
+
with_lock_wait do
|
|
150
|
+
Dash::Cli::Scale.new([], scale_options, invocations: { Dash::Cli::Scale => [ "set" ] }).replace_unreachable(role, member, count: count)
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def report(message, color = nil)
|
|
155
|
+
say message, color
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
private
|
|
160
|
+
# A signal during the tick lets it finish (a join in flight included), then it exits.
|
|
161
|
+
def run_once(controller)
|
|
162
|
+
trapping_signals(controller) do
|
|
163
|
+
controller.start
|
|
164
|
+
controller.tick
|
|
165
|
+
end
|
|
166
|
+
ensure
|
|
167
|
+
controller.release
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def run_loop(controller)
|
|
171
|
+
server = Dash::Autoscale::MetricsServer.new(controller.metrics, bind: options[:metrics_bind], port: options[:metrics_port]).start if options[:metrics_port]
|
|
172
|
+
say "Serving autoscale metrics on http://#{options[:metrics_bind]}:#{server.port}/metrics", :magenta if server
|
|
173
|
+
say "Autoscale controller #{controller.id}#{" (dry run)" if options[:dry_run]} ticking every #{controller.interval}s", :magenta
|
|
174
|
+
|
|
175
|
+
trapping_signals(controller) { controller.run }
|
|
176
|
+
ensure
|
|
177
|
+
server&.stop
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# TERM and INT only ask the controller to stop: the tick in progress, and a join in it, finish.
|
|
181
|
+
def trapping_signals(controller)
|
|
182
|
+
previous = %w[ TERM INT ].to_h { |signal| [ signal, trap(signal) { controller.stop! } ] }
|
|
183
|
+
yield
|
|
184
|
+
ensure
|
|
185
|
+
previous&.each { |signal, handler| trap(signal, handler || "DEFAULT") }
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def tick_interval
|
|
189
|
+
return unless options[:interval]
|
|
190
|
+
|
|
191
|
+
interval = Integer(options[:interval].to_s, 10, exception: false)
|
|
192
|
+
if interval.nil? || interval < MINIMUM_INTERVAL
|
|
193
|
+
raise ArgumentError, "--interval must be a whole number of seconds, at least #{MINIMUM_INTERVAL}, not #{options[:interval]}"
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
interval
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def with_lock_wait
|
|
200
|
+
saved = [ DASH.lock_wait, DASH.lock_wait_timeout, DASH.lock_wait_interval ]
|
|
201
|
+
DASH.lock_wait, DASH.lock_wait_timeout, DASH.lock_wait_interval = true, DASH.config.autoscale.lock_wait_timeout, LOCK_WAIT_INTERVAL
|
|
202
|
+
yield
|
|
203
|
+
ensure
|
|
204
|
+
DASH.lock_wait, DASH.lock_wait_timeout, DASH.lock_wait_interval = saved
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def scale_options
|
|
208
|
+
options.slice("config_file", "destination", "skip_hooks", "verbose", "quiet")
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def scaled_role(role_name)
|
|
212
|
+
role = DASH.config.role(role_name) || raise(ArgumentError, "No role named #{role_name}, expected one of #{DASH.config.roles.map(&:name).join(", ")}")
|
|
213
|
+
raise ArgumentError, "#{role} has no scale, so the autoscale controller never touches it" unless role.scaled?
|
|
214
|
+
|
|
215
|
+
role
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def pause_duration(value)
|
|
219
|
+
seconds = Dash::Autoscale::Duration.parse(value)
|
|
220
|
+
raise ArgumentError, "--for must be longer than 0 seconds" unless seconds.positive?
|
|
221
|
+
|
|
222
|
+
seconds
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def performer
|
|
226
|
+
Dash::Git.user_name.presence || ENV["USER"] || "unknown"
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# On the primary host, where `dash audit` reads, wherever the state lives.
|
|
230
|
+
def audit(line)
|
|
231
|
+
on(DASH.config.primary_role.baseline_hosts.first) { execute *DASH.auditor.record(line), verbosity: :debug }
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def print_explain(explain)
|
|
235
|
+
decision = explain[:decision]
|
|
236
|
+
say "#{decision[:role]}: #{decision[:action]} #{decision[:from].inspect} -> #{decision[:to].inspect} (#{decision[:reasons].join(", ")})"
|
|
237
|
+
say " not controlled: it has no scale/schedule or scale/signal, so dash autoscale run leaves it alone", :yellow unless explain[:controlled]
|
|
238
|
+
say " error: #{decision[:error]}", :red if decision[:error]
|
|
239
|
+
say " eligible at: #{decision[:eligible_at]}" if decision[:eligible_at]
|
|
240
|
+
decision[:inputs].each { |name, value| say " #{name}: #{value.to_json}" unless value.nil? || value == [] || value == {} }
|
|
241
|
+
say " state on #{explain[:state_host]} could not be read (#{explain[:state_error]}), evaluated without it", :yellow if explain[:state_error]
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def print_signal(sample)
|
|
245
|
+
return say(sample[:message], :yellow) unless sample[:configured]
|
|
246
|
+
|
|
247
|
+
signal, band = sample[:signal], sample[:band]
|
|
248
|
+
if signal[:error] || signal[:value].nil?
|
|
249
|
+
say "#{sample[:role]}: #{signal[:source]} could not be read from #{sample[:from]}: #{signal[:error] || "no value"}", :red
|
|
250
|
+
else
|
|
251
|
+
position = sample[:position] == "inside" ? "inside the band" : "#{sample[:position]} the band"
|
|
252
|
+
say "#{sample[:role]}: #{signal[:source]} #{signal[:value]}, #{position} (up above #{band[:up_above]} for #{band[:up_for]}s, " \
|
|
253
|
+
"down below #{band[:down_below]} for #{band[:down_for]}s), read from #{sample[:from]} in #{signal[:latency_ms]}ms"
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
sample[:guards].each do |guard|
|
|
257
|
+
name = "hold_when/#{guard[:index]} #{guard[:source]}"
|
|
258
|
+
reading = guard[:value].nil? ? "#{name}: #{guard[:error] || "no value"}" : "#{name} #{guard[:value]}#{" > #{guard[:above]}" if guard[:holding]}"
|
|
259
|
+
say " #{reading}#{", holding a scale-out" if guard[:holding]} (#{guard[:latency_ms]}ms)", (:yellow if guard[:holding])
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def print_simulation(simulation)
|
|
264
|
+
simulation[:roles].each do |role|
|
|
265
|
+
say "#{role[:role]}: #{role[:ticks]} #{"tick".pluralize(role[:ticks])}, #{role[:replayed]} replayed"
|
|
266
|
+
role[:actions].each { |action| say " #{action[:at]} #{action[:summary]}" }
|
|
267
|
+
say " simulated: #{simulation_summary(role[:simulated])}"
|
|
268
|
+
say " recorded: #{simulation_summary(role[:recorded])}"
|
|
269
|
+
if role[:not_replayable].positive?
|
|
270
|
+
say " #{role[:not_replayable]} #{"tick".pluralize(role[:not_replayable])} not replayable: recorded without a signal (before Phase 3), or not evaluated", :yellow
|
|
271
|
+
end
|
|
272
|
+
say " #{role[:unrecorded_guards]} hold_when #{"guard".pluralize(role[:unrecorded_guards])} not in the recording, left out", :yellow if role[:unrecorded_guards]
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
say "No role in #{simulation[:file]} to replay", :yellow if simulation[:roles].empty?
|
|
276
|
+
simulation[:skipped].each { |skipped| say "#{skipped[:role]}: skipped, #{skipped[:reason]}", :yellow }
|
|
277
|
+
simulation[:errors].each { |error| say "line #{error[:line]}: #{error[:error]}", :red }
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def simulation_summary(summary)
|
|
281
|
+
return "no tick replayed" if summary[:peak].nil?
|
|
282
|
+
|
|
283
|
+
"#{summary[:scale_outs]} #{"scale-out".pluralize(summary[:scale_outs])}, #{summary[:scale_ins]} #{"scale-in".pluralize(summary[:scale_ins])}, " \
|
|
284
|
+
"peak #{summary[:peak]}, minimum #{summary[:minimum]}, #{summary[:container_seconds]} container-seconds"
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
def print_history(history)
|
|
288
|
+
return say("Could not read the decision log on #{history[:host]}: #{history[:error]}", :red) if history[:error]
|
|
289
|
+
return say("No autoscale decisions logged yet") if history[:decisions].empty?
|
|
290
|
+
|
|
291
|
+
history[:decisions].each do |decision|
|
|
292
|
+
counts = decision["action"] == "replace_member" ? decision.dig("inputs", "replace") : "#{decision["from"].inspect} -> #{decision["to"].inspect}"
|
|
293
|
+
error = " - #{decision["error"]}" if decision["error"]
|
|
294
|
+
say "#{decision["at"]} #{decision["role"]}: #{decision["action"]} #{counts} (#{decision["reasons"].to_a.join(", ")})#{error}", (:red if error)
|
|
295
|
+
end
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
def print_status(status)
|
|
299
|
+
return say("Could not read the controller state on #{status[:state_host]}: #{status[:error]}", :red) if status[:error]
|
|
300
|
+
|
|
301
|
+
if (controller = status[:controller])
|
|
302
|
+
tick = controller[:age_seconds] ? "last tick #{controller[:age_seconds]}s ago" : "no tick yet"
|
|
303
|
+
say "Controller #{controller[:controller_id]} on #{controller[:hostname]} (pid #{controller[:pid]}, dash #{controller[:version]}, #{controller[:mode]}): " \
|
|
304
|
+
"#{controller[:status]}, #{tick}", controller[:status] == "running" ? :green : :yellow
|
|
305
|
+
else
|
|
306
|
+
say "No autoscale controller has run for this app", :yellow
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
say "Scheduled roles: #{status[:roles].any? ? status[:roles].join(", ") : "none"}"
|
|
310
|
+
status[:pauses].each { |pause| say "#{pause[:role]} paused #{pause[:until] == "indefinite" ? "until resumed" : "until #{pause[:until]}"} by #{pause[:by]}", :yellow }
|
|
311
|
+
end
|
|
312
|
+
end
|
data/lib/dash/cli/base.rb
CHANGED
|
@@ -103,9 +103,10 @@ module Dash::Cli
|
|
|
103
103
|
# Process-scoped rather than commander-scoped: Dash::Cli::Alias::Command
|
|
104
104
|
# resets DASH and re-enters Dash::Cli::Main.start, so an aliased command
|
|
105
105
|
# builds a second commander and would otherwise warn twice. `dash migrate`
|
|
106
|
-
# is exempt - telling an operator to run the command they are running is noise
|
|
106
|
+
# is exempt - telling an operator to run the command they are running is noise - and
|
|
107
|
+
# so is `dash mcp`, whose stdout is the protocol.
|
|
107
108
|
def warn_on_legacy_project_directory(command_name)
|
|
108
|
-
return if
|
|
109
|
+
return if %w[ migrate mcp ].include?(command_name)
|
|
109
110
|
return if Dash::Cli::Base.legacy_project_directory_warned
|
|
110
111
|
return unless Dash::ProjectDirectory.legacy?
|
|
111
112
|
|
|
@@ -129,7 +130,9 @@ module Dash::Cli
|
|
|
129
130
|
commander.configure \
|
|
130
131
|
config_file: Pathname.new(File.expand_path(options[:config_file])),
|
|
131
132
|
destination: options[:destination],
|
|
132
|
-
version: options[:version]
|
|
133
|
+
version: options[:version],
|
|
134
|
+
explicit_hosts: options[:hosts]&.split(","),
|
|
135
|
+
explicit_roles: options[:roles]&.split(",")
|
|
133
136
|
|
|
134
137
|
commander.specific_hosts = options[:hosts]&.split(",")
|
|
135
138
|
commander.specific_roles = options[:roles]&.split(",")
|
|
@@ -544,6 +547,15 @@ module Dash::Cli
|
|
|
544
547
|
super
|
|
545
548
|
end
|
|
546
549
|
|
|
550
|
+
# A diagnostic as pure JSON on stdout, returned too. capture_with_info logs every command
|
|
551
|
+
# it runs at info, onto stdout, so the capture runs at :error - a `| jq` must see only the JSON.
|
|
552
|
+
def puts_json(ssh: true)
|
|
553
|
+
DASH.with_verbosity(:error) do
|
|
554
|
+
pre_connect_if_required if ssh
|
|
555
|
+
yield.tap { |snapshot| puts JSON.pretty_generate(snapshot) }
|
|
556
|
+
end
|
|
557
|
+
end
|
|
558
|
+
|
|
547
559
|
def pre_connect_if_required
|
|
548
560
|
if !DASH.connected?
|
|
549
561
|
run_hook "pre-connect", secrets: true unless options[:skip_hooks]
|
data/lib/dash/cli/lock.rb
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
class Dash::Cli::Lock < Dash::Cli::Base
|
|
2
2
|
desc "status", "Report lock status"
|
|
3
3
|
option :server, type: :boolean, default: false, desc: "Report the shared server lock instead of the deploy lock"
|
|
4
|
+
option :json, type: :boolean, default: false, desc: "Print the deploy lock status as JSON"
|
|
4
5
|
def status
|
|
5
|
-
if options[:
|
|
6
|
+
if options[:json]
|
|
7
|
+
raise ArgumentError, "--json reports the deploy lock; it cannot be combined with --server" if options[:server]
|
|
8
|
+
|
|
9
|
+
puts_json { Dash::Diagnostics::Lock.new.to_h }
|
|
10
|
+
elsif options[:server]
|
|
6
11
|
report_server_lock_status
|
|
7
12
|
else
|
|
8
13
|
handle_missing_lock do
|
data/lib/dash/cli/main.rb
CHANGED
|
@@ -151,7 +151,10 @@ class Dash::Cli::Main < Dash::Cli::Base
|
|
|
151
151
|
end
|
|
152
152
|
|
|
153
153
|
desc "audit", "Show audit log from servers"
|
|
154
|
+
option :json, type: :boolean, default: false, desc: "Print the audit log as JSON, parsed per line"
|
|
154
155
|
def audit
|
|
156
|
+
return puts_json { Dash::Diagnostics::Audit.new.to_h } if options[:json]
|
|
157
|
+
|
|
155
158
|
quiet = options[:quiet]
|
|
156
159
|
on(DASH.hosts) do |host|
|
|
157
160
|
puts_by_host host, capture_with_info(*DASH.auditor.reveal), quiet: quiet
|
|
@@ -159,7 +162,10 @@ class Dash::Cli::Main < Dash::Cli::Base
|
|
|
159
162
|
end
|
|
160
163
|
|
|
161
164
|
desc "config", "Show combined config (including secrets!)"
|
|
165
|
+
option :json, type: :boolean, default: false, desc: "Print the config and its topology (roles, hosts, proxies) as JSON"
|
|
162
166
|
def config
|
|
167
|
+
return puts_json(ssh: false) { Dash::Diagnostics::Config.new.to_h } if options[:json]
|
|
168
|
+
|
|
163
169
|
run_locally do
|
|
164
170
|
puts Dash::Utils.redacted(DASH.config.to_h).to_yaml
|
|
165
171
|
end
|
|
@@ -177,12 +183,15 @@ class Dash::Cli::Main < Dash::Cli::Base
|
|
|
177
183
|
puts "No documentation found for #{section}"
|
|
178
184
|
end
|
|
179
185
|
|
|
180
|
-
desc "doctor", "Diagnose deploy readiness of servers, registry, proxy, ports, DNS, certificates,
|
|
186
|
+
desc "doctor", "Diagnose deploy readiness of servers, registry, proxy, ports, DNS, certificates, per-role readiness gates, and proxy drift"
|
|
187
|
+
option :json, type: :boolean, default: false, desc: "Print the results as JSON (exits 1 when a check fails)"
|
|
181
188
|
def doctor
|
|
189
|
+
return doctor_json if options[:json]
|
|
190
|
+
|
|
182
191
|
say "Running readiness checks...", :magenta
|
|
183
|
-
|
|
192
|
+
doctor_pre_connect
|
|
184
193
|
|
|
185
|
-
doctor = Dash::
|
|
194
|
+
doctor = Dash::Diagnostics::Doctor.new
|
|
186
195
|
doctor.run
|
|
187
196
|
|
|
188
197
|
print_doctor_report doctor.results
|
|
@@ -194,6 +203,18 @@ class Dash::Cli::Main < Dash::Cli::Base
|
|
|
194
203
|
end
|
|
195
204
|
end
|
|
196
205
|
|
|
206
|
+
desc "mcp", "Serve read-only diagnostics to an AI agent over stdio (Model Context Protocol)"
|
|
207
|
+
option :allow_logs, type: :boolean, default: false, desc: "Enable the logs tool (container logs can carry personal data; or DASH_MCP_ALLOW_LOGS=true)"
|
|
208
|
+
def mcp
|
|
209
|
+
protocol = Dash::Mcp.reserve_stdout!
|
|
210
|
+
Dash::Mcp.load!
|
|
211
|
+
Dash::Mcp::Runner.authorize!(ENV)
|
|
212
|
+
DASH.verbosity = :error
|
|
213
|
+
pre_connect_if_required
|
|
214
|
+
|
|
215
|
+
Dash::Mcp::Runner.run(mcp_session, output: protocol)
|
|
216
|
+
end
|
|
217
|
+
|
|
197
218
|
desc "init", "Create config stub in config/deploy.yml and secrets stub in .dash"
|
|
198
219
|
option :bundle, type: :boolean, default: false, desc: "Add dash to the Gemfile and create a bin/dash binstub"
|
|
199
220
|
def init
|
|
@@ -301,6 +322,9 @@ class Dash::Cli::Main < Dash::Cli::Base
|
|
|
301
322
|
desc "app", "Manage application"
|
|
302
323
|
subcommand "app", Dash::Cli::App
|
|
303
324
|
|
|
325
|
+
desc "autoscale", "Run the autoscale controller, and read what it decided"
|
|
326
|
+
subcommand "autoscale", Dash::Cli::Autoscale
|
|
327
|
+
|
|
304
328
|
desc "build", "Build application image"
|
|
305
329
|
subcommand "build", Dash::Cli::Build
|
|
306
330
|
|
|
@@ -319,6 +343,9 @@ class Dash::Cli::Main < Dash::Cli::Base
|
|
|
319
343
|
desc "registry", "Login and -out of the image registry"
|
|
320
344
|
subcommand "registry", Dash::Cli::Registry
|
|
321
345
|
|
|
346
|
+
desc "scale", "Change how many containers of a role run, without a deploy"
|
|
347
|
+
subcommand "scale", Dash::Cli::Scale
|
|
348
|
+
|
|
322
349
|
desc "secrets", "Helpers for extracting secrets"
|
|
323
350
|
subcommand "secrets", Dash::Cli::Secrets
|
|
324
351
|
|
|
@@ -352,10 +379,34 @@ class Dash::Cli::Main < Dash::Cli::Base
|
|
|
352
379
|
{ "version" => DASH.config.version }.merge(base_options)
|
|
353
380
|
end
|
|
354
381
|
|
|
382
|
+
# Pure JSON on stdout, so the failure is the exit status rather than an ERROR line.
|
|
383
|
+
# The pre-connect hook is handed every host, and with the pool unreadable those are
|
|
384
|
+
# unknown. The doctor carries on without it and reports the pool failure itself.
|
|
385
|
+
# `--json` keeps stdout for the JSON, so it says so on stderr.
|
|
386
|
+
def doctor_pre_connect(json: false)
|
|
387
|
+
pre_connect_if_required
|
|
388
|
+
rescue Dash::Autoscale::ProviderError => e
|
|
389
|
+
message = "Skipped the pre-connect hook, the hosts are unknown: #{e.message}"
|
|
390
|
+
json ? warn(message) : say(message, :yellow)
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
def doctor_json
|
|
394
|
+
DASH.with_verbosity(:error) { doctor_pre_connect(json: true) }
|
|
395
|
+
exit 1 unless puts_json(ssh: false) { Dash::Diagnostics::Doctor.new.to_h }[:successful]
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
# The scope the operator started `dash mcp` with is the ceiling for every question.
|
|
399
|
+
def mcp_session
|
|
400
|
+
Dash::Mcp::Session.new(
|
|
401
|
+
config_file: Pathname.new(File.expand_path(options[:config_file])), destination: options[:destination], version: options[:version],
|
|
402
|
+
hosts: options[:primary] ? [ DASH.primary_host ] : options[:hosts]&.split(","), roles: options[:roles]&.split(","),
|
|
403
|
+
allow_logs: Dash::Mcp::Runner.allow_logs?(options[:allow_logs], ENV), redactor: Dash::Diagnostics::Redactor.for(DASH.config))
|
|
404
|
+
end
|
|
405
|
+
|
|
355
406
|
def print_doctor_report(results)
|
|
356
407
|
results.group_by(&:title).each do |title, rows|
|
|
357
408
|
say title
|
|
358
|
-
rows.each { |row| say " #{row}", Dash::
|
|
409
|
+
rows.each { |row| say " #{row}", Dash::Diagnostics::Doctor::STATUS_COLORS[row.status] }
|
|
359
410
|
end
|
|
360
411
|
end
|
|
361
412
|
|
|
@@ -377,14 +428,32 @@ class Dash::Cli::Main < Dash::Cli::Base
|
|
|
377
428
|
|
|
378
429
|
say "Deploying #{config.service}#{" to #{config.destination}" if config.destination} (version #{config.abbreviated_version})", :magenta
|
|
379
430
|
config.roles.each do |role|
|
|
380
|
-
|
|
381
|
-
say " #{role.name}: #{
|
|
431
|
+
replicas = " #{role.replicas}" if role.replicas.scalable?
|
|
432
|
+
say " #{role.name}: #{banner_hosts(role)}#{replicas} — readiness: #{role.readiness_description}", (:yellow if role.readiness_source == :none)
|
|
382
433
|
end
|
|
383
434
|
say " proxy: #{config.proxy_hosts.join(", ")}" if config.proxy_hosts.any?
|
|
384
435
|
if config.proxy.load_balancing?
|
|
385
|
-
|
|
386
|
-
say " loadbalancer: #{config.proxy.effective_loadbalancer}#{reason}"
|
|
436
|
+
say " loadbalancer: #{config.proxy.effective_loadbalancer}#{loadbalancer_reason(config) unless config.proxy.loadbalancer.present?}"
|
|
387
437
|
end
|
|
388
438
|
say " timeouts: deploy #{config.deploy_timeout}s, drain #{config.drain_timeout}s, readiness delay #{config.readiness_delay}s"
|
|
389
439
|
end
|
|
440
|
+
|
|
441
|
+
def banner_hosts(role)
|
|
442
|
+
baseline = role.baseline_hosts
|
|
443
|
+
hosts = "#{baseline.count} #{"host".pluralize(baseline.count)} (#{baseline.join(", ")})"
|
|
444
|
+
return hosts unless role.scaled?
|
|
445
|
+
|
|
446
|
+
active = role.active_members.map(&:host)
|
|
447
|
+
"#{hosts} + #{active.count} of #{role.members.count} members active#{" (#{active.join(", ")})" if active.any?}"
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
def loadbalancer_reason(config)
|
|
451
|
+
primary = config.primary_role
|
|
452
|
+
|
|
453
|
+
if primary.running_proxy? && primary.baseline_hosts.count > 1
|
|
454
|
+
" (auto-enabled: primary role #{primary.name} has #{primary.baseline_hosts.count} hosts)"
|
|
455
|
+
elsif (scaled = config.scaled_roles.find(&:running_proxy?))
|
|
456
|
+
" (auto-enabled: role #{scaled.name} scales across hosts)"
|
|
457
|
+
end
|
|
458
|
+
end
|
|
390
459
|
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# `dash proxy services` and `dash proxy drift`: the read-only questions about what the
|
|
2
|
+
# proxies route, kept out of the proxy command file that boots, reboots and manages them.
|
|
3
|
+
# The JSON is the diagnostic's own `to_h`; the rest here is the table an operator reads.
|
|
4
|
+
module Dash::Cli::Proxy::Inspection
|
|
5
|
+
extend ActiveSupport::Concern
|
|
6
|
+
|
|
7
|
+
included do
|
|
8
|
+
desc "services", "Show what dash-proxy routes for this deploy on each proxy host (and the load balancer)"
|
|
9
|
+
option :json, type: :boolean, default: false, desc: "Print the services as JSON"
|
|
10
|
+
def services
|
|
11
|
+
return puts_json { Dash::Diagnostics::ProxyServices.new.to_h } if options[:json]
|
|
12
|
+
|
|
13
|
+
pre_connect_if_required
|
|
14
|
+
puts services_lines(Dash::Diagnostics::ProxyServices.new.to_h)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
desc "drift", "Compare the running containers with the proxy and load balancer targets"
|
|
18
|
+
option :json, type: :boolean, default: false, desc: "Print the drift as JSON"
|
|
19
|
+
def drift
|
|
20
|
+
return puts_json { Dash::Diagnostics::Drift.take.to_h } if options[:json]
|
|
21
|
+
|
|
22
|
+
pre_connect_if_required
|
|
23
|
+
puts drift_lines(Dash::Diagnostics::Drift.take.to_h)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
def services_lines(snapshot)
|
|
29
|
+
proxies = snapshot[:hosts].flat_map { |host| host_services_lines("Proxy", host) }
|
|
30
|
+
proxies + (snapshot[:loadbalancer] ? host_services_lines("Loadbalancer", snapshot[:loadbalancer]) : [])
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def drift_lines(snapshot)
|
|
34
|
+
if snapshot[:consistent]
|
|
35
|
+
[ "No drift: proxy targets match the running containers" ]
|
|
36
|
+
else
|
|
37
|
+
snapshot[:drift].map { |entry| "#{entry[:code]} #{entry[:host] || entry[:role]}: #{entry[:detail]}" } +
|
|
38
|
+
Array(snapshot[:unread]).map { |host| "unread #{host[:host]}: could not read #{host[:source]} (#{host[:error]})" }
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def host_services_lines(type, host)
|
|
43
|
+
return [ "#{type} Host: #{host[:host]}", " ERROR #{host[:error]}" ] if host[:error]
|
|
44
|
+
|
|
45
|
+
services = host[:services].map do |name, service|
|
|
46
|
+
targets = Array(service["targets"]).presence || service["target"].to_s.split(",")
|
|
47
|
+
" #{name} #{service["host"].presence || "*"} -> #{targets.join(", ").presence || "no targets"}#{" (#{service["state"]})" if service["state"]}"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
[ "#{type} Host: #{host[:host]}", *services.presence || [ " no services" ] ]
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Points the load balancer at a target list: every host of every proxied role by default.
|
|
2
|
+
# `dash proxy loadbalancer deploy` and `dash scale set` share it, so a member leaving the
|
|
3
|
+
# pool is the same deploy with that host left out - dash-proxy replaces the whole list,
|
|
4
|
+
# draining the hosts that are no longer on it. Runs inside `on(<load balancer host>)`.
|
|
5
|
+
class Dash::Cli::Proxy::LoadbalancerDeploy
|
|
6
|
+
attr_reader :host, :sshkit
|
|
7
|
+
delegate :execute, :info, to: :sshkit
|
|
8
|
+
|
|
9
|
+
def initialize(host, sshkit, targets: DASH.loadbalancer_config.target_hosts)
|
|
10
|
+
@host = host
|
|
11
|
+
@sshkit = sshkit
|
|
12
|
+
@targets = targets
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def run
|
|
16
|
+
Dash::Cli::Proxy::LoadbalancerClaim.new(host, sshkit).claim_service
|
|
17
|
+
info "Deploying to loadbalancer on #{host} with targets: #{@targets.join(', ')}"
|
|
18
|
+
execute *DASH.loadbalancer.deploy(targets: @targets)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -96,7 +96,7 @@ class Dash::Cli::Proxy::LoadbalancerReboot
|
|
|
96
96
|
# `list --json` returns {"services": {"<name>": ...}} - exact key
|
|
97
97
|
# membership, same as the per-host proxy reboot's verification.
|
|
98
98
|
def verify_service
|
|
99
|
-
listed =
|
|
99
|
+
listed = Dash::Commands::Proxy::Services.parse(capture_with_info(*DASH.loadbalancer.list(json: true))).keys
|
|
100
100
|
|
|
101
101
|
unless listed.include?(DASH.config.service)
|
|
102
102
|
raise Dash::Cli::BootError, "the load balancer on #{host} is missing service #{DASH.config.service} after reboot"
|
|
@@ -143,16 +143,11 @@ class Dash::Cli::Proxy::Reboot
|
|
|
143
143
|
registered_services = []
|
|
144
144
|
|
|
145
145
|
DASH.roles_on(host).select(&:running_proxy?).each do |role|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip.presence
|
|
149
|
-
next unless version
|
|
150
|
-
|
|
151
|
-
endpoint = capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).strip.presence
|
|
152
|
-
next unless endpoint
|
|
146
|
+
endpoints = Dash::Cli::App::RunningTargets.new(sshkit, role: role, host: host).container_ids(only_running: true)
|
|
147
|
+
next if endpoints.empty?
|
|
153
148
|
|
|
154
149
|
info "Re-registering #{role.container_prefix} with dash-proxy on #{host}..."
|
|
155
|
-
execute *app.deploy(
|
|
150
|
+
execute *DASH.app(role: role, host: host).deploy(targets: endpoints)
|
|
156
151
|
registered_services << role.container_prefix
|
|
157
152
|
end
|
|
158
153
|
|
|
@@ -177,7 +172,7 @@ class Dash::Cli::Proxy::Reboot
|
|
|
177
172
|
def verify_services(registered_services)
|
|
178
173
|
return if registered_services.empty?
|
|
179
174
|
|
|
180
|
-
listed =
|
|
175
|
+
listed = Dash::Commands::Proxy::Services.parse(capture_with_info(*proxy.list(json: true))).keys
|
|
181
176
|
missing = registered_services - listed
|
|
182
177
|
|
|
183
178
|
if missing.any?
|