dash 4.1.2 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. checksums.yaml +4 -4
  2. data/lib/dash/autoscale/connections/builder.rb +51 -0
  3. data/lib/dash/autoscale/connections.rb +102 -0
  4. data/lib/dash/autoscale/controller.rb +296 -0
  5. data/lib/dash/autoscale/cron.rb +75 -0
  6. data/lib/dash/autoscale/decision.rb +89 -0
  7. data/lib/dash/autoscale/duration.rb +19 -0
  8. data/lib/dash/autoscale/evaluation.rb +52 -0
  9. data/lib/dash/autoscale/guard.rb +48 -0
  10. data/lib/dash/autoscale/lease.rb +55 -0
  11. data/lib/dash/autoscale/lease_held.rb +2 -0
  12. data/lib/dash/autoscale/lease_lost.rb +2 -0
  13. data/lib/dash/autoscale/member.rb +28 -0
  14. data/lib/dash/autoscale/metrics.rb +70 -0
  15. data/lib/dash/autoscale/metrics_server.rb +71 -0
  16. data/lib/dash/autoscale/observation.rb +41 -0
  17. data/lib/dash/autoscale/pause.rb +34 -0
  18. data/lib/dash/autoscale/policy.rb +198 -0
  19. data/lib/dash/autoscale/pool.rb +103 -0
  20. data/lib/dash/autoscale/prometheus_client.rb +76 -0
  21. data/lib/dash/autoscale/provider/base.rb +43 -0
  22. data/lib/dash/autoscale/provider/exec.rb +94 -0
  23. data/lib/dash/autoscale/provider/upcloud.rb +140 -0
  24. data/lib/dash/autoscale/provider.rb +9 -0
  25. data/lib/dash/autoscale/provider_error.rb +4 -0
  26. data/lib/dash/autoscale/read_only_query.rb +38 -0
  27. data/lib/dash/autoscale/reader.rb +17 -0
  28. data/lib/dash/autoscale/recorder.rb +13 -0
  29. data/lib/dash/autoscale/role_state.rb +110 -0
  30. data/lib/dash/autoscale/sample.rb +13 -0
  31. data/lib/dash/autoscale/signal/pgbus.rb +19 -0
  32. data/lib/dash/autoscale/signal/prometheus.rb +8 -0
  33. data/lib/dash/autoscale/signal/sidekiq.rb +49 -0
  34. data/lib/dash/autoscale/signal.rb +31 -0
  35. data/lib/dash/autoscale/simulation.rb +200 -0
  36. data/lib/dash/autoscale/state_store.rb +132 -0
  37. data/lib/dash/autoscale/timestamp.rb +15 -0
  38. data/lib/dash/autoscale/tunnel.rb +48 -0
  39. data/lib/dash/cli/accessory.rb +16 -0
  40. data/lib/dash/cli/app/boot.rb +69 -44
  41. data/lib/dash/cli/app/boot_state.rb +70 -0
  42. data/lib/dash/cli/app/rollout_boot.rb +28 -9
  43. data/lib/dash/cli/app/running_targets.rb +39 -0
  44. data/lib/dash/cli/app/slot_version.rb +29 -0
  45. data/lib/dash/cli/app.rb +89 -32
  46. data/lib/dash/cli/autoscale.rb +312 -0
  47. data/lib/dash/cli/base.rb +15 -3
  48. data/lib/dash/cli/lock.rb +6 -1
  49. data/lib/dash/cli/main.rb +77 -8
  50. data/lib/dash/cli/proxy/inspection.rb +52 -0
  51. data/lib/dash/cli/proxy/loadbalancer_deploy.rb +20 -0
  52. data/lib/dash/cli/proxy/loadbalancer_reboot.rb +1 -1
  53. data/lib/dash/cli/proxy/reboot.rb +4 -9
  54. data/lib/dash/cli/proxy.rb +12 -7
  55. data/lib/dash/cli/prune.rb +5 -3
  56. data/lib/dash/cli/report.rb +4 -1
  57. data/lib/dash/cli/scale/host_plan.rb +63 -0
  58. data/lib/dash/cli/scale/join.rb +128 -0
  59. data/lib/dash/cli/scale/leave.rb +58 -0
  60. data/lib/dash/cli/scale/placement.rb +54 -0
  61. data/lib/dash/cli/scale/power_off.rb +26 -0
  62. data/lib/dash/cli/scale/replica_join.rb +74 -0
  63. data/lib/dash/cli/scale/replica_leave.rb +121 -0
  64. data/lib/dash/cli/scale.rb +276 -0
  65. data/lib/dash/cli/server.rb +41 -0
  66. data/lib/dash/cli/stats_table.rb +34 -0
  67. data/lib/dash/cli/templates/sample_hooks/post-scale-in.sample +3 -0
  68. data/lib/dash/cli/templates/sample_hooks/post-scale-out.sample +3 -0
  69. data/lib/dash/cli/templates/sample_hooks/pre-scale-in.sample +6 -0
  70. data/lib/dash/cli/templates/sample_hooks/pre-scale-out.sample +6 -0
  71. data/lib/dash/commander.rb +17 -3
  72. data/lib/dash/commands/accessory/proxy.rb +1 -1
  73. data/lib/dash/commands/accessory.rb +21 -2
  74. data/lib/dash/commands/app/containers.rb +28 -3
  75. data/lib/dash/commands/app/logging.rb +13 -2
  76. data/lib/dash/commands/app/proxy.rb +4 -4
  77. data/lib/dash/commands/app.rb +90 -10
  78. data/lib/dash/commands/auditor.rb +2 -2
  79. data/lib/dash/commands/autoscale.rb +133 -0
  80. data/lib/dash/commands/base.rb +4 -0
  81. data/lib/dash/commands/proxy/services.rb +10 -0
  82. data/lib/dash/commands/prune.rb +10 -2
  83. data/lib/dash/commands/server.rb +16 -0
  84. data/lib/dash/configuration/accessory.rb +2 -2
  85. data/lib/dash/configuration/autoscale.rb +123 -0
  86. data/lib/dash/configuration/docs/autoscale.yml +141 -0
  87. data/lib/dash/configuration/docs/configuration.yml +8 -2
  88. data/lib/dash/configuration/docs/proxy.yml +12 -9
  89. data/lib/dash/configuration/docs/role.yml +203 -0
  90. data/lib/dash/configuration/loadbalancer.rb +3 -2
  91. data/lib/dash/configuration/proxy/run.rb +1 -1
  92. data/lib/dash/configuration/proxy.rb +19 -7
  93. data/lib/dash/configuration/role/drain.rb +49 -0
  94. data/lib/dash/configuration/role/replicas.rb +72 -0
  95. data/lib/dash/configuration/role/scale/guard.rb +40 -0
  96. data/lib/dash/configuration/role/scale/signal.rb +93 -0
  97. data/lib/dash/configuration/role/scale/window.rb +37 -0
  98. data/lib/dash/configuration/role/scale.rb +199 -0
  99. data/lib/dash/configuration/role.rb +123 -6
  100. data/lib/dash/configuration/validator/autoscale.rb +126 -0
  101. data/lib/dash/configuration/validator/proxy.rb +13 -0
  102. data/lib/dash/configuration/validator/role.rb +26 -5
  103. data/lib/dash/configuration.rb +126 -15
  104. data/lib/dash/diagnostics/audit.rb +28 -0
  105. data/lib/dash/diagnostics/autoscale_decisions.rb +18 -0
  106. data/lib/dash/diagnostics/autoscale_explain.rb +30 -0
  107. data/lib/dash/diagnostics/autoscale_signal.rb +82 -0
  108. data/lib/dash/diagnostics/base.rb +53 -0
  109. data/lib/dash/diagnostics/config.rb +32 -0
  110. data/lib/dash/diagnostics/container_stats.rb +67 -0
  111. data/lib/dash/diagnostics/containers.rb +20 -0
  112. data/lib/dash/diagnostics/controller_status.rb +44 -0
  113. data/lib/dash/diagnostics/docker_ps.rb +40 -0
  114. data/lib/dash/{cli → diagnostics}/doctor/config_checks.rb +3 -3
  115. data/lib/dash/diagnostics/doctor/controller_checks.rb +46 -0
  116. data/lib/dash/{cli → diagnostics}/doctor/endpoint_checks.rb +12 -4
  117. data/lib/dash/{cli → diagnostics}/doctor/host_checks.rb +5 -4
  118. data/lib/dash/diagnostics/doctor/pool_checks.rb +65 -0
  119. data/lib/dash/diagnostics/doctor/signal_checks.rb +54 -0
  120. data/lib/dash/diagnostics/doctor.rb +182 -0
  121. data/lib/dash/diagnostics/drift.rb +168 -0
  122. data/lib/dash/diagnostics/host_stats.rb +59 -0
  123. data/lib/dash/diagnostics/lines.rb +14 -0
  124. data/lib/dash/diagnostics/lock.rb +27 -0
  125. data/lib/dash/diagnostics/logs.rb +76 -0
  126. data/lib/dash/diagnostics/pool.rb +41 -0
  127. data/lib/dash/diagnostics/proxy_services.rb +36 -0
  128. data/lib/dash/diagnostics/redactor.rb +48 -0
  129. data/lib/dash/diagnostics/reports.rb +17 -0
  130. data/lib/dash/diagnostics/scale.rb +51 -0
  131. data/lib/dash/diagnostics/units.rb +31 -0
  132. data/lib/dash/mcp/base_tool.rb +43 -0
  133. data/lib/dash/mcp/runner.rb +28 -0
  134. data/lib/dash/mcp/server.rb +40 -0
  135. data/lib/dash/mcp/session.rb +51 -0
  136. data/lib/dash/mcp/tools/audit_tool.rb +11 -0
  137. data/lib/dash/mcp/tools/autoscale_decisions_tool.rb +22 -0
  138. data/lib/dash/mcp/tools/autoscale_explain_tool.rb +56 -0
  139. data/lib/dash/mcp/tools/config_tool.rb +10 -0
  140. data/lib/dash/mcp/tools/container_stats_tool.rb +15 -0
  141. data/lib/dash/mcp/tools/containers_tool.rb +10 -0
  142. data/lib/dash/mcp/tools/controller_status_tool.rb +18 -0
  143. data/lib/dash/mcp/tools/deploy_reports_tool.rb +10 -0
  144. data/lib/dash/mcp/tools/doctor_tool.rb +13 -0
  145. data/lib/dash/mcp/tools/drift_tool.rb +16 -0
  146. data/lib/dash/mcp/tools/host_stats_tool.rb +14 -0
  147. data/lib/dash/mcp/tools/lock_status_tool.rb +15 -0
  148. data/lib/dash/mcp/tools/logs_tool.rb +41 -0
  149. data/lib/dash/mcp/tools/pool_members_tool.rb +15 -0
  150. data/lib/dash/mcp/tools/proxy_services_tool.rb +10 -0
  151. data/lib/dash/mcp/tools/scale_status_tool.rb +10 -0
  152. data/lib/dash/mcp/tools/signal_sample_tool.rb +19 -0
  153. data/lib/dash/mcp/transport.rb +14 -0
  154. data/lib/dash/mcp.rb +56 -0
  155. data/lib/dash/sshkit_with_ext.rb +4 -4
  156. data/lib/dash/utils.rb +7 -0
  157. data/lib/dash/version.rb +1 -1
  158. data/lib/dash.rb +2 -0
  159. metadata +119 -5
  160. data/lib/dash/cli/doctor.rb +0 -113
@@ -0,0 +1,312 @@
1
+ # `dash autoscale`: the controller that holds each scheduled or signalled role at its target
2
+ # (Dash::Autoscale::Controller), and the commands that read what it decided and why. The
3
+ # controller's state lives on one host - `autoscale.controller`, or the primary host
4
+ # without it - so every operator's checkout reads the same heartbeat, decisions and pauses.
5
+ class Dash::Cli::Autoscale < Dash::Cli::Base
6
+ MINIMUM_INTERVAL = 5
7
+ LOCK_WAIT_INTERVAL = 5
8
+
9
+ # `run` is a Thor reserved word, so the method has another name.
10
+ map "run" => :start_controller
11
+
12
+ desc "run", "Run the autoscale controller: every interval, scale each role with a scale/schedule or scale/signal toward its target through dash scale set"
13
+ option :once, type: :boolean, default: false, desc: "Run one tick and exit (for cron)"
14
+ option :dry_run, type: :boolean, default: false, desc: "Decide and print, without acting or writing any state"
15
+ option :interval, type: :numeric, desc: "Seconds between ticks (default autoscale/interval)"
16
+ option :metrics_port, type: :numeric, desc: "Serve Prometheus metrics on this port at /metrics"
17
+ option :metrics_bind, default: "127.0.0.1", desc: "Address the metrics endpoint binds"
18
+ option :record, desc: "Append every tick's decisions, with their inputs, to this local JSONL file"
19
+ option :takeover, type: :boolean, default: false, desc: "Start even while another controller's heartbeat is alive (it stops at its next tick)"
20
+ def start_controller
21
+ raise ArgumentError, "dash autoscale run works on whole roles: narrow it with --roles, not --hosts or --primary" if options[:hosts] || options[:primary]
22
+
23
+ controller = Dash::Autoscale::Controller.new(cli: self, dry_run: options[:dry_run], takeover: options[:takeover], interval: tick_interval,
24
+ mode: options[:once] ? "once" : "loop", recorder: (Dash::Autoscale::Recorder.new(options[:record]) if options[:record]))
25
+ pre_connect_if_required
26
+
27
+ options[:once] ? run_once(controller) : run_loop(controller)
28
+ end
29
+
30
+ desc "explain ROLE", "Evaluate ROLE as the controller's next tick would, with every input, without acting"
31
+ option :json, type: :boolean, default: false, desc: "Print the evaluation as JSON"
32
+ def explain(role_name)
33
+ role = scaled_role(role_name)
34
+ explain = -> { Dash::Diagnostics::AutoscaleExplain.new(role: role).to_h }
35
+
36
+ if options[:json]
37
+ puts_json(&explain)
38
+ else
39
+ pre_connect_if_required
40
+ print_explain(DASH.with_verbosity(:error) { explain.call })
41
+ end
42
+ end
43
+
44
+ desc "signal ROLE", "Read one live sample of ROLE's scale/signal and of each scale/hold_when guard, from this machine"
45
+ option :json, type: :boolean, default: false, desc: "Print the sample as JSON"
46
+ def signal(role_name)
47
+ role = scaled_role(role_name)
48
+ sample = -> { Dash::Diagnostics::AutoscaleSignal.new(role: role).to_h }
49
+
50
+ if options[:json]
51
+ puts_json(&sample)
52
+ else
53
+ # A source with `via:` is reached through an SSH tunnel.
54
+ pre_connect_if_required
55
+ print_signal(sample.call)
56
+ end
57
+ end
58
+
59
+ desc "simulate", "Replay a dash autoscale run --record file with deploy.yml's thresholds now: every simulated action, and a summary per role"
60
+ option :samples, required: true, desc: "The --record file (JSONL, one tick per line) to replay"
61
+ option :json, type: :boolean, default: false, desc: "Print the simulation as JSON"
62
+ def simulate
63
+ raise ArgumentError, "No such file: #{options[:samples]}" unless File.file?(options[:samples])
64
+
65
+ simulation = Dash::Autoscale::Simulation.new(path: options[:samples], config: DASH.config, roles: DASH.specific_roles&.map(&:name)).run
66
+ options[:json] ? puts(JSON.pretty_generate(simulation)) : print_simulation(simulation)
67
+ end
68
+
69
+ desc "history [ROLE]", "Show the controller's last decisions (of ROLE, or of every role)"
70
+ option :lines, type: :numeric, default: Dash::Diagnostics::AutoscaleDecisions::DEFAULT_LINES, desc: "How many decisions"
71
+ option :json, type: :boolean, default: false, desc: "Print the decisions as JSON"
72
+ def history(role_name = nil)
73
+ role = scaled_role(role_name).name if role_name
74
+ history = -> { Dash::Diagnostics::AutoscaleDecisions.new(roles: ([ role ] if role), lines: options[:lines]).to_h }
75
+
76
+ if options[:json]
77
+ puts_json(&history)
78
+ else
79
+ pre_connect_if_required
80
+ print_history(DASH.with_verbosity(:error) { history.call })
81
+ end
82
+ end
83
+
84
+ desc "status", "Show which controller runs, when it last ticked, and the paused roles"
85
+ option :json, type: :boolean, default: false, desc: "Print the status as JSON"
86
+ def status
87
+ status = -> { Dash::Diagnostics::ControllerStatus.new.to_h }
88
+
89
+ if options[:json]
90
+ puts_json(&status)
91
+ else
92
+ pre_connect_if_required
93
+ print_status(DASH.with_verbosity(:error) { status.call })
94
+ end
95
+ end
96
+
97
+ desc "pause ROLE", "Stop the controller from scaling ROLE, for good or --for a while (resume ROLE lifts it)"
98
+ option :for, desc: "How long: seconds, or a number followed by s, m, h or d (30m, 2h, 1d)"
99
+ def pause(role_name)
100
+ role = scaled_role(role_name)
101
+ now = Time.now.utc
102
+ ends_at = options[:for] ? now + pause_duration(options[:for]) : :indefinite
103
+ pause = Dash::Autoscale::Pause.new(ends_at: ends_at, by: performer, at: now)
104
+ until_text = pause.indefinite? ? "until resumed" : "until #{Dash::Autoscale::Timestamp.dump(ends_at)}"
105
+
106
+ on(Dash::Autoscale::StateStore.host) do
107
+ store = Dash::Autoscale::StateStore.new(self)
108
+ store.ensure_directory
109
+ store.write_pause(role.name, pause.to_h)
110
+ end
111
+ audit "Paused autoscaling of #{role} #{until_text}"
112
+
113
+ say "Paused autoscaling of #{role} #{until_text}", :magenta
114
+ end
115
+
116
+ desc "resume ROLE", "Let the controller scale ROLE again"
117
+ def resume(role_name)
118
+ role = scaled_role(role_name)
119
+
120
+ on(Dash::Autoscale::StateStore.host) do
121
+ Dash::Autoscale::StateStore.new(self).remove_pause(role.name)
122
+ end
123
+ audit "Resumed autoscaling of #{role}"
124
+
125
+ say "Resumed autoscaling of #{role}", :magenta
126
+ end
127
+
128
+ # What Dash::Autoscale::Controller asks of the CLI.
129
+ no_commands do
130
+ # A fresh deploy.yml every tick, as `dash mcp` reads one per question, with the run's
131
+ # --roles applied again (reconfigure drops them).
132
+ def reconfigure!
133
+ roles = options[:roles]&.split(",")
134
+ DASH.reconfigure(config_file: Pathname.new(File.expand_path(options[:config_file])), destination: options[:destination], explicit_roles: roles)
135
+ DASH.specific_roles = roles
136
+ end
137
+
138
+ # `dash scale set ROLE COUNT`, waiting up to autoscale/lock_wait_timeout for the deploy
139
+ # lock. The invocation is reset even when it fails, or Thor would skip the next one.
140
+ def invoke_scale_set(role, count)
141
+ with_lock_wait do
142
+ invoke "dash:cli:scale:set", [ role.name, count.to_s ], scale_options
143
+ ensure
144
+ reset_invocation(Dash::Cli::Scale)
145
+ end
146
+ end
147
+
148
+ def replace_member(role, member, count:)
149
+ with_lock_wait do
150
+ Dash::Cli::Scale.new([], scale_options, invocations: { Dash::Cli::Scale => [ "set" ] }).replace_unreachable(role, member, count: count)
151
+ end
152
+ end
153
+
154
+ def report(message, color = nil)
155
+ say message, color
156
+ end
157
+ end
158
+
159
+ private
160
+ # A signal during the tick lets it finish (a join in flight included), then it exits.
161
+ def run_once(controller)
162
+ trapping_signals(controller) do
163
+ controller.start
164
+ controller.tick
165
+ end
166
+ ensure
167
+ controller.release
168
+ end
169
+
170
+ def run_loop(controller)
171
+ server = Dash::Autoscale::MetricsServer.new(controller.metrics, bind: options[:metrics_bind], port: options[:metrics_port]).start if options[:metrics_port]
172
+ say "Serving autoscale metrics on http://#{options[:metrics_bind]}:#{server.port}/metrics", :magenta if server
173
+ say "Autoscale controller #{controller.id}#{" (dry run)" if options[:dry_run]} ticking every #{controller.interval}s", :magenta
174
+
175
+ trapping_signals(controller) { controller.run }
176
+ ensure
177
+ server&.stop
178
+ end
179
+
180
+ # TERM and INT only ask the controller to stop: the tick in progress, and a join in it, finish.
181
+ def trapping_signals(controller)
182
+ previous = %w[ TERM INT ].to_h { |signal| [ signal, trap(signal) { controller.stop! } ] }
183
+ yield
184
+ ensure
185
+ previous&.each { |signal, handler| trap(signal, handler || "DEFAULT") }
186
+ end
187
+
188
+ def tick_interval
189
+ return unless options[:interval]
190
+
191
+ interval = Integer(options[:interval].to_s, 10, exception: false)
192
+ if interval.nil? || interval < MINIMUM_INTERVAL
193
+ raise ArgumentError, "--interval must be a whole number of seconds, at least #{MINIMUM_INTERVAL}, not #{options[:interval]}"
194
+ end
195
+
196
+ interval
197
+ end
198
+
199
+ def with_lock_wait
200
+ saved = [ DASH.lock_wait, DASH.lock_wait_timeout, DASH.lock_wait_interval ]
201
+ DASH.lock_wait, DASH.lock_wait_timeout, DASH.lock_wait_interval = true, DASH.config.autoscale.lock_wait_timeout, LOCK_WAIT_INTERVAL
202
+ yield
203
+ ensure
204
+ DASH.lock_wait, DASH.lock_wait_timeout, DASH.lock_wait_interval = saved
205
+ end
206
+
207
+ def scale_options
208
+ options.slice("config_file", "destination", "skip_hooks", "verbose", "quiet")
209
+ end
210
+
211
+ def scaled_role(role_name)
212
+ role = DASH.config.role(role_name) || raise(ArgumentError, "No role named #{role_name}, expected one of #{DASH.config.roles.map(&:name).join(", ")}")
213
+ raise ArgumentError, "#{role} has no scale, so the autoscale controller never touches it" unless role.scaled?
214
+
215
+ role
216
+ end
217
+
218
+ def pause_duration(value)
219
+ seconds = Dash::Autoscale::Duration.parse(value)
220
+ raise ArgumentError, "--for must be longer than 0 seconds" unless seconds.positive?
221
+
222
+ seconds
223
+ end
224
+
225
+ def performer
226
+ Dash::Git.user_name.presence || ENV["USER"] || "unknown"
227
+ end
228
+
229
+ # On the primary host, where `dash audit` reads, wherever the state lives.
230
+ def audit(line)
231
+ on(DASH.config.primary_role.baseline_hosts.first) { execute *DASH.auditor.record(line), verbosity: :debug }
232
+ end
233
+
234
+ def print_explain(explain)
235
+ decision = explain[:decision]
236
+ say "#{decision[:role]}: #{decision[:action]} #{decision[:from].inspect} -> #{decision[:to].inspect} (#{decision[:reasons].join(", ")})"
237
+ say " not controlled: it has no scale/schedule or scale/signal, so dash autoscale run leaves it alone", :yellow unless explain[:controlled]
238
+ say " error: #{decision[:error]}", :red if decision[:error]
239
+ say " eligible at: #{decision[:eligible_at]}" if decision[:eligible_at]
240
+ decision[:inputs].each { |name, value| say " #{name}: #{value.to_json}" unless value.nil? || value == [] || value == {} }
241
+ say " state on #{explain[:state_host]} could not be read (#{explain[:state_error]}), evaluated without it", :yellow if explain[:state_error]
242
+ end
243
+
244
+ def print_signal(sample)
245
+ return say(sample[:message], :yellow) unless sample[:configured]
246
+
247
+ signal, band = sample[:signal], sample[:band]
248
+ if signal[:error] || signal[:value].nil?
249
+ say "#{sample[:role]}: #{signal[:source]} could not be read from #{sample[:from]}: #{signal[:error] || "no value"}", :red
250
+ else
251
+ position = sample[:position] == "inside" ? "inside the band" : "#{sample[:position]} the band"
252
+ say "#{sample[:role]}: #{signal[:source]} #{signal[:value]}, #{position} (up above #{band[:up_above]} for #{band[:up_for]}s, " \
253
+ "down below #{band[:down_below]} for #{band[:down_for]}s), read from #{sample[:from]} in #{signal[:latency_ms]}ms"
254
+ end
255
+
256
+ sample[:guards].each do |guard|
257
+ name = "hold_when/#{guard[:index]} #{guard[:source]}"
258
+ reading = guard[:value].nil? ? "#{name}: #{guard[:error] || "no value"}" : "#{name} #{guard[:value]}#{" > #{guard[:above]}" if guard[:holding]}"
259
+ say " #{reading}#{", holding a scale-out" if guard[:holding]} (#{guard[:latency_ms]}ms)", (:yellow if guard[:holding])
260
+ end
261
+ end
262
+
263
+ def print_simulation(simulation)
264
+ simulation[:roles].each do |role|
265
+ say "#{role[:role]}: #{role[:ticks]} #{"tick".pluralize(role[:ticks])}, #{role[:replayed]} replayed"
266
+ role[:actions].each { |action| say " #{action[:at]} #{action[:summary]}" }
267
+ say " simulated: #{simulation_summary(role[:simulated])}"
268
+ say " recorded: #{simulation_summary(role[:recorded])}"
269
+ if role[:not_replayable].positive?
270
+ say " #{role[:not_replayable]} #{"tick".pluralize(role[:not_replayable])} not replayable: recorded without a signal (before Phase 3), or not evaluated", :yellow
271
+ end
272
+ say " #{role[:unrecorded_guards]} hold_when #{"guard".pluralize(role[:unrecorded_guards])} not in the recording, left out", :yellow if role[:unrecorded_guards]
273
+ end
274
+
275
+ say "No role in #{simulation[:file]} to replay", :yellow if simulation[:roles].empty?
276
+ simulation[:skipped].each { |skipped| say "#{skipped[:role]}: skipped, #{skipped[:reason]}", :yellow }
277
+ simulation[:errors].each { |error| say "line #{error[:line]}: #{error[:error]}", :red }
278
+ end
279
+
280
+ def simulation_summary(summary)
281
+ return "no tick replayed" if summary[:peak].nil?
282
+
283
+ "#{summary[:scale_outs]} #{"scale-out".pluralize(summary[:scale_outs])}, #{summary[:scale_ins]} #{"scale-in".pluralize(summary[:scale_ins])}, " \
284
+ "peak #{summary[:peak]}, minimum #{summary[:minimum]}, #{summary[:container_seconds]} container-seconds"
285
+ end
286
+
287
+ def print_history(history)
288
+ return say("Could not read the decision log on #{history[:host]}: #{history[:error]}", :red) if history[:error]
289
+ return say("No autoscale decisions logged yet") if history[:decisions].empty?
290
+
291
+ history[:decisions].each do |decision|
292
+ counts = decision["action"] == "replace_member" ? decision.dig("inputs", "replace") : "#{decision["from"].inspect} -> #{decision["to"].inspect}"
293
+ error = " - #{decision["error"]}" if decision["error"]
294
+ say "#{decision["at"]} #{decision["role"]}: #{decision["action"]} #{counts} (#{decision["reasons"].to_a.join(", ")})#{error}", (:red if error)
295
+ end
296
+ end
297
+
298
+ def print_status(status)
299
+ return say("Could not read the controller state on #{status[:state_host]}: #{status[:error]}", :red) if status[:error]
300
+
301
+ if (controller = status[:controller])
302
+ tick = controller[:age_seconds] ? "last tick #{controller[:age_seconds]}s ago" : "no tick yet"
303
+ say "Controller #{controller[:controller_id]} on #{controller[:hostname]} (pid #{controller[:pid]}, dash #{controller[:version]}, #{controller[:mode]}): " \
304
+ "#{controller[:status]}, #{tick}", controller[:status] == "running" ? :green : :yellow
305
+ else
306
+ say "No autoscale controller has run for this app", :yellow
307
+ end
308
+
309
+ say "Scheduled roles: #{status[:roles].any? ? status[:roles].join(", ") : "none"}"
310
+ status[:pauses].each { |pause| say "#{pause[:role]} paused #{pause[:until] == "indefinite" ? "until resumed" : "until #{pause[:until]}"} by #{pause[:by]}", :yellow }
311
+ end
312
+ end
data/lib/dash/cli/base.rb CHANGED
@@ -103,9 +103,10 @@ module Dash::Cli
103
103
  # Process-scoped rather than commander-scoped: Dash::Cli::Alias::Command
104
104
  # resets DASH and re-enters Dash::Cli::Main.start, so an aliased command
105
105
  # builds a second commander and would otherwise warn twice. `dash migrate`
106
- # is exempt - telling an operator to run the command they are running is noise.
106
+ # is exempt - telling an operator to run the command they are running is noise - and
107
+ # so is `dash mcp`, whose stdout is the protocol.
107
108
  def warn_on_legacy_project_directory(command_name)
108
- return if command_name == "migrate"
109
+ return if %w[ migrate mcp ].include?(command_name)
109
110
  return if Dash::Cli::Base.legacy_project_directory_warned
110
111
  return unless Dash::ProjectDirectory.legacy?
111
112
 
@@ -129,7 +130,9 @@ module Dash::Cli
129
130
  commander.configure \
130
131
  config_file: Pathname.new(File.expand_path(options[:config_file])),
131
132
  destination: options[:destination],
132
- version: options[:version]
133
+ version: options[:version],
134
+ explicit_hosts: options[:hosts]&.split(","),
135
+ explicit_roles: options[:roles]&.split(",")
133
136
 
134
137
  commander.specific_hosts = options[:hosts]&.split(",")
135
138
  commander.specific_roles = options[:roles]&.split(",")
@@ -544,6 +547,15 @@ module Dash::Cli
544
547
  super
545
548
  end
546
549
 
550
+ # A diagnostic as pure JSON on stdout, returned too. capture_with_info logs every command
551
+ # it runs at info, onto stdout, so the capture runs at :error - a `| jq` must see only the JSON.
552
+ def puts_json(ssh: true)
553
+ DASH.with_verbosity(:error) do
554
+ pre_connect_if_required if ssh
555
+ yield.tap { |snapshot| puts JSON.pretty_generate(snapshot) }
556
+ end
557
+ end
558
+
547
559
  def pre_connect_if_required
548
560
  if !DASH.connected?
549
561
  run_hook "pre-connect", secrets: true unless options[:skip_hooks]
data/lib/dash/cli/lock.rb CHANGED
@@ -1,8 +1,13 @@
1
1
  class Dash::Cli::Lock < Dash::Cli::Base
2
2
  desc "status", "Report lock status"
3
3
  option :server, type: :boolean, default: false, desc: "Report the shared server lock instead of the deploy lock"
4
+ option :json, type: :boolean, default: false, desc: "Print the deploy lock status as JSON"
4
5
  def status
5
- if options[:server]
6
+ if options[:json]
7
+ raise ArgumentError, "--json reports the deploy lock; it cannot be combined with --server" if options[:server]
8
+
9
+ puts_json { Dash::Diagnostics::Lock.new.to_h }
10
+ elsif options[:server]
6
11
  report_server_lock_status
7
12
  else
8
13
  handle_missing_lock do
data/lib/dash/cli/main.rb CHANGED
@@ -151,7 +151,10 @@ class Dash::Cli::Main < Dash::Cli::Base
151
151
  end
152
152
 
153
153
  desc "audit", "Show audit log from servers"
154
+ option :json, type: :boolean, default: false, desc: "Print the audit log as JSON, parsed per line"
154
155
  def audit
156
+ return puts_json { Dash::Diagnostics::Audit.new.to_h } if options[:json]
157
+
155
158
  quiet = options[:quiet]
156
159
  on(DASH.hosts) do |host|
157
160
  puts_by_host host, capture_with_info(*DASH.auditor.reveal), quiet: quiet
@@ -159,7 +162,10 @@ class Dash::Cli::Main < Dash::Cli::Base
159
162
  end
160
163
 
161
164
  desc "config", "Show combined config (including secrets!)"
165
+ option :json, type: :boolean, default: false, desc: "Print the config and its topology (roles, hosts, proxies) as JSON"
162
166
  def config
167
+ return puts_json(ssh: false) { Dash::Diagnostics::Config.new.to_h } if options[:json]
168
+
163
169
  run_locally do
164
170
  puts Dash::Utils.redacted(DASH.config.to_h).to_yaml
165
171
  end
@@ -177,12 +183,15 @@ class Dash::Cli::Main < Dash::Cli::Base
177
183
  puts "No documentation found for #{section}"
178
184
  end
179
185
 
180
- desc "doctor", "Diagnose deploy readiness of servers, registry, proxy, ports, DNS, certificates, and per-role readiness gates"
186
+ desc "doctor", "Diagnose deploy readiness of servers, registry, proxy, ports, DNS, certificates, per-role readiness gates, and proxy drift"
187
+ option :json, type: :boolean, default: false, desc: "Print the results as JSON (exits 1 when a check fails)"
181
188
  def doctor
189
+ return doctor_json if options[:json]
190
+
182
191
  say "Running readiness checks...", :magenta
183
- pre_connect_if_required
192
+ doctor_pre_connect
184
193
 
185
- doctor = Dash::Cli::Doctor.new
194
+ doctor = Dash::Diagnostics::Doctor.new
186
195
  doctor.run
187
196
 
188
197
  print_doctor_report doctor.results
@@ -194,6 +203,18 @@ class Dash::Cli::Main < Dash::Cli::Base
194
203
  end
195
204
  end
196
205
 
206
+ desc "mcp", "Serve read-only diagnostics to an AI agent over stdio (Model Context Protocol)"
207
+ option :allow_logs, type: :boolean, default: false, desc: "Enable the logs tool (container logs can carry personal data; or DASH_MCP_ALLOW_LOGS=true)"
208
+ def mcp
209
+ protocol = Dash::Mcp.reserve_stdout!
210
+ Dash::Mcp.load!
211
+ Dash::Mcp::Runner.authorize!(ENV)
212
+ DASH.verbosity = :error
213
+ pre_connect_if_required
214
+
215
+ Dash::Mcp::Runner.run(mcp_session, output: protocol)
216
+ end
217
+
197
218
  desc "init", "Create config stub in config/deploy.yml and secrets stub in .dash"
198
219
  option :bundle, type: :boolean, default: false, desc: "Add dash to the Gemfile and create a bin/dash binstub"
199
220
  def init
@@ -301,6 +322,9 @@ class Dash::Cli::Main < Dash::Cli::Base
301
322
  desc "app", "Manage application"
302
323
  subcommand "app", Dash::Cli::App
303
324
 
325
+ desc "autoscale", "Run the autoscale controller, and read what it decided"
326
+ subcommand "autoscale", Dash::Cli::Autoscale
327
+
304
328
  desc "build", "Build application image"
305
329
  subcommand "build", Dash::Cli::Build
306
330
 
@@ -319,6 +343,9 @@ class Dash::Cli::Main < Dash::Cli::Base
319
343
  desc "registry", "Login and -out of the image registry"
320
344
  subcommand "registry", Dash::Cli::Registry
321
345
 
346
+ desc "scale", "Change how many containers of a role run, without a deploy"
347
+ subcommand "scale", Dash::Cli::Scale
348
+
322
349
  desc "secrets", "Helpers for extracting secrets"
323
350
  subcommand "secrets", Dash::Cli::Secrets
324
351
 
@@ -352,10 +379,34 @@ class Dash::Cli::Main < Dash::Cli::Base
352
379
  { "version" => DASH.config.version }.merge(base_options)
353
380
  end
354
381
 
382
+ # Pure JSON on stdout, so the failure is the exit status rather than an ERROR line.
383
+ # The pre-connect hook is handed every host, and with the pool unreadable those are
384
+ # unknown. The doctor carries on without it and reports the pool failure itself.
385
+ # `--json` keeps stdout for the JSON, so it says so on stderr.
386
+ def doctor_pre_connect(json: false)
387
+ pre_connect_if_required
388
+ rescue Dash::Autoscale::ProviderError => e
389
+ message = "Skipped the pre-connect hook, the hosts are unknown: #{e.message}"
390
+ json ? warn(message) : say(message, :yellow)
391
+ end
392
+
393
+ def doctor_json
394
+ DASH.with_verbosity(:error) { doctor_pre_connect(json: true) }
395
+ exit 1 unless puts_json(ssh: false) { Dash::Diagnostics::Doctor.new.to_h }[:successful]
396
+ end
397
+
398
+ # The scope the operator started `dash mcp` with is the ceiling for every question.
399
+ def mcp_session
400
+ Dash::Mcp::Session.new(
401
+ config_file: Pathname.new(File.expand_path(options[:config_file])), destination: options[:destination], version: options[:version],
402
+ hosts: options[:primary] ? [ DASH.primary_host ] : options[:hosts]&.split(","), roles: options[:roles]&.split(","),
403
+ allow_logs: Dash::Mcp::Runner.allow_logs?(options[:allow_logs], ENV), redactor: Dash::Diagnostics::Redactor.for(DASH.config))
404
+ end
405
+
355
406
  def print_doctor_report(results)
356
407
  results.group_by(&:title).each do |title, rows|
357
408
  say title
358
- rows.each { |row| say " #{row}", Dash::Cli::Doctor::STATUS_COLORS[row.status] }
409
+ rows.each { |row| say " #{row}", Dash::Diagnostics::Doctor::STATUS_COLORS[row.status] }
359
410
  end
360
411
  end
361
412
 
@@ -377,14 +428,32 @@ class Dash::Cli::Main < Dash::Cli::Base
377
428
 
378
429
  say "Deploying #{config.service}#{" to #{config.destination}" if config.destination} (version #{config.abbreviated_version})", :magenta
379
430
  config.roles.each do |role|
380
- hosts = "#{role.hosts.count} #{"host".pluralize(role.hosts.count)} (#{role.hosts.join(", ")})"
381
- say " #{role.name}: #{hosts} — readiness: #{role.readiness_description}", (:yellow if role.readiness_source == :none)
431
+ replicas = " #{role.replicas}" if role.replicas.scalable?
432
+ say " #{role.name}: #{banner_hosts(role)}#{replicas} — readiness: #{role.readiness_description}", (:yellow if role.readiness_source == :none)
382
433
  end
383
434
  say " proxy: #{config.proxy_hosts.join(", ")}" if config.proxy_hosts.any?
384
435
  if config.proxy.load_balancing?
385
- reason = " (auto-enabled: primary role #{config.primary_role.name} has #{config.primary_role.hosts.count} hosts)" unless config.proxy.loadbalancer.present?
386
- say " loadbalancer: #{config.proxy.effective_loadbalancer}#{reason}"
436
+ say " loadbalancer: #{config.proxy.effective_loadbalancer}#{loadbalancer_reason(config) unless config.proxy.loadbalancer.present?}"
387
437
  end
388
438
  say " timeouts: deploy #{config.deploy_timeout}s, drain #{config.drain_timeout}s, readiness delay #{config.readiness_delay}s"
389
439
  end
440
+
441
+ def banner_hosts(role)
442
+ baseline = role.baseline_hosts
443
+ hosts = "#{baseline.count} #{"host".pluralize(baseline.count)} (#{baseline.join(", ")})"
444
+ return hosts unless role.scaled?
445
+
446
+ active = role.active_members.map(&:host)
447
+ "#{hosts} + #{active.count} of #{role.members.count} members active#{" (#{active.join(", ")})" if active.any?}"
448
+ end
449
+
450
+ def loadbalancer_reason(config)
451
+ primary = config.primary_role
452
+
453
+ if primary.running_proxy? && primary.baseline_hosts.count > 1
454
+ " (auto-enabled: primary role #{primary.name} has #{primary.baseline_hosts.count} hosts)"
455
+ elsif (scaled = config.scaled_roles.find(&:running_proxy?))
456
+ " (auto-enabled: role #{scaled.name} scales across hosts)"
457
+ end
458
+ end
390
459
  end
@@ -0,0 +1,52 @@
1
+ # `dash proxy services` and `dash proxy drift`: the read-only questions about what the
2
+ # proxies route, kept out of the proxy command file that boots, reboots and manages them.
3
+ # The JSON is the diagnostic's own `to_h`; the rest here is the table an operator reads.
4
+ module Dash::Cli::Proxy::Inspection
5
+ extend ActiveSupport::Concern
6
+
7
+ included do
8
+ desc "services", "Show what dash-proxy routes for this deploy on each proxy host (and the load balancer)"
9
+ option :json, type: :boolean, default: false, desc: "Print the services as JSON"
10
+ def services
11
+ return puts_json { Dash::Diagnostics::ProxyServices.new.to_h } if options[:json]
12
+
13
+ pre_connect_if_required
14
+ puts services_lines(Dash::Diagnostics::ProxyServices.new.to_h)
15
+ end
16
+
17
+ desc "drift", "Compare the running containers with the proxy and load balancer targets"
18
+ option :json, type: :boolean, default: false, desc: "Print the drift as JSON"
19
+ def drift
20
+ return puts_json { Dash::Diagnostics::Drift.take.to_h } if options[:json]
21
+
22
+ pre_connect_if_required
23
+ puts drift_lines(Dash::Diagnostics::Drift.take.to_h)
24
+ end
25
+ end
26
+
27
+ private
28
+ def services_lines(snapshot)
29
+ proxies = snapshot[:hosts].flat_map { |host| host_services_lines("Proxy", host) }
30
+ proxies + (snapshot[:loadbalancer] ? host_services_lines("Loadbalancer", snapshot[:loadbalancer]) : [])
31
+ end
32
+
33
+ def drift_lines(snapshot)
34
+ if snapshot[:consistent]
35
+ [ "No drift: proxy targets match the running containers" ]
36
+ else
37
+ snapshot[:drift].map { |entry| "#{entry[:code]} #{entry[:host] || entry[:role]}: #{entry[:detail]}" } +
38
+ Array(snapshot[:unread]).map { |host| "unread #{host[:host]}: could not read #{host[:source]} (#{host[:error]})" }
39
+ end
40
+ end
41
+
42
+ def host_services_lines(type, host)
43
+ return [ "#{type} Host: #{host[:host]}", " ERROR #{host[:error]}" ] if host[:error]
44
+
45
+ services = host[:services].map do |name, service|
46
+ targets = Array(service["targets"]).presence || service["target"].to_s.split(",")
47
+ " #{name} #{service["host"].presence || "*"} -> #{targets.join(", ").presence || "no targets"}#{" (#{service["state"]})" if service["state"]}"
48
+ end
49
+
50
+ [ "#{type} Host: #{host[:host]}", *services.presence || [ " no services" ] ]
51
+ end
52
+ end
@@ -0,0 +1,20 @@
1
+ # Points the load balancer at a target list: every host of every proxied role by default.
2
+ # `dash proxy loadbalancer deploy` and `dash scale set` share it, so a member leaving the
3
+ # pool is the same deploy with that host left out - dash-proxy replaces the whole list,
4
+ # draining the hosts that are no longer on it. Runs inside `on(<load balancer host>)`.
5
+ class Dash::Cli::Proxy::LoadbalancerDeploy
6
+ attr_reader :host, :sshkit
7
+ delegate :execute, :info, to: :sshkit
8
+
9
+ def initialize(host, sshkit, targets: DASH.loadbalancer_config.target_hosts)
10
+ @host = host
11
+ @sshkit = sshkit
12
+ @targets = targets
13
+ end
14
+
15
+ def run
16
+ Dash::Cli::Proxy::LoadbalancerClaim.new(host, sshkit).claim_service
17
+ info "Deploying to loadbalancer on #{host} with targets: #{@targets.join(', ')}"
18
+ execute *DASH.loadbalancer.deploy(targets: @targets)
19
+ end
20
+ end
@@ -96,7 +96,7 @@ class Dash::Cli::Proxy::LoadbalancerReboot
96
96
  # `list --json` returns {"services": {"<name>": ...}} - exact key
97
97
  # membership, same as the per-host proxy reboot's verification.
98
98
  def verify_service
99
- listed = JSON.parse(capture_with_info(*DASH.loadbalancer.list(json: true))).fetch("services", {}).keys
99
+ listed = Dash::Commands::Proxy::Services.parse(capture_with_info(*DASH.loadbalancer.list(json: true))).keys
100
100
 
101
101
  unless listed.include?(DASH.config.service)
102
102
  raise Dash::Cli::BootError, "the load balancer on #{host} is missing service #{DASH.config.service} after reboot"
@@ -143,16 +143,11 @@ class Dash::Cli::Proxy::Reboot
143
143
  registered_services = []
144
144
 
145
145
  DASH.roles_on(host).select(&:running_proxy?).each do |role|
146
- app = DASH.app(role: role, host: host)
147
-
148
- version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip.presence
149
- next unless version
150
-
151
- endpoint = capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).strip.presence
152
- next unless endpoint
146
+ endpoints = Dash::Cli::App::RunningTargets.new(sshkit, role: role, host: host).container_ids(only_running: true)
147
+ next if endpoints.empty?
153
148
 
154
149
  info "Re-registering #{role.container_prefix} with dash-proxy on #{host}..."
155
- execute *app.deploy(target: endpoint)
150
+ execute *DASH.app(role: role, host: host).deploy(targets: endpoints)
156
151
  registered_services << role.container_prefix
157
152
  end
158
153
 
@@ -177,7 +172,7 @@ class Dash::Cli::Proxy::Reboot
177
172
  def verify_services(registered_services)
178
173
  return if registered_services.empty?
179
174
 
180
- listed = JSON.parse(capture_with_info(*proxy.list(json: true))).fetch("services", {}).keys
175
+ listed = Dash::Commands::Proxy::Services.parse(capture_with_info(*proxy.list(json: true))).keys
181
176
  missing = registered_services - listed
182
177
 
183
178
  if missing.any?