dash 4.1.3 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. checksums.yaml +4 -4
  2. data/lib/dash/autoscale/connections/builder.rb +51 -0
  3. data/lib/dash/autoscale/connections.rb +102 -0
  4. data/lib/dash/autoscale/controller.rb +296 -0
  5. data/lib/dash/autoscale/cron.rb +75 -0
  6. data/lib/dash/autoscale/decision.rb +89 -0
  7. data/lib/dash/autoscale/duration.rb +19 -0
  8. data/lib/dash/autoscale/evaluation.rb +52 -0
  9. data/lib/dash/autoscale/guard.rb +48 -0
  10. data/lib/dash/autoscale/lease.rb +55 -0
  11. data/lib/dash/autoscale/lease_held.rb +2 -0
  12. data/lib/dash/autoscale/lease_lost.rb +2 -0
  13. data/lib/dash/autoscale/member.rb +28 -0
  14. data/lib/dash/autoscale/metrics.rb +70 -0
  15. data/lib/dash/autoscale/metrics_server.rb +71 -0
  16. data/lib/dash/autoscale/observation.rb +41 -0
  17. data/lib/dash/autoscale/pause.rb +34 -0
  18. data/lib/dash/autoscale/policy.rb +198 -0
  19. data/lib/dash/autoscale/pool.rb +103 -0
  20. data/lib/dash/autoscale/prometheus_client.rb +76 -0
  21. data/lib/dash/autoscale/provider/base.rb +43 -0
  22. data/lib/dash/autoscale/provider/exec.rb +94 -0
  23. data/lib/dash/autoscale/provider/upcloud.rb +140 -0
  24. data/lib/dash/autoscale/provider.rb +9 -0
  25. data/lib/dash/autoscale/provider_error.rb +4 -0
  26. data/lib/dash/autoscale/read_only_query.rb +38 -0
  27. data/lib/dash/autoscale/reader.rb +17 -0
  28. data/lib/dash/autoscale/recorder.rb +13 -0
  29. data/lib/dash/autoscale/role_state.rb +110 -0
  30. data/lib/dash/autoscale/sample.rb +13 -0
  31. data/lib/dash/autoscale/signal/pgbus.rb +19 -0
  32. data/lib/dash/autoscale/signal/prometheus.rb +8 -0
  33. data/lib/dash/autoscale/signal/sidekiq.rb +49 -0
  34. data/lib/dash/autoscale/signal.rb +31 -0
  35. data/lib/dash/autoscale/simulation.rb +200 -0
  36. data/lib/dash/autoscale/state_store.rb +132 -0
  37. data/lib/dash/autoscale/timestamp.rb +15 -0
  38. data/lib/dash/autoscale/tunnel.rb +48 -0
  39. data/lib/dash/cli/accessory.rb +16 -0
  40. data/lib/dash/cli/app/boot.rb +69 -44
  41. data/lib/dash/cli/app/boot_state.rb +70 -0
  42. data/lib/dash/cli/app/rollout_boot.rb +28 -9
  43. data/lib/dash/cli/app/running_targets.rb +39 -0
  44. data/lib/dash/cli/app/slot_version.rb +29 -0
  45. data/lib/dash/cli/app.rb +89 -32
  46. data/lib/dash/cli/autoscale.rb +312 -0
  47. data/lib/dash/cli/base.rb +15 -3
  48. data/lib/dash/cli/lock.rb +6 -1
  49. data/lib/dash/cli/main.rb +77 -8
  50. data/lib/dash/cli/proxy/inspection.rb +52 -0
  51. data/lib/dash/cli/proxy/loadbalancer_deploy.rb +20 -0
  52. data/lib/dash/cli/proxy/loadbalancer_reboot.rb +1 -1
  53. data/lib/dash/cli/proxy/reboot.rb +4 -9
  54. data/lib/dash/cli/proxy.rb +12 -7
  55. data/lib/dash/cli/prune.rb +5 -3
  56. data/lib/dash/cli/report.rb +4 -1
  57. data/lib/dash/cli/scale/host_plan.rb +63 -0
  58. data/lib/dash/cli/scale/join.rb +128 -0
  59. data/lib/dash/cli/scale/leave.rb +58 -0
  60. data/lib/dash/cli/scale/placement.rb +54 -0
  61. data/lib/dash/cli/scale/power_off.rb +26 -0
  62. data/lib/dash/cli/scale/replica_join.rb +74 -0
  63. data/lib/dash/cli/scale/replica_leave.rb +121 -0
  64. data/lib/dash/cli/scale.rb +276 -0
  65. data/lib/dash/cli/server.rb +41 -0
  66. data/lib/dash/cli/stats_table.rb +34 -0
  67. data/lib/dash/cli/templates/sample_hooks/post-scale-in.sample +3 -0
  68. data/lib/dash/cli/templates/sample_hooks/post-scale-out.sample +3 -0
  69. data/lib/dash/cli/templates/sample_hooks/pre-scale-in.sample +6 -0
  70. data/lib/dash/cli/templates/sample_hooks/pre-scale-out.sample +6 -0
  71. data/lib/dash/commander.rb +17 -3
  72. data/lib/dash/commands/accessory/proxy.rb +1 -1
  73. data/lib/dash/commands/accessory.rb +21 -2
  74. data/lib/dash/commands/app/containers.rb +28 -3
  75. data/lib/dash/commands/app/logging.rb +13 -2
  76. data/lib/dash/commands/app/proxy.rb +4 -4
  77. data/lib/dash/commands/app.rb +90 -10
  78. data/lib/dash/commands/auditor.rb +2 -2
  79. data/lib/dash/commands/autoscale.rb +133 -0
  80. data/lib/dash/commands/base.rb +4 -0
  81. data/lib/dash/commands/proxy/services.rb +10 -0
  82. data/lib/dash/commands/prune.rb +10 -2
  83. data/lib/dash/commands/server.rb +16 -0
  84. data/lib/dash/configuration/accessory.rb +2 -2
  85. data/lib/dash/configuration/autoscale.rb +123 -0
  86. data/lib/dash/configuration/docs/autoscale.yml +141 -0
  87. data/lib/dash/configuration/docs/configuration.yml +8 -2
  88. data/lib/dash/configuration/docs/proxy.yml +1 -1
  89. data/lib/dash/configuration/docs/role.yml +203 -0
  90. data/lib/dash/configuration/loadbalancer.rb +3 -2
  91. data/lib/dash/configuration/proxy/run.rb +1 -1
  92. data/lib/dash/configuration/proxy.rb +19 -7
  93. data/lib/dash/configuration/role/drain.rb +49 -0
  94. data/lib/dash/configuration/role/replicas.rb +72 -0
  95. data/lib/dash/configuration/role/scale/guard.rb +40 -0
  96. data/lib/dash/configuration/role/scale/signal.rb +93 -0
  97. data/lib/dash/configuration/role/scale/window.rb +37 -0
  98. data/lib/dash/configuration/role/scale.rb +199 -0
  99. data/lib/dash/configuration/role.rb +123 -6
  100. data/lib/dash/configuration/validator/autoscale.rb +126 -0
  101. data/lib/dash/configuration/validator/proxy.rb +13 -0
  102. data/lib/dash/configuration/validator/role.rb +26 -5
  103. data/lib/dash/configuration.rb +126 -15
  104. data/lib/dash/diagnostics/audit.rb +28 -0
  105. data/lib/dash/diagnostics/autoscale_decisions.rb +18 -0
  106. data/lib/dash/diagnostics/autoscale_explain.rb +30 -0
  107. data/lib/dash/diagnostics/autoscale_signal.rb +82 -0
  108. data/lib/dash/diagnostics/base.rb +53 -0
  109. data/lib/dash/diagnostics/config.rb +32 -0
  110. data/lib/dash/diagnostics/container_stats.rb +67 -0
  111. data/lib/dash/diagnostics/containers.rb +20 -0
  112. data/lib/dash/diagnostics/controller_status.rb +44 -0
  113. data/lib/dash/diagnostics/docker_ps.rb +40 -0
  114. data/lib/dash/{cli → diagnostics}/doctor/config_checks.rb +3 -3
  115. data/lib/dash/diagnostics/doctor/controller_checks.rb +46 -0
  116. data/lib/dash/{cli → diagnostics}/doctor/endpoint_checks.rb +12 -4
  117. data/lib/dash/{cli → diagnostics}/doctor/host_checks.rb +5 -4
  118. data/lib/dash/diagnostics/doctor/pool_checks.rb +65 -0
  119. data/lib/dash/diagnostics/doctor/signal_checks.rb +54 -0
  120. data/lib/dash/diagnostics/doctor.rb +182 -0
  121. data/lib/dash/diagnostics/drift.rb +168 -0
  122. data/lib/dash/diagnostics/host_stats.rb +59 -0
  123. data/lib/dash/diagnostics/lines.rb +14 -0
  124. data/lib/dash/diagnostics/lock.rb +27 -0
  125. data/lib/dash/diagnostics/logs.rb +76 -0
  126. data/lib/dash/diagnostics/pool.rb +41 -0
  127. data/lib/dash/diagnostics/proxy_services.rb +36 -0
  128. data/lib/dash/diagnostics/redactor.rb +48 -0
  129. data/lib/dash/diagnostics/reports.rb +17 -0
  130. data/lib/dash/diagnostics/scale.rb +51 -0
  131. data/lib/dash/diagnostics/units.rb +31 -0
  132. data/lib/dash/mcp/base_tool.rb +43 -0
  133. data/lib/dash/mcp/runner.rb +28 -0
  134. data/lib/dash/mcp/server.rb +40 -0
  135. data/lib/dash/mcp/session.rb +51 -0
  136. data/lib/dash/mcp/tools/audit_tool.rb +11 -0
  137. data/lib/dash/mcp/tools/autoscale_decisions_tool.rb +22 -0
  138. data/lib/dash/mcp/tools/autoscale_explain_tool.rb +56 -0
  139. data/lib/dash/mcp/tools/config_tool.rb +10 -0
  140. data/lib/dash/mcp/tools/container_stats_tool.rb +15 -0
  141. data/lib/dash/mcp/tools/containers_tool.rb +10 -0
  142. data/lib/dash/mcp/tools/controller_status_tool.rb +18 -0
  143. data/lib/dash/mcp/tools/deploy_reports_tool.rb +10 -0
  144. data/lib/dash/mcp/tools/doctor_tool.rb +13 -0
  145. data/lib/dash/mcp/tools/drift_tool.rb +16 -0
  146. data/lib/dash/mcp/tools/host_stats_tool.rb +14 -0
  147. data/lib/dash/mcp/tools/lock_status_tool.rb +15 -0
  148. data/lib/dash/mcp/tools/logs_tool.rb +41 -0
  149. data/lib/dash/mcp/tools/pool_members_tool.rb +15 -0
  150. data/lib/dash/mcp/tools/proxy_services_tool.rb +10 -0
  151. data/lib/dash/mcp/tools/scale_status_tool.rb +10 -0
  152. data/lib/dash/mcp/tools/signal_sample_tool.rb +19 -0
  153. data/lib/dash/mcp/transport.rb +14 -0
  154. data/lib/dash/mcp.rb +56 -0
  155. data/lib/dash/sshkit_with_ext.rb +4 -4
  156. data/lib/dash/utils.rb +7 -0
  157. data/lib/dash/version.rb +1 -1
  158. data/lib/dash.rb +2 -0
  159. metadata +119 -5
  160. data/lib/dash/cli/doctor.rb +0 -113
@@ -136,8 +136,8 @@ allow_empty_roles: false
136
136
 
137
137
  # Retain containers
138
138
  #
139
- # How many old containers we retain per role, and how many images we retain,
140
- # defaults to 5:
139
+ # How many old containers we retain per role (per replica slot, for a role with
140
+ # `replicas`), and how many images we retain, defaults to 5:
141
141
  retain_containers: 3
142
142
 
143
143
  # Minimum version
@@ -205,6 +205,12 @@ accessories:
205
205
  proxy:
206
206
  ...
207
207
 
208
+ # Autoscale
209
+ #
210
+ # The provider that holds the pool members of scaled roles, see dash docs autoscale:
211
+ autoscale:
212
+ ...
213
+
208
214
  # SSHKit
209
215
  #
210
216
  # See dash docs sshkit:
@@ -837,7 +837,7 @@ proxy:
837
837
  # below already embeds its ghcr.io host
838
838
  repository: ghcr.io/zoolutions/dash-proxy # Container repository for the
839
839
  # dash-proxy image (this is the default)
840
- version: v1.1.0.3 # Version tag of the dash-proxy image to use.
840
+ version: v1.1.0.4 # Version tag of the dash-proxy image to use.
841
841
  # Defaults to the minimum version this gem
842
842
  # requires - only pin it to roll forward early,
843
843
  # never below the default
@@ -66,6 +66,209 @@ servers:
66
66
  limit: 25%
67
67
  wait: 10
68
68
 
69
+ # Replicas
70
+ #
71
+ # How many containers of this role run on EACH of its hosts. A process that cannot
72
+ # use more than one core by itself — open-source Sidekiq is one Ruby process — gets
73
+ # the rest of a bigger host this way, without declaring the same worker twice.
74
+ #
75
+ # `replicas: 2` is shorthand for `{ min: 2, max: 2 }`; the default is one, which is
76
+ # the behaviour without the key. A deploy boots as many replicas as are running now,
77
+ # kept within `min` and `max`, so a count changed at runtime with
78
+ # `dash scale set <role> <count>` survives the next deploy. With nothing running it
79
+ # boots `min`. `max` is the ceiling for `dash scale set`.
80
+ #
81
+ # Replicas of a proxied role are all targets of the role's one dash-proxy service,
82
+ # which balances across them. Replicas of any other role are independent containers.
83
+ # Replica 1 keeps the container name the role always had; replica 2 and up put the
84
+ # slot into the role segment (`app-workers.2-<version>`), and every replica of a
85
+ # scalable role gets the `replica` label and the `DASH_REPLICA` env var.
86
+ #
87
+ # Every replica opens its own database pool, so `pool × replicas × hosts` has to fit
88
+ # the database's (or the pooler's) connection limit.
89
+ #
90
+ # More than one replica cannot set the `publish`, `p`, `name` or `hostname` docker
91
+ # options, which every replica on a host would claim, and cannot yet be combined with
92
+ # `proxy/sleep`. Host volumes are shared by every replica on a host, which is fine for
93
+ # read-only config or a shared cache and wrong for a per-process store, so they warn.
94
+ replicas:
95
+ min: 1
96
+ max: 4
97
+
98
+ # Drain
99
+ #
100
+ # How a replica of a role WITHOUT the proxy leaves when `dash scale set` removes it:
101
+ # `signal` is sent first (e.g. TSTP, so Sidekiq stops taking new jobs), then dash
102
+ # waits `wait` seconds before `docker stop`. A role with a `sidekiq` scale signal stops
103
+ # waiting as soon as Sidekiq reports the leaving containers' processes quiet with
104
+ # nothing busy (it reports both every few seconds), so `wait` is then the most it waits. Keep `wait` inside the stop budget described on the worker
105
+ # roles page. A role behind dash-proxy has no drain: the proxy drains it, so setting one
106
+ # there is an error.
107
+ drain:
108
+ signal: TSTP
109
+ wait: 20
110
+
111
+ # Scale
112
+ #
113
+ # Extra hosts a role can run on beyond the ones listed under `hosts`, which stay the
114
+ # baseline: always on, deployed as before, never stopped by dash. The extra hosts are
115
+ # pool members that live in the cloud provider named under the root `autoscale:` key
116
+ # (see dash docs autoscale), not in this file. dash finds them by their labels —
117
+ # `dash.service=<service>`, `dash.destination=<destination, or - without one>` and
118
+ # `dash.role=<role>` — and every member that is `started` is a host of the role for
119
+ # every command; stopped members are never touched. A provider that cannot be reached
120
+ # fails the command. `--hosts` then names the member hosts to act on, with a warning —
121
+ # for one scaled role at a time (pick it with `--roles` when there are more).
122
+ #
123
+ # `min` and `max` count HOSTS, the baseline included, so `min` is at least the number of
124
+ # `hosts`; `replicas` counts containers per host. `dash scale set <role> <count>` counts
125
+ # containers across all of them.
126
+ #
127
+ # A scaled role behind dash-proxy needs the load balancer, which turns on by itself for
128
+ # it even with one baseline host; `proxy/loadbalancer: false` is an error for it.
129
+ #
130
+ # `schedule`, `signal`, `warmup`, `cooldown` and `step` drive the autoscaling controller
131
+ # (`dash autoscale run`), which then owns the role's count: a manual `dash scale set`
132
+ # lasts until its next tick unless the role is paused (`dash autoscale pause ROLE`). A
133
+ # scaled role with neither a `schedule` nor a `signal` stays manual.
134
+ scale:
135
+ min: 2
136
+ max: 4
137
+
138
+ # How extra hosts come to exist. `power` (the default) powers stopped members on and
139
+ # off again; they are created beforehand, with Terraform for example, carrying the
140
+ # three labels. `create` creates a member from `template` when one is needed and
141
+ # destroys it when it leaves. On UpCloud a stopped server's compute is free only on
142
+ # Cloud Native plans; any other plan bills a powered-off member as if it ran.
143
+ members: power
144
+
145
+ # What a `create` member is built from (UpCloud): the template storage to clone, the
146
+ # plan and zone, the user dash connects as with its SSH keys, the private network
147
+ # (SDN) to attach — required when `address` is private — and cloud-init user data,
148
+ # for a Tailscale join for example. Only `create` uses it, and `create` requires it.
149
+ template:
150
+ storage: 01234567-89ab-cdef-0123-456789abcdef
151
+ plan: CLOUDNATIVE-8xCPU-16GB
152
+ zone: de-fra1
153
+ login_user: deploy
154
+ ssh_keys:
155
+ - "ssh-ed25519 AAAA... deploy"
156
+ network: 03abcdef-0123-4567-89ab-cdef01234567
157
+ user_data: |
158
+ #!/bin/sh
159
+ tailscale up --authkey ...
160
+
161
+ # Which of a member's addresses dash connects to: private (the default), public or
162
+ # utility.
163
+ address: private
164
+
165
+ # Seconds from powering a member on (or creating it) until it answers SSH, default 300.
166
+ # The controller also replaces a started member that has not answered SSH for this
167
+ # long: out of the load balancer, powered off (deleted for `create`), and the next
168
+ # tick restores the count with a healthy one.
169
+ boot_timeout: 300
170
+
171
+ # Windows of guaranteed capacity, in containers (the unit of `dash scale set`). From
172
+ # every minute `cron` matches, for `for` (seconds, or a number with s, m, h or d, at
173
+ # most 7d), the controller keeps at least `min` containers running; overlapping
174
+ # windows take the largest `min`. Outside every window it returns to scale min ×
175
+ # replicas min. The target never goes above scale max × replicas max.
176
+ #
177
+ # `cron` has five fields - minute, hour, day-of-month, month, day-of-week - each `*`,
178
+ # a number, a range `a-b`, a list `a,b` or a step `*/n` or `a-b/n`; day-of-week 0 and
179
+ # 7 are Sunday. When both day fields are set, a day matches if either does. There
180
+ # are no names, `L`, `W` or `#`. It is evaluated in `autoscale/timezone`.
181
+ #
182
+ # "The evening before the 1st and the 15th" has no exact cron: this example starts on
183
+ # the 14th and on each of the 28th to 31st (so on several evenings at a month end).
184
+ # When a few hours of lead are not needed, start on the day itself: `0 0 1,15 * *`.
185
+ schedule:
186
+ - cron: "0 22 14,28-31 * *"
187
+ for: 30h
188
+ min: 10
189
+
190
+ # Seconds a member that just joined counts as warming up: the controller never scales
191
+ # in while one is, nor replaces it for not answering SSH. Default: `boot_timeout`.
192
+ warmup: 300
193
+
194
+ # Seconds between the controller's actions. A scale-in step waits `down` (default
195
+ # 600) after the last scale-out or scale-in. A signal-driven scale-out waits `up`
196
+ # (default 60) after the last one. A schedule floor scales out at once, whatever the
197
+ # cooldown.
198
+ cooldown:
199
+ up: 60
200
+ down: 600
201
+
202
+ # Containers added or removed per signal-driven step, and removed per scale-in step
203
+ # back to a floor; default `replicas/max` (one host's worth). A step lands on a count
204
+ # the hosts can split. Scale-out to a schedule floor goes straight to it.
205
+ step: 3
206
+
207
+ # The reactive rules: the controller reads `signal` every tick and holds it inside a
208
+ # band. Above `up/above` for `up/for` it scales out by `step`, at most once per
209
+ # `cooldown/up`, up to scale max × replicas max. Below `down/below` for `down/for` it
210
+ # scales in by `step`, at most once per `cooldown/down`, never below the floor (scale
211
+ # min × replicas min, raised by schedule windows) and never while a member warms up.
212
+ # Inside the band it holds. A sample it cannot read holds the role (`missing_data`),
213
+ # and starts both windows over; a floor still applies.
214
+ #
215
+ # `source` is one of:
216
+ #
217
+ # - `sidekiq`: the age in seconds of the oldest job across `queues`, read from Redis
218
+ # (`autoscale/redis`). Sidekiq 7's float seconds and Sidekiq 8's integer
219
+ # milliseconds are both read. Draining a Sidekiq container on scale-in ends as soon
220
+ # as Sidekiq reports it quiet with nothing busy, instead of always waiting
221
+ # `drain/wait`.
222
+ # - `pgbus`: the age in seconds of the oldest claimable message across `queues`,
223
+ # read from Postgres (`autoscale/postgres`): the pgmq table `pgmq.q_<prefix><queue>`,
224
+ # `prefix` defaulting to `pgbus_`. Scheduled messages do not count. pgbus has no
225
+ # quiet mode, so leave `drain/signal` unset and give the role a stop timeout of at
226
+ # least pgbus's `drain_timeout`.
227
+ # - `prometheus`: the value of one PromQL `query` (`autoscale/prometheus`), the
228
+ # largest when it returns several series. An empty result is a missing sample.
229
+ #
230
+ # A role with a `schedule` and no `signal` keeps returning to its floor a step at a time.
231
+ #
232
+ # Tuning: record a busy day with `dash autoscale run --record FILE`, then try a band,
233
+ # a step or a cooldown with `dash autoscale simulate --samples FILE` before deploying
234
+ # it. Judoscale's worker defaults make a sound start: a queue-latency band of 1-5
235
+ # seconds, up after one breach, down after 10 minutes. In an app whose roles share a
236
+ # database, give every signalled role a `hold_when` guard on it. `dash autoscale signal
237
+ # ROLE` reads the signal and guards live, as the controller does. Simulation replays
238
+ # what the queue did with the containers that really ran, not the simulated ones; it
239
+ # starts its cooldowns at the tick that acted, where live they start once the scale
240
+ # finishes, and its breach windows empty - so it is a guide, not a forecast.
241
+ signal:
242
+ source: sidekiq
243
+ queues:
244
+ - default
245
+ - billing
246
+
247
+ # Scale out above this, after this long above it (seconds, or a number with s, m, h
248
+ # or d, at most 1d). `for` defaults to 0: the first sample above.
249
+ up:
250
+ above: 10
251
+ for: 30s
252
+
253
+ # Scale in below this, after this long below it (default 10m). `up/above` must be
254
+ # greater than `down/below`.
255
+ down:
256
+ below: 2
257
+ for: 10m
258
+
259
+ # Saturation guards: while any query reads above its `above` - or cannot be read -
260
+ # a signal-driven scale-out holds (`held_by_guard`), since more containers would only
261
+ # queue on what is already full. `source` is `prometheus` (a PromQL query) or
262
+ # `postgres` (SQL, run read-only with a 5 second timeout; the first column of the
263
+ # first row). Guards never hold a schedule floor: a window is capacity asked for.
264
+ hold_when:
265
+ - source: prometheus
266
+ query: max(pgbouncer_pools_client_waiting_connections)
267
+ above: 0
268
+ - source: postgres
269
+ query: SELECT count(*) FROM pg_stat_activity
270
+ above: 180
271
+
69
272
  # Option values are shell-escaped before reaching `docker run`, but `${...}` is
70
273
  # deliberately left for the deploy host's shell to expand. Role `env` is never
71
274
  # defined in that shell, so `${MY_VAR}` resolves to empty. Use the bare `$MY_VAR`
@@ -26,8 +26,9 @@ class Dash::Configuration::Loadbalancer < Dash::Configuration::Proxy
26
26
  # The hosts the load balancer forwards to: every host of every role that
27
27
  # runs a proxy. One source of truth for the deploy step and the reboot
28
28
  # re-registration, so their target lists cannot diverge.
29
+ # Once each: a host two proxied roles share is one target.
29
30
  def target_hosts
30
- config.roles.select(&:running_proxy?).flat_map(&:hosts)
31
+ config.roles.select(&:running_proxy?).flat_map(&:hosts).uniq
31
32
  end
32
33
 
33
34
  def directory
@@ -95,7 +96,7 @@ class Dash::Configuration::Loadbalancer < Dash::Configuration::Proxy
95
96
 
96
97
  # When loadbalancer is on a proxy host, it takes over the proxy role
97
98
  def on_proxy_host?
98
- config.proxy_hosts.include?(config.proxy.effective_loadbalancer)
99
+ config.baseline_proxy_hosts.include?(config.proxy.effective_loadbalancer)
99
100
  end
100
101
 
101
102
  private
@@ -1,5 +1,5 @@
1
1
  class Dash::Configuration::Proxy::Run
2
- MINIMUM_VERSION = "v1.1.0.3"
2
+ MINIMUM_VERSION = "v1.1.0.4"
3
3
  DEFAULT_HTTP_PORT = 80
4
4
  DEFAULT_HTTPS_PORT = 443
5
5
  DEFAULT_LOG_MAX_SIZE = "10m"
@@ -251,7 +251,7 @@ class Dash::Configuration::Proxy
251
251
  end
252
252
 
253
253
  def loadbalancer_on_proxy_host?
254
- load_balancing? && config.proxy_hosts.include?(effective_loadbalancer)
254
+ load_balancing? && config.baseline_proxy_hosts.include?(effective_loadbalancer)
255
255
  end
256
256
 
257
257
  def custom_ssl_certificate?
@@ -391,8 +391,11 @@ class Dash::Configuration::Proxy
391
391
  .merge(target_options).compact
392
392
  end
393
393
 
394
- def deploy_command_args(target:)
395
- optionize ({ target: "#{target}:#{app_port}" }).merge(deploy_options), with: "="
394
+ # dash-proxy takes a comma-separated target list, waits until every target is healthy,
395
+ # then swaps the whole pool and drains the one it replaced - which is how all replicas
396
+ # of a role on a host go live together.
397
+ def deploy_command_args(targets:)
398
+ optionize ({ target: target_list(targets) }).merge(deploy_options), with: "="
396
399
  end
397
400
 
398
401
  # dash-proxy rollout deploy only accepts the target and the timeouts - the service already
@@ -404,8 +407,8 @@ class Dash::Configuration::Proxy
404
407
  }.compact
405
408
  end
406
409
 
407
- def rollout_deploy_command_args(target:)
408
- optionize ({ target: "#{target}:#{app_port}" }).merge(rollout_deploy_options), with: "="
410
+ def rollout_deploy_command_args(targets:)
411
+ optionize ({ target: target_list(targets) }).merge(rollout_deploy_options), with: "="
409
412
  end
410
413
 
411
414
  def rollout_set_command_args(percent: nil, list: nil)
@@ -428,6 +431,10 @@ class Dash::Configuration::Proxy
428
431
  end
429
432
 
430
433
  private
434
+ def target_list(targets)
435
+ Array(targets).map { |target| "#{target}:#{app_port}" }.join(",")
436
+ end
437
+
431
438
  # Which dispositions this layer keeps. The per-app proxy behind a load
432
439
  # balancer sheds the edge concerns; without load balancing there is no
433
440
  # other layer to defer to. Dash::Configuration::Loadbalancer overrides
@@ -436,8 +443,9 @@ class Dash::Configuration::Proxy
436
443
  load_balancing? ? %i[ per_app both ] : %i[ edge per_app both ]
437
444
  end
438
445
 
446
+ # A baseline host: the load balancer must not move with the pool.
439
447
  def primary_role_first_host
440
- config.primary_role&.hosts&.first
448
+ config.primary_role&.baseline_hosts&.first
441
449
  end
442
450
 
443
451
  # Auto-activation needs a role the load balancer can actually front. The
@@ -445,10 +453,14 @@ class Dash::Configuration::Proxy
445
453
  # Dash::Cli::Proxy#loadbalancer), so a proxy-less primary role would boot a
446
454
  # load balancer with an empty --target. An explicit `loadbalancer:` setting
447
455
  # skips this check — the operator asked for it.
456
+ #
457
+ # A scaled proxied role turns it on too, even with one baseline host: its members can
458
+ # only reach traffic through the load balancer's target list.
448
459
  def auto_load_balanced_primary_role?
449
460
  primary_role = config.primary_role
461
+ return false unless primary_role.present?
450
462
 
451
- primary_role.present? && primary_role.running_proxy? && Array(primary_role.hosts).size > 1
463
+ (primary_role.running_proxy? && primary_role.baseline_hosts.size > 1) || config.scaled_roles.any?(&:running_proxy?)
452
464
  end
453
465
 
454
466
  # Flags for dash-proxy's dynamic domain source (runtime TLS hostnames).
@@ -0,0 +1,49 @@
1
+ # How a container of a role without the proxy leaves when it is scaled in: an optional
2
+ # signal (e.g. TSTP, so Sidekiq stops fetching), then a wait, then `docker stop`. A
3
+ # proxied role has none - dash-proxy drains it.
4
+ class Dash::Configuration::Role::Drain
5
+ # The Linux signal names `docker kill --signal` accepts, without the SIG prefix.
6
+ SIGNALS = %w[
7
+ HUP INT QUIT ILL TRAP ABRT IOT BUS FPE KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT CHLD CONT
8
+ STOP TSTP TTIN TTOU URG XCPU XFSZ VTALRM PROF WINCH IO POLL PWR SYS
9
+ ]
10
+
11
+ # Linux signal numbers, SIGHUP through SIGRTMAX.
12
+ SIGNAL_NUMBERS = 1..64
13
+
14
+ attr_reader :context
15
+
16
+ def initialize(drain_config:, context:)
17
+ @drain_config = drain_config || {}
18
+ @context = context
19
+ validate!
20
+ end
21
+
22
+ def signal
23
+ @drain_config["signal"]&.to_s
24
+ end
25
+
26
+ def wait
27
+ @drain_config.fetch("wait", 0)
28
+ end
29
+
30
+ def configured?
31
+ @drain_config.present?
32
+ end
33
+
34
+ private
35
+ def validate!
36
+ raise Dash::ConfigurationError, "#{context}/wait: must be 0 or more seconds" if wait.negative?
37
+ return if signal.nil? || valid_signal?
38
+
39
+ raise Dash::ConfigurationError, "#{context}/signal: #{signal} is not a signal docker can send"
40
+ end
41
+
42
+ def valid_signal?
43
+ if signal.match?(/\A\d+\z/)
44
+ SIGNAL_NUMBERS.cover?(signal.to_i)
45
+ else
46
+ SIGNALS.include?(signal.upcase.delete_prefix("SIG"))
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,72 @@
1
+ # How many containers of one role run on each of its hosts. The bounds come from
2
+ # deploy.yml; the count in between is never stored - it is whatever is running, which
3
+ # `dash scale set` changes and a deploy reads back (see Dash::Cli::App::Boot).
4
+ class Dash::Configuration::Role::Replicas
5
+ # Docker options every replica on a host would claim for itself.
6
+ EXCLUSIVE_OPTIONS = %w[ publish p name hostname ]
7
+ VOLUME_OPTIONS = %w[ volume v mount volumes-from ]
8
+
9
+ attr_reader :min, :max, :context
10
+
11
+ def initialize(replicas_config:, context:)
12
+ @context = context
13
+
14
+ case replicas_config
15
+ when nil then @min = @max = 1
16
+ when Integer then @min = @max = replicas_config
17
+ else
18
+ @min = replicas_config.fetch("min", 1)
19
+ @max = replicas_config.fetch("max", [ @min, 1 ].max)
20
+ end
21
+
22
+ validate!
23
+ end
24
+
25
+ def scalable?
26
+ max > 1
27
+ end
28
+
29
+ def numbers
30
+ (1..max).to_a
31
+ end
32
+
33
+ # A deploy boots the running count, kept within the bounds: a crash cannot shrink the
34
+ # role below min, and a lowered max takes effect.
35
+ def clamp(running)
36
+ running.clamp(min, max)
37
+ end
38
+
39
+ def to_s
40
+ min == max ? "× #{max} #{"replica".pluralize(max)}" : "× #{min}–#{max} replicas"
41
+ end
42
+
43
+ # Checks that need the rest of the role (its docker options, its proxy), which is only
44
+ # complete once Dash::Configuration has finished building - see
45
+ # Dash::Configuration#ensure_replicas_fit_their_roles.
46
+ def ensure_fits!(role, volumes:)
47
+ return unless scalable?
48
+
49
+ if (option = role.docker_option_keys.find { |key| EXCLUSIVE_OPTIONS.include?(key) })
50
+ error "max > 1 cannot be combined with options/#{option}, every replica on a host would claim it"
51
+ end
52
+
53
+ if role.running_proxy? && role.proxy.proxy_config.dig("sleep", "after").present?
54
+ error "max > 1 cannot be combined with proxy/sleep until dash-proxy is proven to sleep and wake every replica together"
55
+ end
56
+
57
+ shared = volumes + role.docker_option_values(*VOLUME_OPTIONS)
58
+ shared.each do |volume|
59
+ warn "#{context}: every replica on a host shares the volume #{volume} - fine for read-only config or a shared cache, wrong for a per-process store"
60
+ end
61
+ end
62
+
63
+ private
64
+ def validate!
65
+ error "min must be at least 1" unless min >= 1
66
+ error "min (#{min}) cannot be greater than max (#{max})" if min > max
67
+ end
68
+
69
+ def error(message)
70
+ raise Dash::ConfigurationError, "#{context}: #{message}"
71
+ end
72
+ end
@@ -0,0 +1,40 @@
1
+ # One `scale.hold_when` entry: a saturation guard. While its query reads above `above` (or
2
+ # cannot be read), the controller holds a signal-driven scale-out - more containers would
3
+ # only queue on the database. A schedule floor still applies. See Dash::Autoscale::Guard.
4
+ class Dash::Configuration::Role::Scale::Guard
5
+ SOURCES = %w[ prometheus postgres ]
6
+ KEYS = %w[ source query above ]
7
+
8
+ attr_reader :source, :query, :above
9
+
10
+ def initialize(guard_config:, context:)
11
+ @context = context
12
+ error nil, "should be a hash" unless guard_config.is_a?(Hash)
13
+ if (unknown = (guard_config.keys.map(&:to_s) - KEYS).first)
14
+ error nil, "unknown key: #{unknown}"
15
+ end
16
+
17
+ @source = guard_config["source"].to_s
18
+ @query = guard_config["query"]
19
+ @above = guard_config["above"]
20
+
21
+ error "source", "must be #{SOURCES.join(" or ")}, not #{source}" unless SOURCES.include?(source)
22
+ error "query", "is required" unless query.is_a?(String) && query.present?
23
+ error "above", "is required" if above.nil?
24
+ error "above", "should be a finite number" unless above.is_a?(Numeric) && above.real? && above.finite?
25
+ end
26
+
27
+ # The autoscale/<connection> block the guard reads through.
28
+ def connection
29
+ source
30
+ end
31
+
32
+ def to_h
33
+ { source: source, query: query, above: above }
34
+ end
35
+
36
+ private
37
+ def error(key, message)
38
+ raise Dash::ConfigurationError, "#{[ @context, key ].compact.join("/")}: #{message}"
39
+ end
40
+ end
@@ -0,0 +1,93 @@
1
+ # The `scale.signal` entry: the one number the controller scales a role on. `sidekiq` and
2
+ # `pgbus` are the age in seconds of the oldest job waiting across `queues`; `prometheus` is
3
+ # whatever its `query` returns (the largest value of a vector). See Dash::Autoscale::Signal.
4
+ class Dash::Configuration::Role::Scale::Signal
5
+ SOURCES = %w[ sidekiq pgbus prometheus ]
6
+ KEYS = %w[ source queues prefix query ]
7
+ NAME = /\A[A-Za-z0-9_.:-]+\z/
8
+ DEFAULT_PGBUS_PREFIX = "pgbus_"
9
+
10
+ # The autoscale/<connection> block each source reads through.
11
+ CONNECTIONS = { "sidekiq" => "redis", "pgbus" => "postgres", "prometheus" => "prometheus" }.freeze
12
+
13
+ attr_reader :source, :queues, :prefix, :query
14
+
15
+ def initialize(signal_config:, context:)
16
+ @context = context
17
+ error nil, "should be a hash" unless signal_config.is_a?(Hash)
18
+ if (unknown = (signal_config.keys.map(&:to_s) - KEYS).first)
19
+ error nil, "unknown key: #{unknown}"
20
+ end
21
+
22
+ @source = signal_config["source"].to_s
23
+ validate_source!
24
+ @queues = parse_queues(signal_config["queues"])
25
+ @prefix = parse_prefix(signal_config["prefix"])
26
+ @query = parse_query(signal_config["query"])
27
+ end
28
+
29
+ def queues?
30
+ source != "prometheus"
31
+ end
32
+
33
+ def connection
34
+ CONNECTIONS.fetch(source)
35
+ end
36
+
37
+ def to_h
38
+ { source: source, queues: queues, prefix: prefix, query: query }.compact
39
+ end
40
+
41
+ private
42
+ def validate_source!
43
+ if source == "proxy"
44
+ error "source", "proxy (web queue time) comes with Phase 4 (zoolutions/dash#180), use #{SOURCES.to_sentence(last_word_connector: " or ")}"
45
+ end
46
+ error "source", "must be #{SOURCES.to_sentence(last_word_connector: " or ")}, not #{source}" unless SOURCES.include?(source)
47
+ end
48
+
49
+ def parse_queues(queues)
50
+ unless queues?
51
+ error "queues", "is only used with sidekiq and pgbus" unless queues.nil?
52
+ return
53
+ end
54
+
55
+ error "queues", "is required for #{source}" if queues.nil? || queues == []
56
+ error "queues", "should be an array" unless queues.is_a?(Array)
57
+ queues.each_with_index { |queue, index| error "queues/#{index}", "#{queue.inspect} #{name_rule}" unless name?(queue) }
58
+ queues
59
+ end
60
+
61
+ def parse_prefix(prefix)
62
+ unless source == "pgbus"
63
+ error "prefix", "is only used with pgbus" unless prefix.nil?
64
+ return
65
+ end
66
+
67
+ return DEFAULT_PGBUS_PREFIX if prefix.nil?
68
+ error "prefix", "#{prefix.inspect} #{name_rule}" unless name?(prefix)
69
+ prefix
70
+ end
71
+
72
+ def parse_query(query)
73
+ if queues?
74
+ error "query", "is only used with prometheus" unless query.nil?
75
+ return
76
+ end
77
+
78
+ error "query", "is required for prometheus" unless query.is_a?(String) && query.present?
79
+ query
80
+ end
81
+
82
+ def name?(value)
83
+ value.is_a?(String) && NAME.match?(value)
84
+ end
85
+
86
+ def name_rule
87
+ "should be letters, digits, _ . : or -"
88
+ end
89
+
90
+ def error(key, message)
91
+ raise Dash::ConfigurationError, "#{[ @context, key ].compact.join("/")}: #{message}"
92
+ end
93
+ end
@@ -0,0 +1,37 @@
1
+ # One `scale.schedule` entry: from every minute `cron` matches, for `duration` seconds, the
2
+ # role runs at least `min` containers. Times are handed in already in `autoscale.timezone`,
3
+ # so the cron is wall-clock there while the duration is real seconds (a window that spans a
4
+ # DST change still lasts `for`).
5
+ class Dash::Configuration::Role::Scale::Window
6
+ attr_reader :cron, :duration, :min
7
+
8
+ def initialize(cron:, duration:, min:)
9
+ @cron = cron
10
+ @duration = duration
11
+ @min = min
12
+ end
13
+
14
+ def active_at?(now)
15
+ !started_at(now).nil?
16
+ end
17
+
18
+ # The latest minute in (now - duration, now] the cron matches, scanned backwards: at most
19
+ # 10 080 steps for the 7-day limit.
20
+ def started_at(now)
21
+ earliest = now - duration
22
+ minute = now.change(sec: 0)
23
+
24
+ while minute > earliest
25
+ return minute if cron.match?(minute)
26
+ minute -= 60
27
+ end
28
+ end
29
+
30
+ def ends_at(now)
31
+ started_at(now)&.+(duration)
32
+ end
33
+
34
+ def to_h
35
+ { cron: cron.to_s, for: duration, min: min }
36
+ end
37
+ end