dash 4.1.3 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/dash/autoscale/connections/builder.rb +51 -0
- data/lib/dash/autoscale/connections.rb +102 -0
- data/lib/dash/autoscale/controller.rb +296 -0
- data/lib/dash/autoscale/cron.rb +75 -0
- data/lib/dash/autoscale/decision.rb +89 -0
- data/lib/dash/autoscale/duration.rb +19 -0
- data/lib/dash/autoscale/evaluation.rb +52 -0
- data/lib/dash/autoscale/guard.rb +48 -0
- data/lib/dash/autoscale/lease.rb +55 -0
- data/lib/dash/autoscale/lease_held.rb +2 -0
- data/lib/dash/autoscale/lease_lost.rb +2 -0
- data/lib/dash/autoscale/member.rb +28 -0
- data/lib/dash/autoscale/metrics.rb +70 -0
- data/lib/dash/autoscale/metrics_server.rb +71 -0
- data/lib/dash/autoscale/observation.rb +41 -0
- data/lib/dash/autoscale/pause.rb +34 -0
- data/lib/dash/autoscale/policy.rb +198 -0
- data/lib/dash/autoscale/pool.rb +103 -0
- data/lib/dash/autoscale/prometheus_client.rb +76 -0
- data/lib/dash/autoscale/provider/base.rb +43 -0
- data/lib/dash/autoscale/provider/exec.rb +94 -0
- data/lib/dash/autoscale/provider/upcloud.rb +140 -0
- data/lib/dash/autoscale/provider.rb +9 -0
- data/lib/dash/autoscale/provider_error.rb +4 -0
- data/lib/dash/autoscale/read_only_query.rb +38 -0
- data/lib/dash/autoscale/reader.rb +17 -0
- data/lib/dash/autoscale/recorder.rb +13 -0
- data/lib/dash/autoscale/role_state.rb +110 -0
- data/lib/dash/autoscale/sample.rb +13 -0
- data/lib/dash/autoscale/signal/pgbus.rb +19 -0
- data/lib/dash/autoscale/signal/prometheus.rb +8 -0
- data/lib/dash/autoscale/signal/sidekiq.rb +49 -0
- data/lib/dash/autoscale/signal.rb +31 -0
- data/lib/dash/autoscale/simulation.rb +200 -0
- data/lib/dash/autoscale/state_store.rb +132 -0
- data/lib/dash/autoscale/timestamp.rb +15 -0
- data/lib/dash/autoscale/tunnel.rb +48 -0
- data/lib/dash/cli/accessory.rb +16 -0
- data/lib/dash/cli/app/boot.rb +69 -44
- data/lib/dash/cli/app/boot_state.rb +70 -0
- data/lib/dash/cli/app/rollout_boot.rb +28 -9
- data/lib/dash/cli/app/running_targets.rb +39 -0
- data/lib/dash/cli/app/slot_version.rb +29 -0
- data/lib/dash/cli/app.rb +89 -32
- data/lib/dash/cli/autoscale.rb +312 -0
- data/lib/dash/cli/base.rb +15 -3
- data/lib/dash/cli/lock.rb +6 -1
- data/lib/dash/cli/main.rb +77 -8
- data/lib/dash/cli/proxy/inspection.rb +52 -0
- data/lib/dash/cli/proxy/loadbalancer_deploy.rb +20 -0
- data/lib/dash/cli/proxy/loadbalancer_reboot.rb +1 -1
- data/lib/dash/cli/proxy/reboot.rb +4 -9
- data/lib/dash/cli/proxy.rb +12 -7
- data/lib/dash/cli/prune.rb +5 -3
- data/lib/dash/cli/report.rb +4 -1
- data/lib/dash/cli/scale/host_plan.rb +63 -0
- data/lib/dash/cli/scale/join.rb +128 -0
- data/lib/dash/cli/scale/leave.rb +58 -0
- data/lib/dash/cli/scale/placement.rb +54 -0
- data/lib/dash/cli/scale/power_off.rb +26 -0
- data/lib/dash/cli/scale/replica_join.rb +74 -0
- data/lib/dash/cli/scale/replica_leave.rb +121 -0
- data/lib/dash/cli/scale.rb +276 -0
- data/lib/dash/cli/server.rb +41 -0
- data/lib/dash/cli/stats_table.rb +34 -0
- data/lib/dash/cli/templates/sample_hooks/post-scale-in.sample +3 -0
- data/lib/dash/cli/templates/sample_hooks/post-scale-out.sample +3 -0
- data/lib/dash/cli/templates/sample_hooks/pre-scale-in.sample +6 -0
- data/lib/dash/cli/templates/sample_hooks/pre-scale-out.sample +6 -0
- data/lib/dash/commander.rb +17 -3
- data/lib/dash/commands/accessory/proxy.rb +1 -1
- data/lib/dash/commands/accessory.rb +21 -2
- data/lib/dash/commands/app/containers.rb +28 -3
- data/lib/dash/commands/app/logging.rb +13 -2
- data/lib/dash/commands/app/proxy.rb +4 -4
- data/lib/dash/commands/app.rb +90 -10
- data/lib/dash/commands/auditor.rb +2 -2
- data/lib/dash/commands/autoscale.rb +133 -0
- data/lib/dash/commands/base.rb +4 -0
- data/lib/dash/commands/proxy/services.rb +10 -0
- data/lib/dash/commands/prune.rb +10 -2
- data/lib/dash/commands/server.rb +16 -0
- data/lib/dash/configuration/accessory.rb +2 -2
- data/lib/dash/configuration/autoscale.rb +123 -0
- data/lib/dash/configuration/docs/autoscale.yml +141 -0
- data/lib/dash/configuration/docs/configuration.yml +8 -2
- data/lib/dash/configuration/docs/proxy.yml +1 -1
- data/lib/dash/configuration/docs/role.yml +203 -0
- data/lib/dash/configuration/loadbalancer.rb +3 -2
- data/lib/dash/configuration/proxy/run.rb +1 -1
- data/lib/dash/configuration/proxy.rb +19 -7
- data/lib/dash/configuration/role/drain.rb +49 -0
- data/lib/dash/configuration/role/replicas.rb +72 -0
- data/lib/dash/configuration/role/scale/guard.rb +40 -0
- data/lib/dash/configuration/role/scale/signal.rb +93 -0
- data/lib/dash/configuration/role/scale/window.rb +37 -0
- data/lib/dash/configuration/role/scale.rb +199 -0
- data/lib/dash/configuration/role.rb +123 -6
- data/lib/dash/configuration/validator/autoscale.rb +126 -0
- data/lib/dash/configuration/validator/proxy.rb +13 -0
- data/lib/dash/configuration/validator/role.rb +26 -5
- data/lib/dash/configuration.rb +126 -15
- data/lib/dash/diagnostics/audit.rb +28 -0
- data/lib/dash/diagnostics/autoscale_decisions.rb +18 -0
- data/lib/dash/diagnostics/autoscale_explain.rb +30 -0
- data/lib/dash/diagnostics/autoscale_signal.rb +82 -0
- data/lib/dash/diagnostics/base.rb +53 -0
- data/lib/dash/diagnostics/config.rb +32 -0
- data/lib/dash/diagnostics/container_stats.rb +67 -0
- data/lib/dash/diagnostics/containers.rb +20 -0
- data/lib/dash/diagnostics/controller_status.rb +44 -0
- data/lib/dash/diagnostics/docker_ps.rb +40 -0
- data/lib/dash/{cli → diagnostics}/doctor/config_checks.rb +3 -3
- data/lib/dash/diagnostics/doctor/controller_checks.rb +46 -0
- data/lib/dash/{cli → diagnostics}/doctor/endpoint_checks.rb +12 -4
- data/lib/dash/{cli → diagnostics}/doctor/host_checks.rb +5 -4
- data/lib/dash/diagnostics/doctor/pool_checks.rb +65 -0
- data/lib/dash/diagnostics/doctor/signal_checks.rb +54 -0
- data/lib/dash/diagnostics/doctor.rb +182 -0
- data/lib/dash/diagnostics/drift.rb +168 -0
- data/lib/dash/diagnostics/host_stats.rb +59 -0
- data/lib/dash/diagnostics/lines.rb +14 -0
- data/lib/dash/diagnostics/lock.rb +27 -0
- data/lib/dash/diagnostics/logs.rb +76 -0
- data/lib/dash/diagnostics/pool.rb +41 -0
- data/lib/dash/diagnostics/proxy_services.rb +36 -0
- data/lib/dash/diagnostics/redactor.rb +48 -0
- data/lib/dash/diagnostics/reports.rb +17 -0
- data/lib/dash/diagnostics/scale.rb +51 -0
- data/lib/dash/diagnostics/units.rb +31 -0
- data/lib/dash/mcp/base_tool.rb +43 -0
- data/lib/dash/mcp/runner.rb +28 -0
- data/lib/dash/mcp/server.rb +40 -0
- data/lib/dash/mcp/session.rb +51 -0
- data/lib/dash/mcp/tools/audit_tool.rb +11 -0
- data/lib/dash/mcp/tools/autoscale_decisions_tool.rb +22 -0
- data/lib/dash/mcp/tools/autoscale_explain_tool.rb +56 -0
- data/lib/dash/mcp/tools/config_tool.rb +10 -0
- data/lib/dash/mcp/tools/container_stats_tool.rb +15 -0
- data/lib/dash/mcp/tools/containers_tool.rb +10 -0
- data/lib/dash/mcp/tools/controller_status_tool.rb +18 -0
- data/lib/dash/mcp/tools/deploy_reports_tool.rb +10 -0
- data/lib/dash/mcp/tools/doctor_tool.rb +13 -0
- data/lib/dash/mcp/tools/drift_tool.rb +16 -0
- data/lib/dash/mcp/tools/host_stats_tool.rb +14 -0
- data/lib/dash/mcp/tools/lock_status_tool.rb +15 -0
- data/lib/dash/mcp/tools/logs_tool.rb +41 -0
- data/lib/dash/mcp/tools/pool_members_tool.rb +15 -0
- data/lib/dash/mcp/tools/proxy_services_tool.rb +10 -0
- data/lib/dash/mcp/tools/scale_status_tool.rb +10 -0
- data/lib/dash/mcp/tools/signal_sample_tool.rb +19 -0
- data/lib/dash/mcp/transport.rb +14 -0
- data/lib/dash/mcp.rb +56 -0
- data/lib/dash/sshkit_with_ext.rb +4 -4
- data/lib/dash/utils.rb +7 -0
- data/lib/dash/version.rb +1 -1
- data/lib/dash.rb +2 -0
- metadata +119 -5
- data/lib/dash/cli/doctor.rb +0 -113
|
@@ -136,8 +136,8 @@ allow_empty_roles: false
|
|
|
136
136
|
|
|
137
137
|
# Retain containers
|
|
138
138
|
#
|
|
139
|
-
# How many old containers we retain per role
|
|
140
|
-
# defaults to 5:
|
|
139
|
+
# How many old containers we retain per role (per replica slot, for a role with
|
|
140
|
+
# `replicas`), and how many images we retain, defaults to 5:
|
|
141
141
|
retain_containers: 3
|
|
142
142
|
|
|
143
143
|
# Minimum version
|
|
@@ -205,6 +205,12 @@ accessories:
|
|
|
205
205
|
proxy:
|
|
206
206
|
...
|
|
207
207
|
|
|
208
|
+
# Autoscale
|
|
209
|
+
#
|
|
210
|
+
# The provider that holds the pool members of scaled roles, see dash docs autoscale:
|
|
211
|
+
autoscale:
|
|
212
|
+
...
|
|
213
|
+
|
|
208
214
|
# SSHKit
|
|
209
215
|
#
|
|
210
216
|
# See dash docs sshkit:
|
|
@@ -837,7 +837,7 @@ proxy:
|
|
|
837
837
|
# below already embeds its ghcr.io host
|
|
838
838
|
repository: ghcr.io/zoolutions/dash-proxy # Container repository for the
|
|
839
839
|
# dash-proxy image (this is the default)
|
|
840
|
-
version: v1.1.0.
|
|
840
|
+
version: v1.1.0.4 # Version tag of the dash-proxy image to use.
|
|
841
841
|
# Defaults to the minimum version this gem
|
|
842
842
|
# requires - only pin it to roll forward early,
|
|
843
843
|
# never below the default
|
|
@@ -66,6 +66,209 @@ servers:
|
|
|
66
66
|
limit: 25%
|
|
67
67
|
wait: 10
|
|
68
68
|
|
|
69
|
+
# Replicas
|
|
70
|
+
#
|
|
71
|
+
# How many containers of this role run on EACH of its hosts. A process that cannot
|
|
72
|
+
# use more than one core by itself — open-source Sidekiq is one Ruby process — gets
|
|
73
|
+
# the rest of a bigger host this way, without declaring the same worker twice.
|
|
74
|
+
#
|
|
75
|
+
# `replicas: 2` is shorthand for `{ min: 2, max: 2 }`; the default is one, which is
|
|
76
|
+
# the behaviour without the key. A deploy boots as many replicas as are running now,
|
|
77
|
+
# kept within `min` and `max`, so a count changed at runtime with
|
|
78
|
+
# `dash scale set <role> <count>` survives the next deploy. With nothing running it
|
|
79
|
+
# boots `min`. `max` is the ceiling for `dash scale set`.
|
|
80
|
+
#
|
|
81
|
+
# Replicas of a proxied role are all targets of the role's one dash-proxy service,
|
|
82
|
+
# which balances across them. Replicas of any other role are independent containers.
|
|
83
|
+
# Replica 1 keeps the container name the role always had; replica 2 and up put the
|
|
84
|
+
# slot into the role segment (`app-workers.2-<version>`), and every replica of a
|
|
85
|
+
# scalable role gets the `replica` label and the `DASH_REPLICA` env var.
|
|
86
|
+
#
|
|
87
|
+
# Every replica opens its own database pool, so `pool × replicas × hosts` has to fit
|
|
88
|
+
# the database's (or the pooler's) connection limit.
|
|
89
|
+
#
|
|
90
|
+
# More than one replica cannot set the `publish`, `p`, `name` or `hostname` docker
|
|
91
|
+
# options, which every replica on a host would claim, and cannot yet be combined with
|
|
92
|
+
# `proxy/sleep`. Host volumes are shared by every replica on a host, which is fine for
|
|
93
|
+
# read-only config or a shared cache and wrong for a per-process store, so they warn.
|
|
94
|
+
replicas:
|
|
95
|
+
min: 1
|
|
96
|
+
max: 4
|
|
97
|
+
|
|
98
|
+
# Drain
|
|
99
|
+
#
|
|
100
|
+
# How a replica of a role WITHOUT the proxy leaves when `dash scale set` removes it:
|
|
101
|
+
# `signal` is sent first (e.g. TSTP, so Sidekiq stops taking new jobs), then dash
|
|
102
|
+
# waits `wait` seconds before `docker stop`. A role with a `sidekiq` scale signal stops
|
|
103
|
+
# waiting as soon as Sidekiq reports the leaving containers' processes quiet with
|
|
104
|
+
# nothing busy (it reports both every few seconds), so `wait` is then the most it waits. Keep `wait` inside the stop budget described on the worker
|
|
105
|
+
# roles page. A role behind dash-proxy has no drain: the proxy drains it, so setting one
|
|
106
|
+
# there is an error.
|
|
107
|
+
drain:
|
|
108
|
+
signal: TSTP
|
|
109
|
+
wait: 20
|
|
110
|
+
|
|
111
|
+
# Scale
|
|
112
|
+
#
|
|
113
|
+
# Extra hosts a role can run on beyond the ones listed under `hosts`, which stay the
|
|
114
|
+
# baseline: always on, deployed as before, never stopped by dash. The extra hosts are
|
|
115
|
+
# pool members that live in the cloud provider named under the root `autoscale:` key
|
|
116
|
+
# (see dash docs autoscale), not in this file. dash finds them by their labels —
|
|
117
|
+
# `dash.service=<service>`, `dash.destination=<destination, or - without one>` and
|
|
118
|
+
# `dash.role=<role>` — and every member that is `started` is a host of the role for
|
|
119
|
+
# every command; stopped members are never touched. A provider that cannot be reached
|
|
120
|
+
# fails the command. `--hosts` then names the member hosts to act on, with a warning —
|
|
121
|
+
# for one scaled role at a time (pick it with `--roles` when there are more).
|
|
122
|
+
#
|
|
123
|
+
# `min` and `max` count HOSTS, the baseline included, so `min` is at least the number of
|
|
124
|
+
# `hosts`; `replicas` counts containers per host. `dash scale set <role> <count>` counts
|
|
125
|
+
# containers across all of them.
|
|
126
|
+
#
|
|
127
|
+
# A scaled role behind dash-proxy needs the load balancer, which turns on by itself for
|
|
128
|
+
# it even with one baseline host; `proxy/loadbalancer: false` is an error for it.
|
|
129
|
+
#
|
|
130
|
+
# `schedule`, `signal`, `warmup`, `cooldown` and `step` drive the autoscaling controller
|
|
131
|
+
# (`dash autoscale run`), which then owns the role's count: a manual `dash scale set`
|
|
132
|
+
# lasts until its next tick unless the role is paused (`dash autoscale pause ROLE`). A
|
|
133
|
+
# scaled role with neither a `schedule` nor a `signal` stays manual.
|
|
134
|
+
scale:
|
|
135
|
+
min: 2
|
|
136
|
+
max: 4
|
|
137
|
+
|
|
138
|
+
# How extra hosts come to exist. `power` (the default) powers stopped members on and
|
|
139
|
+
# off again; they are created beforehand, with Terraform for example, carrying the
|
|
140
|
+
# three labels. `create` creates a member from `template` when one is needed and
|
|
141
|
+
# destroys it when it leaves. On UpCloud a stopped server's compute is free only on
|
|
142
|
+
# Cloud Native plans; any other plan bills a powered-off member as if it ran.
|
|
143
|
+
members: power
|
|
144
|
+
|
|
145
|
+
# What a `create` member is built from (UpCloud): the template storage to clone, the
|
|
146
|
+
# plan and zone, the user dash connects as with its SSH keys, the private network
|
|
147
|
+
# (SDN) to attach — required when `address` is private — and cloud-init user data,
|
|
148
|
+
# for a Tailscale join for example. Only `create` uses it, and `create` requires it.
|
|
149
|
+
template:
|
|
150
|
+
storage: 01234567-89ab-cdef-0123-456789abcdef
|
|
151
|
+
plan: CLOUDNATIVE-8xCPU-16GB
|
|
152
|
+
zone: de-fra1
|
|
153
|
+
login_user: deploy
|
|
154
|
+
ssh_keys:
|
|
155
|
+
- "ssh-ed25519 AAAA... deploy"
|
|
156
|
+
network: 03abcdef-0123-4567-89ab-cdef01234567
|
|
157
|
+
user_data: |
|
|
158
|
+
#!/bin/sh
|
|
159
|
+
tailscale up --authkey ...
|
|
160
|
+
|
|
161
|
+
# Which of a member's addresses dash connects to: private (the default), public or
|
|
162
|
+
# utility.
|
|
163
|
+
address: private
|
|
164
|
+
|
|
165
|
+
# Seconds from powering a member on (or creating it) until it answers SSH, default 300.
|
|
166
|
+
# The controller also replaces a started member that has not answered SSH for this
|
|
167
|
+
# long: out of the load balancer, powered off (deleted for `create`), and the next
|
|
168
|
+
# tick restores the count with a healthy one.
|
|
169
|
+
boot_timeout: 300
|
|
170
|
+
|
|
171
|
+
# Windows of guaranteed capacity, in containers (the unit of `dash scale set`). From
|
|
172
|
+
# every minute `cron` matches, for `for` (seconds, or a number with s, m, h or d, at
|
|
173
|
+
# most 7d), the controller keeps at least `min` containers running; overlapping
|
|
174
|
+
# windows take the largest `min`. Outside every window it returns to scale min ×
|
|
175
|
+
# replicas min. The target never goes above scale max × replicas max.
|
|
176
|
+
#
|
|
177
|
+
# `cron` has five fields - minute, hour, day-of-month, month, day-of-week - each `*`,
|
|
178
|
+
# a number, a range `a-b`, a list `a,b` or a step `*/n` or `a-b/n`; day-of-week 0 and
|
|
179
|
+
# 7 are Sunday. When both day fields are set, a day matches if either does. There
|
|
180
|
+
# are no names, `L`, `W` or `#`. It is evaluated in `autoscale/timezone`.
|
|
181
|
+
#
|
|
182
|
+
# "The evening before the 1st and the 15th" has no exact cron: this example starts on
|
|
183
|
+
# the 14th and on each of the 28th to 31st (so on several evenings at a month end).
|
|
184
|
+
# When a few hours of lead are not needed, start on the day itself: `0 0 1,15 * *`.
|
|
185
|
+
schedule:
|
|
186
|
+
- cron: "0 22 14,28-31 * *"
|
|
187
|
+
for: 30h
|
|
188
|
+
min: 10
|
|
189
|
+
|
|
190
|
+
# Seconds a member that just joined counts as warming up: the controller never scales
|
|
191
|
+
# in while one is, nor replaces it for not answering SSH. Default: `boot_timeout`.
|
|
192
|
+
warmup: 300
|
|
193
|
+
|
|
194
|
+
# Seconds between the controller's actions. A scale-in step waits `down` (default
|
|
195
|
+
# 600) after the last scale-out or scale-in. A signal-driven scale-out waits `up`
|
|
196
|
+
# (default 60) after the last one. A schedule floor scales out at once, whatever the
|
|
197
|
+
# cooldown.
|
|
198
|
+
cooldown:
|
|
199
|
+
up: 60
|
|
200
|
+
down: 600
|
|
201
|
+
|
|
202
|
+
# Containers added or removed per signal-driven step, and removed per scale-in step
|
|
203
|
+
# back to a floor; default `replicas/max` (one host's worth). A step lands on a count
|
|
204
|
+
# the hosts can split. Scale-out to a schedule floor goes straight to it.
|
|
205
|
+
step: 3
|
|
206
|
+
|
|
207
|
+
# The reactive rules: the controller reads `signal` every tick and holds it inside a
|
|
208
|
+
# band. Above `up/above` for `up/for` it scales out by `step`, at most once per
|
|
209
|
+
# `cooldown/up`, up to scale max × replicas max. Below `down/below` for `down/for` it
|
|
210
|
+
# scales in by `step`, at most once per `cooldown/down`, never below the floor (scale
|
|
211
|
+
# min × replicas min, raised by schedule windows) and never while a member warms up.
|
|
212
|
+
# Inside the band it holds. A sample it cannot read holds the role (`missing_data`),
|
|
213
|
+
# and starts both windows over; a floor still applies.
|
|
214
|
+
#
|
|
215
|
+
# `source` is one of:
|
|
216
|
+
#
|
|
217
|
+
# - `sidekiq`: the age in seconds of the oldest job across `queues`, read from Redis
|
|
218
|
+
# (`autoscale/redis`). Sidekiq 7's float seconds and Sidekiq 8's integer
|
|
219
|
+
# milliseconds are both read. Draining a Sidekiq container on scale-in ends as soon
|
|
220
|
+
# as Sidekiq reports it quiet with nothing busy, instead of always waiting
|
|
221
|
+
# `drain/wait`.
|
|
222
|
+
# - `pgbus`: the age in seconds of the oldest claimable message across `queues`,
|
|
223
|
+
# read from Postgres (`autoscale/postgres`): the pgmq table `pgmq.q_<prefix><queue>`,
|
|
224
|
+
# `prefix` defaulting to `pgbus_`. Scheduled messages do not count. pgbus has no
|
|
225
|
+
# quiet mode, so leave `drain/signal` unset and give the role a stop timeout of at
|
|
226
|
+
# least pgbus's `drain_timeout`.
|
|
227
|
+
# - `prometheus`: the value of one PromQL `query` (`autoscale/prometheus`), the
|
|
228
|
+
# largest when it returns several series. An empty result is a missing sample.
|
|
229
|
+
#
|
|
230
|
+
# A role with a `schedule` and no `signal` keeps returning to its floor a step at a time.
|
|
231
|
+
#
|
|
232
|
+
# Tuning: record a busy day with `dash autoscale run --record FILE`, then try a band,
|
|
233
|
+
# a step or a cooldown with `dash autoscale simulate --samples FILE` before deploying
|
|
234
|
+
# it. Judoscale's worker defaults make a sound start: a queue-latency band of 1-5
|
|
235
|
+
# seconds, up after one breach, down after 10 minutes. In an app whose roles share a
|
|
236
|
+
# database, give every signalled role a `hold_when` guard on it. `dash autoscale signal
|
|
237
|
+
# ROLE` reads the signal and guards live, as the controller does. Simulation replays
|
|
238
|
+
# what the queue did with the containers that really ran, not the simulated ones; it
|
|
239
|
+
# starts its cooldowns at the tick that acted, where live they start once the scale
|
|
240
|
+
# finishes, and its breach windows empty - so it is a guide, not a forecast.
|
|
241
|
+
signal:
|
|
242
|
+
source: sidekiq
|
|
243
|
+
queues:
|
|
244
|
+
- default
|
|
245
|
+
- billing
|
|
246
|
+
|
|
247
|
+
# Scale out above this, after this long above it (seconds, or a number with s, m, h
|
|
248
|
+
# or d, at most 1d). `for` defaults to 0: the first sample above.
|
|
249
|
+
up:
|
|
250
|
+
above: 10
|
|
251
|
+
for: 30s
|
|
252
|
+
|
|
253
|
+
# Scale in below this, after this long below it (default 10m). `up/above` must be
|
|
254
|
+
# greater than `down/below`.
|
|
255
|
+
down:
|
|
256
|
+
below: 2
|
|
257
|
+
for: 10m
|
|
258
|
+
|
|
259
|
+
# Saturation guards: while any query reads above its `above` - or cannot be read -
|
|
260
|
+
# a signal-driven scale-out holds (`held_by_guard`), since more containers would only
|
|
261
|
+
# queue on what is already full. `source` is `prometheus` (a PromQL query) or
|
|
262
|
+
# `postgres` (SQL, run read-only with a 5 second timeout; the first column of the
|
|
263
|
+
# first row). Guards never hold a schedule floor: a window is capacity asked for.
|
|
264
|
+
hold_when:
|
|
265
|
+
- source: prometheus
|
|
266
|
+
query: max(pgbouncer_pools_client_waiting_connections)
|
|
267
|
+
above: 0
|
|
268
|
+
- source: postgres
|
|
269
|
+
query: SELECT count(*) FROM pg_stat_activity
|
|
270
|
+
above: 180
|
|
271
|
+
|
|
69
272
|
# Option values are shell-escaped before reaching `docker run`, but `${...}` is
|
|
70
273
|
# deliberately left for the deploy host's shell to expand. Role `env` is never
|
|
71
274
|
# defined in that shell, so `${MY_VAR}` resolves to empty. Use the bare `$MY_VAR`
|
|
@@ -26,8 +26,9 @@ class Dash::Configuration::Loadbalancer < Dash::Configuration::Proxy
|
|
|
26
26
|
# The hosts the load balancer forwards to: every host of every role that
|
|
27
27
|
# runs a proxy. One source of truth for the deploy step and the reboot
|
|
28
28
|
# re-registration, so their target lists cannot diverge.
|
|
29
|
+
# Once each: a host two proxied roles share is one target.
|
|
29
30
|
def target_hosts
|
|
30
|
-
config.roles.select(&:running_proxy?).flat_map(&:hosts)
|
|
31
|
+
config.roles.select(&:running_proxy?).flat_map(&:hosts).uniq
|
|
31
32
|
end
|
|
32
33
|
|
|
33
34
|
def directory
|
|
@@ -95,7 +96,7 @@ class Dash::Configuration::Loadbalancer < Dash::Configuration::Proxy
|
|
|
95
96
|
|
|
96
97
|
# When loadbalancer is on a proxy host, it takes over the proxy role
|
|
97
98
|
def on_proxy_host?
|
|
98
|
-
config.
|
|
99
|
+
config.baseline_proxy_hosts.include?(config.proxy.effective_loadbalancer)
|
|
99
100
|
end
|
|
100
101
|
|
|
101
102
|
private
|
|
@@ -251,7 +251,7 @@ class Dash::Configuration::Proxy
|
|
|
251
251
|
end
|
|
252
252
|
|
|
253
253
|
def loadbalancer_on_proxy_host?
|
|
254
|
-
load_balancing? && config.
|
|
254
|
+
load_balancing? && config.baseline_proxy_hosts.include?(effective_loadbalancer)
|
|
255
255
|
end
|
|
256
256
|
|
|
257
257
|
def custom_ssl_certificate?
|
|
@@ -391,8 +391,11 @@ class Dash::Configuration::Proxy
|
|
|
391
391
|
.merge(target_options).compact
|
|
392
392
|
end
|
|
393
393
|
|
|
394
|
-
|
|
395
|
-
|
|
394
|
+
# dash-proxy takes a comma-separated target list, waits until every target is healthy,
|
|
395
|
+
# then swaps the whole pool and drains the one it replaced - which is how all replicas
|
|
396
|
+
# of a role on a host go live together.
|
|
397
|
+
def deploy_command_args(targets:)
|
|
398
|
+
optionize ({ target: target_list(targets) }).merge(deploy_options), with: "="
|
|
396
399
|
end
|
|
397
400
|
|
|
398
401
|
# dash-proxy rollout deploy only accepts the target and the timeouts - the service already
|
|
@@ -404,8 +407,8 @@ class Dash::Configuration::Proxy
|
|
|
404
407
|
}.compact
|
|
405
408
|
end
|
|
406
409
|
|
|
407
|
-
def rollout_deploy_command_args(
|
|
408
|
-
optionize ({ target:
|
|
410
|
+
def rollout_deploy_command_args(targets:)
|
|
411
|
+
optionize ({ target: target_list(targets) }).merge(rollout_deploy_options), with: "="
|
|
409
412
|
end
|
|
410
413
|
|
|
411
414
|
def rollout_set_command_args(percent: nil, list: nil)
|
|
@@ -428,6 +431,10 @@ class Dash::Configuration::Proxy
|
|
|
428
431
|
end
|
|
429
432
|
|
|
430
433
|
private
|
|
434
|
+
def target_list(targets)
|
|
435
|
+
Array(targets).map { |target| "#{target}:#{app_port}" }.join(",")
|
|
436
|
+
end
|
|
437
|
+
|
|
431
438
|
# Which dispositions this layer keeps. The per-app proxy behind a load
|
|
432
439
|
# balancer sheds the edge concerns; without load balancing there is no
|
|
433
440
|
# other layer to defer to. Dash::Configuration::Loadbalancer overrides
|
|
@@ -436,8 +443,9 @@ class Dash::Configuration::Proxy
|
|
|
436
443
|
load_balancing? ? %i[ per_app both ] : %i[ edge per_app both ]
|
|
437
444
|
end
|
|
438
445
|
|
|
446
|
+
# A baseline host: the load balancer must not move with the pool.
|
|
439
447
|
def primary_role_first_host
|
|
440
|
-
config.primary_role&.
|
|
448
|
+
config.primary_role&.baseline_hosts&.first
|
|
441
449
|
end
|
|
442
450
|
|
|
443
451
|
# Auto-activation needs a role the load balancer can actually front. The
|
|
@@ -445,10 +453,14 @@ class Dash::Configuration::Proxy
|
|
|
445
453
|
# Dash::Cli::Proxy#loadbalancer), so a proxy-less primary role would boot a
|
|
446
454
|
# load balancer with an empty --target. An explicit `loadbalancer:` setting
|
|
447
455
|
# skips this check — the operator asked for it.
|
|
456
|
+
#
|
|
457
|
+
# A scaled proxied role turns it on too, even with one baseline host: its members can
|
|
458
|
+
# only reach traffic through the load balancer's target list.
|
|
448
459
|
def auto_load_balanced_primary_role?
|
|
449
460
|
primary_role = config.primary_role
|
|
461
|
+
return false unless primary_role.present?
|
|
450
462
|
|
|
451
|
-
primary_role.
|
|
463
|
+
(primary_role.running_proxy? && primary_role.baseline_hosts.size > 1) || config.scaled_roles.any?(&:running_proxy?)
|
|
452
464
|
end
|
|
453
465
|
|
|
454
466
|
# Flags for dash-proxy's dynamic domain source (runtime TLS hostnames).
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# How a container of a role without the proxy leaves when it is scaled in: an optional
|
|
2
|
+
# signal (e.g. TSTP, so Sidekiq stops fetching), then a wait, then `docker stop`. A
|
|
3
|
+
# proxied role has none - dash-proxy drains it.
|
|
4
|
+
class Dash::Configuration::Role::Drain
|
|
5
|
+
# The Linux signal names `docker kill --signal` accepts, without the SIG prefix.
|
|
6
|
+
SIGNALS = %w[
|
|
7
|
+
HUP INT QUIT ILL TRAP ABRT IOT BUS FPE KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT CHLD CONT
|
|
8
|
+
STOP TSTP TTIN TTOU URG XCPU XFSZ VTALRM PROF WINCH IO POLL PWR SYS
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
# Linux signal numbers, SIGHUP through SIGRTMAX.
|
|
12
|
+
SIGNAL_NUMBERS = 1..64
|
|
13
|
+
|
|
14
|
+
attr_reader :context
|
|
15
|
+
|
|
16
|
+
def initialize(drain_config:, context:)
|
|
17
|
+
@drain_config = drain_config || {}
|
|
18
|
+
@context = context
|
|
19
|
+
validate!
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def signal
|
|
23
|
+
@drain_config["signal"]&.to_s
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def wait
|
|
27
|
+
@drain_config.fetch("wait", 0)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def configured?
|
|
31
|
+
@drain_config.present?
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
def validate!
|
|
36
|
+
raise Dash::ConfigurationError, "#{context}/wait: must be 0 or more seconds" if wait.negative?
|
|
37
|
+
return if signal.nil? || valid_signal?
|
|
38
|
+
|
|
39
|
+
raise Dash::ConfigurationError, "#{context}/signal: #{signal} is not a signal docker can send"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def valid_signal?
|
|
43
|
+
if signal.match?(/\A\d+\z/)
|
|
44
|
+
SIGNAL_NUMBERS.cover?(signal.to_i)
|
|
45
|
+
else
|
|
46
|
+
SIGNALS.include?(signal.upcase.delete_prefix("SIG"))
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# How many containers of one role run on each of its hosts. The bounds come from
|
|
2
|
+
# deploy.yml; the count in between is never stored - it is whatever is running, which
|
|
3
|
+
# `dash scale set` changes and a deploy reads back (see Dash::Cli::App::Boot).
|
|
4
|
+
class Dash::Configuration::Role::Replicas
|
|
5
|
+
# Docker options every replica on a host would claim for itself.
|
|
6
|
+
EXCLUSIVE_OPTIONS = %w[ publish p name hostname ]
|
|
7
|
+
VOLUME_OPTIONS = %w[ volume v mount volumes-from ]
|
|
8
|
+
|
|
9
|
+
attr_reader :min, :max, :context
|
|
10
|
+
|
|
11
|
+
def initialize(replicas_config:, context:)
|
|
12
|
+
@context = context
|
|
13
|
+
|
|
14
|
+
case replicas_config
|
|
15
|
+
when nil then @min = @max = 1
|
|
16
|
+
when Integer then @min = @max = replicas_config
|
|
17
|
+
else
|
|
18
|
+
@min = replicas_config.fetch("min", 1)
|
|
19
|
+
@max = replicas_config.fetch("max", [ @min, 1 ].max)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
validate!
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def scalable?
|
|
26
|
+
max > 1
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def numbers
|
|
30
|
+
(1..max).to_a
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# A deploy boots the running count, kept within the bounds: a crash cannot shrink the
|
|
34
|
+
# role below min, and a lowered max takes effect.
|
|
35
|
+
def clamp(running)
|
|
36
|
+
running.clamp(min, max)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def to_s
|
|
40
|
+
min == max ? "× #{max} #{"replica".pluralize(max)}" : "× #{min}–#{max} replicas"
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Checks that need the rest of the role (its docker options, its proxy), which is only
|
|
44
|
+
# complete once Dash::Configuration has finished building - see
|
|
45
|
+
# Dash::Configuration#ensure_replicas_fit_their_roles.
|
|
46
|
+
def ensure_fits!(role, volumes:)
|
|
47
|
+
return unless scalable?
|
|
48
|
+
|
|
49
|
+
if (option = role.docker_option_keys.find { |key| EXCLUSIVE_OPTIONS.include?(key) })
|
|
50
|
+
error "max > 1 cannot be combined with options/#{option}, every replica on a host would claim it"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
if role.running_proxy? && role.proxy.proxy_config.dig("sleep", "after").present?
|
|
54
|
+
error "max > 1 cannot be combined with proxy/sleep until dash-proxy is proven to sleep and wake every replica together"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
shared = volumes + role.docker_option_values(*VOLUME_OPTIONS)
|
|
58
|
+
shared.each do |volume|
|
|
59
|
+
warn "#{context}: every replica on a host shares the volume #{volume} - fine for read-only config or a shared cache, wrong for a per-process store"
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
def validate!
|
|
65
|
+
error "min must be at least 1" unless min >= 1
|
|
66
|
+
error "min (#{min}) cannot be greater than max (#{max})" if min > max
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def error(message)
|
|
70
|
+
raise Dash::ConfigurationError, "#{context}: #{message}"
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# One `scale.hold_when` entry: a saturation guard. While its query reads above `above` (or
|
|
2
|
+
# cannot be read), the controller holds a signal-driven scale-out - more containers would
|
|
3
|
+
# only queue on the database. A schedule floor still applies. See Dash::Autoscale::Guard.
|
|
4
|
+
class Dash::Configuration::Role::Scale::Guard
|
|
5
|
+
SOURCES = %w[ prometheus postgres ]
|
|
6
|
+
KEYS = %w[ source query above ]
|
|
7
|
+
|
|
8
|
+
attr_reader :source, :query, :above
|
|
9
|
+
|
|
10
|
+
def initialize(guard_config:, context:)
|
|
11
|
+
@context = context
|
|
12
|
+
error nil, "should be a hash" unless guard_config.is_a?(Hash)
|
|
13
|
+
if (unknown = (guard_config.keys.map(&:to_s) - KEYS).first)
|
|
14
|
+
error nil, "unknown key: #{unknown}"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
@source = guard_config["source"].to_s
|
|
18
|
+
@query = guard_config["query"]
|
|
19
|
+
@above = guard_config["above"]
|
|
20
|
+
|
|
21
|
+
error "source", "must be #{SOURCES.join(" or ")}, not #{source}" unless SOURCES.include?(source)
|
|
22
|
+
error "query", "is required" unless query.is_a?(String) && query.present?
|
|
23
|
+
error "above", "is required" if above.nil?
|
|
24
|
+
error "above", "should be a finite number" unless above.is_a?(Numeric) && above.real? && above.finite?
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# The autoscale/<connection> block the guard reads through.
|
|
28
|
+
def connection
|
|
29
|
+
source
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def to_h
|
|
33
|
+
{ source: source, query: query, above: above }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
def error(key, message)
|
|
38
|
+
raise Dash::ConfigurationError, "#{[ @context, key ].compact.join("/")}: #{message}"
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# The `scale.signal` entry: the one number the controller scales a role on. `sidekiq` and
|
|
2
|
+
# `pgbus` are the age in seconds of the oldest job waiting across `queues`; `prometheus` is
|
|
3
|
+
# whatever its `query` returns (the largest value of a vector). See Dash::Autoscale::Signal.
|
|
4
|
+
class Dash::Configuration::Role::Scale::Signal
|
|
5
|
+
SOURCES = %w[ sidekiq pgbus prometheus ]
|
|
6
|
+
KEYS = %w[ source queues prefix query ]
|
|
7
|
+
NAME = /\A[A-Za-z0-9_.:-]+\z/
|
|
8
|
+
DEFAULT_PGBUS_PREFIX = "pgbus_"
|
|
9
|
+
|
|
10
|
+
# The autoscale/<connection> block each source reads through.
|
|
11
|
+
CONNECTIONS = { "sidekiq" => "redis", "pgbus" => "postgres", "prometheus" => "prometheus" }.freeze
|
|
12
|
+
|
|
13
|
+
attr_reader :source, :queues, :prefix, :query
|
|
14
|
+
|
|
15
|
+
def initialize(signal_config:, context:)
|
|
16
|
+
@context = context
|
|
17
|
+
error nil, "should be a hash" unless signal_config.is_a?(Hash)
|
|
18
|
+
if (unknown = (signal_config.keys.map(&:to_s) - KEYS).first)
|
|
19
|
+
error nil, "unknown key: #{unknown}"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
@source = signal_config["source"].to_s
|
|
23
|
+
validate_source!
|
|
24
|
+
@queues = parse_queues(signal_config["queues"])
|
|
25
|
+
@prefix = parse_prefix(signal_config["prefix"])
|
|
26
|
+
@query = parse_query(signal_config["query"])
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def queues?
|
|
30
|
+
source != "prometheus"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def connection
|
|
34
|
+
CONNECTIONS.fetch(source)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def to_h
|
|
38
|
+
{ source: source, queues: queues, prefix: prefix, query: query }.compact
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
def validate_source!
|
|
43
|
+
if source == "proxy"
|
|
44
|
+
error "source", "proxy (web queue time) comes with Phase 4 (zoolutions/dash#180), use #{SOURCES.to_sentence(last_word_connector: " or ")}"
|
|
45
|
+
end
|
|
46
|
+
error "source", "must be #{SOURCES.to_sentence(last_word_connector: " or ")}, not #{source}" unless SOURCES.include?(source)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def parse_queues(queues)
|
|
50
|
+
unless queues?
|
|
51
|
+
error "queues", "is only used with sidekiq and pgbus" unless queues.nil?
|
|
52
|
+
return
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
error "queues", "is required for #{source}" if queues.nil? || queues == []
|
|
56
|
+
error "queues", "should be an array" unless queues.is_a?(Array)
|
|
57
|
+
queues.each_with_index { |queue, index| error "queues/#{index}", "#{queue.inspect} #{name_rule}" unless name?(queue) }
|
|
58
|
+
queues
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def parse_prefix(prefix)
|
|
62
|
+
unless source == "pgbus"
|
|
63
|
+
error "prefix", "is only used with pgbus" unless prefix.nil?
|
|
64
|
+
return
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
return DEFAULT_PGBUS_PREFIX if prefix.nil?
|
|
68
|
+
error "prefix", "#{prefix.inspect} #{name_rule}" unless name?(prefix)
|
|
69
|
+
prefix
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def parse_query(query)
|
|
73
|
+
if queues?
|
|
74
|
+
error "query", "is only used with prometheus" unless query.nil?
|
|
75
|
+
return
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
error "query", "is required for prometheus" unless query.is_a?(String) && query.present?
|
|
79
|
+
query
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def name?(value)
|
|
83
|
+
value.is_a?(String) && NAME.match?(value)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def name_rule
|
|
87
|
+
"should be letters, digits, _ . : or -"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def error(key, message)
|
|
91
|
+
raise Dash::ConfigurationError, "#{[ @context, key ].compact.join("/")}: #{message}"
|
|
92
|
+
end
|
|
93
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# One `scale.schedule` entry: from every minute `cron` matches, for `duration` seconds, the
|
|
2
|
+
# role runs at least `min` containers. Times are handed in already in `autoscale.timezone`,
|
|
3
|
+
# so the cron is wall-clock there while the duration is real seconds (a window that spans a
|
|
4
|
+
# DST change still lasts `for`).
|
|
5
|
+
class Dash::Configuration::Role::Scale::Window
|
|
6
|
+
attr_reader :cron, :duration, :min
|
|
7
|
+
|
|
8
|
+
def initialize(cron:, duration:, min:)
|
|
9
|
+
@cron = cron
|
|
10
|
+
@duration = duration
|
|
11
|
+
@min = min
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def active_at?(now)
|
|
15
|
+
!started_at(now).nil?
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# The latest minute in (now - duration, now] the cron matches, scanned backwards: at most
|
|
19
|
+
# 10 080 steps for the 7-day limit.
|
|
20
|
+
def started_at(now)
|
|
21
|
+
earliest = now - duration
|
|
22
|
+
minute = now.change(sec: 0)
|
|
23
|
+
|
|
24
|
+
while minute > earliest
|
|
25
|
+
return minute if cron.match?(minute)
|
|
26
|
+
minute -= 60
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def ends_at(now)
|
|
31
|
+
started_at(now)&.+(duration)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def to_h
|
|
35
|
+
{ cron: cron.to_s, for: duration, min: min }
|
|
36
|
+
end
|
|
37
|
+
end
|