mrsk 0.11.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +83 -11
- data/lib/mrsk/cli/app.rb +62 -20
- data/lib/mrsk/cli/base.rb +36 -17
- data/lib/mrsk/cli/build.rb +23 -1
- data/lib/mrsk/cli/healthcheck.rb +4 -34
- data/lib/mrsk/cli/main.rb +60 -30
- data/lib/mrsk/cli/prune.rb +2 -2
- data/lib/mrsk/cli/server.rb +13 -9
- data/lib/mrsk/cli/traefik.rb +1 -1
- data/lib/mrsk/commander.rb +26 -5
- data/lib/mrsk/commands/app.rb +25 -12
- data/lib/mrsk/commands/auditor.rb +22 -20
- data/lib/mrsk/commands/base.rb +3 -0
- data/lib/mrsk/commands/builder/base.rb +9 -2
- data/lib/mrsk/commands/builder.rb +21 -1
- data/lib/mrsk/commands/docker.rb +21 -0
- data/lib/mrsk/commands/healthcheck.rb +7 -2
- data/lib/mrsk/commands/lock.rb +2 -2
- data/lib/mrsk/commands/prune.rb +12 -4
- data/lib/mrsk/configuration/boot.rb +20 -0
- data/lib/mrsk/configuration/role.rb +22 -2
- data/lib/mrsk/configuration.rb +4 -0
- data/lib/mrsk/sshkit_with_ext.rb +42 -2
- data/lib/mrsk/utils/healthcheck_poller.rb +39 -0
- data/lib/mrsk/utils.rb +5 -1
- data/lib/mrsk/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '099a4dc2dc59df4e0c5301b85a579970dbc6c46a3c1e2a634f4461c5cff1f241'
|
4
|
+
data.tar.gz: 0a0356837992a9847b7b6bc51956c26a4fb0e8fbb9809753a5bdb373bacd3aee
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 06f1365b5f8a7cc2064f2bd184224dba12b1c0ebf57cc502c9ac423a9d0ae96a601161fe681764f9a660b8a6214f2a86b79d1c4d46d9ff13819c52159db14318
|
7
|
+
data.tar.gz: 7acf9c5d2e26709b391a2c9915300c569fd6cc7d841b55cd3a497122d2d2317b8d2b3df4e634f14c38acd571fbc8aac51096a221f10097f8ce68d6c07dbce038
|
data/README.md
CHANGED
@@ -308,7 +308,7 @@ You can specialize the default Traefik rules by setting labels on the containers
|
|
308
308
|
labels:
|
309
309
|
traefik.http.routers.hey-web.rule: Host(`app.hey.com`)
|
310
310
|
```
|
311
|
-
Traefik rules are in the "service-role-destination" format. The default role will be `web` if no rule is specified. If the destination is not specified, it is not included. To give an example, the above rule would become "traefik.http.routers.hey-web.rule" if it was for the "staging" destination.
|
311
|
+
Traefik rules are in the "service-role-destination" format. The default role will be `web` if no rule is specified. If the destination is not specified, it is not included. To give an example, the above rule would become "traefik.http.routers.hey-web-staging.rule" if it was for the "staging" destination.
|
312
312
|
|
313
313
|
Note: The backticks are needed to ensure the rule is passed in correctly and not treated as command substitution by Bash!
|
314
314
|
|
@@ -331,6 +331,21 @@ servers:
|
|
331
331
|
my-label: "50"
|
332
332
|
```
|
333
333
|
|
334
|
+
### Using shell expansion
|
335
|
+
|
336
|
+
You can use shell expansion to interpolate values from the host machine into labels and env variables with the `${}` syntax.
|
337
|
+
Anything within the curly braces will be executed on the host machine and the result will be interpolated into the label or env variable.
|
338
|
+
|
339
|
+
```yaml
|
340
|
+
labels:
|
341
|
+
host-machine: "${cat /etc/hostname}"
|
342
|
+
|
343
|
+
env:
|
344
|
+
HOST_DEPLOYMENT_DIR: "${PWD}"
|
345
|
+
```
|
346
|
+
|
347
|
+
Note: Any other occurrence of `$` will be escaped to prevent unwanted shell expansion!
|
348
|
+
|
334
349
|
### Using container options
|
335
350
|
|
336
351
|
You can specialize the options used to start containers using the `options` definitions:
|
@@ -514,18 +529,18 @@ traefik:
|
|
514
529
|
|
515
530
|
This starts the Traefik container with `--volume /tmp/example.json:/tmp/example.json --publish 8080:8080 --memory 512m` arguments to `docker run`.
|
516
531
|
|
517
|
-
### Traefik container
|
532
|
+
### Traefik container labels
|
518
533
|
|
519
534
|
Add labels to Traefik Docker container.
|
520
535
|
|
521
536
|
```yaml
|
522
537
|
traefik:
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
538
|
+
labels:
|
539
|
+
traefik.enable: true
|
540
|
+
traefik.http.routers.dashboard.rule: Host(`traefik.example.com`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))
|
541
|
+
traefik.http.routers.dashboard.service: api@internal
|
542
|
+
traefik.http.routers.dashboard.middlewares: auth
|
543
|
+
traefik.http.middlewares.auth.basicauth.users: test:$2y$05$H2o72tMaO.TwY1wNQUV1K.fhjRgLHRDWohFvUZOJHBEtUXNKrqUKi # test:password
|
529
544
|
```
|
530
545
|
|
531
546
|
This labels Traefik container with `--label traefik.http.routers.dashboard.middlewares=\"auth\"` and so on.
|
@@ -662,9 +677,26 @@ That'll post a line like follows to a preconfigured chatbot in Basecamp:
|
|
662
677
|
[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
|
663
678
|
```
|
664
679
|
|
665
|
-
|
680
|
+
`MRSK_*` environment variables are available to the broadcast command for
|
681
|
+
fine-grained audit reporting, e.g. for triggering deployment reports or
|
682
|
+
firing a JSON webhook. These variables include:
|
683
|
+
- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
|
684
|
+
- `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
|
685
|
+
- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
|
686
|
+
- `MRSK_DESTINATION` - optional: destination, e.g. "staging"
|
687
|
+
- `MRSK_ROLE` - optional: role targeted, e.g. "web"
|
666
688
|
|
667
|
-
|
689
|
+
Use `mrsk broadcast` to test and troubleshoot your broadcast command:
|
690
|
+
|
691
|
+
```bash
|
692
|
+
mrsk broadcast -m "test audit message"
|
693
|
+
```
|
694
|
+
|
695
|
+
### Healthcheck
|
696
|
+
|
697
|
+
MRSK uses Docker healtchecks to check the health of your application during deployment. Traefik uses this same healthcheck status to determine when a container is ready to receive traffic.
|
698
|
+
|
699
|
+
The healthcheck defaults to testing the HTTP response to the path `/up` on port 3000, up to 7 times. You can tailor this behaviour with the `healthcheck` setting:
|
668
700
|
|
669
701
|
```yaml
|
670
702
|
healthcheck:
|
@@ -675,7 +707,29 @@ healthcheck:
|
|
675
707
|
|
676
708
|
This will ensure your application is configured with a traefik label for the healthcheck against `/healthz` and that the pre-deploy healthcheck that MRSK performs is done against the same path on port 4000.
|
677
709
|
|
678
|
-
|
710
|
+
You can also specify a custom healthcheck command, which is useful for non-HTTP services:
|
711
|
+
|
712
|
+
```yaml
|
713
|
+
healthcheck:
|
714
|
+
cmd: /bin/check_health
|
715
|
+
```
|
716
|
+
|
717
|
+
The top-level healthcheck configuration applies to all services that use
|
718
|
+
Traefik, by default. You can also specialize the configuration at the role
|
719
|
+
level:
|
720
|
+
|
721
|
+
```yaml
|
722
|
+
servers:
|
723
|
+
job:
|
724
|
+
hosts: ...
|
725
|
+
cmd: bin/jobs
|
726
|
+
healthcheck:
|
727
|
+
cmd: bin/check
|
728
|
+
```
|
729
|
+
|
730
|
+
The healthcheck allows for an optional `max_attempts` setting, which will attempt the healthcheck up to the specified number of times before failing the deploy. This is useful for applications that take a while to start up. The default is 7.
|
731
|
+
|
732
|
+
Note: The HTTP health checks assume that the `curl` command is available inside the container. If that's not the case, use the healthcheck's `cmd` option to specify an alternative check that the container supports.
|
679
733
|
|
680
734
|
## Commands
|
681
735
|
|
@@ -816,6 +870,24 @@ mrsk lock acquire -m "Doing maintanence"
|
|
816
870
|
mrsk lock release
|
817
871
|
```
|
818
872
|
|
873
|
+
## Rolling deployments
|
874
|
+
|
875
|
+
When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time.
|
876
|
+
|
877
|
+
MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `boot/limit` and `boot/wait` as options:
|
878
|
+
|
879
|
+
```yaml
|
880
|
+
service: myservice
|
881
|
+
|
882
|
+
boot:
|
883
|
+
limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
|
884
|
+
wait: 2
|
885
|
+
```
|
886
|
+
|
887
|
+
When `limit` is specified, containers will be booted on, at most, `limit` hosts at once. MRSK will pause for `wait` seconds between batches.
|
888
|
+
|
889
|
+
These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts.
|
890
|
+
|
819
891
|
## Stage of development
|
820
892
|
|
821
893
|
This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).
|
data/lib/mrsk/cli/app.rb
CHANGED
@@ -6,27 +6,33 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
|
|
6
6
|
using_version(version_or_latest) do |version|
|
7
7
|
say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
|
8
8
|
|
9
|
-
|
9
|
+
on(MRSK.hosts) do
|
10
|
+
execute *MRSK.auditor.record("Tagging #{MRSK.config.absolute_image} as the latest image"), verbosity: :debug
|
11
|
+
execute *MRSK.app.tag_current_as_latest
|
12
|
+
end
|
10
13
|
|
11
|
-
on(MRSK.hosts) do |host|
|
14
|
+
on(MRSK.hosts, **MRSK.boot_strategy) do |host|
|
12
15
|
roles = MRSK.roles_on(host)
|
13
16
|
|
14
17
|
roles.each do |role|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
|
26
|
-
execute *MRSK.app(role: role).run
|
27
|
-
sleep MRSK.config.readiness_delay
|
28
|
-
execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
|
18
|
+
app = MRSK.app(role: role)
|
19
|
+
auditor = MRSK.auditor(role: role)
|
20
|
+
|
21
|
+
execute *auditor.record("Booted app version #{version}"), verbosity: :debug
|
22
|
+
|
23
|
+
if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
|
24
|
+
tmp_version = "#{version}_#{SecureRandom.hex(8)}"
|
25
|
+
info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
|
26
|
+
execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
|
27
|
+
execute *app.rename_container(version: version, new_version: tmp_version)
|
29
28
|
end
|
29
|
+
|
30
|
+
old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
|
31
|
+
execute *app.run
|
32
|
+
|
33
|
+
Mrsk::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
|
34
|
+
|
35
|
+
execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
|
30
36
|
end
|
31
37
|
end
|
32
38
|
end
|
@@ -54,7 +60,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
|
|
54
60
|
roles = MRSK.roles_on(host)
|
55
61
|
|
56
62
|
roles.each do |role|
|
57
|
-
execute *MRSK.auditor
|
63
|
+
execute *MRSK.auditor.record("Stopped app", role: role), verbosity: :debug
|
58
64
|
execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
|
59
65
|
end
|
60
66
|
end
|
@@ -101,7 +107,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
|
|
101
107
|
roles = MRSK.roles_on(host)
|
102
108
|
|
103
109
|
roles.each do |role|
|
104
|
-
execute *MRSK.auditor
|
110
|
+
execute *MRSK.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
|
105
111
|
puts_by_host host, capture_with_info(*MRSK.app(role: role).execute_in_existing_container(cmd))
|
106
112
|
end
|
107
113
|
end
|
@@ -124,6 +130,31 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
|
|
124
130
|
on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_containers) }
|
125
131
|
end
|
126
132
|
|
133
|
+
desc "stale_containers", "Detect app stale containers"
|
134
|
+
option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
|
135
|
+
def stale_containers
|
136
|
+
with_lock do
|
137
|
+
stop = options[:stop]
|
138
|
+
|
139
|
+
cli = self
|
140
|
+
|
141
|
+
on(MRSK.hosts) do |host|
|
142
|
+
roles = MRSK.roles_on(host)
|
143
|
+
|
144
|
+
roles.each do |role|
|
145
|
+
cli.send(:stale_versions, host: host, role: role).each do |version|
|
146
|
+
if stop
|
147
|
+
puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
|
148
|
+
execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
|
149
|
+
else
|
150
|
+
puts_by_host host, "Detected stale container for role #{role} with version #{version} (use `mrsk app stale_containers --stop` to stop)"
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
127
158
|
desc "images", "Show app images on servers"
|
128
159
|
def images
|
129
160
|
on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_images) }
|
@@ -183,7 +214,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
|
|
183
214
|
roles = MRSK.roles_on(host)
|
184
215
|
|
185
216
|
roles.each do |role|
|
186
|
-
execute *MRSK.auditor
|
217
|
+
execute *MRSK.auditor.record("Removed app container with version #{version}", role: role), verbosity: :debug
|
187
218
|
execute *MRSK.app(role: role).remove_container(version: version)
|
188
219
|
end
|
189
220
|
end
|
@@ -197,7 +228,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
|
|
197
228
|
roles = MRSK.roles_on(host)
|
198
229
|
|
199
230
|
roles.each do |role|
|
200
|
-
execute *MRSK.auditor
|
231
|
+
execute *MRSK.auditor.record("Removed all app containers", role: role), verbosity: :debug
|
201
232
|
execute *MRSK.app(role: role).remove_containers
|
202
233
|
end
|
203
234
|
end
|
@@ -240,6 +271,17 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
|
|
240
271
|
version.presence
|
241
272
|
end
|
242
273
|
|
274
|
+
def stale_versions(host:, role:)
|
275
|
+
versions = nil
|
276
|
+
on(host) do
|
277
|
+
versions = \
|
278
|
+
capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
|
279
|
+
.split("\n")
|
280
|
+
.drop(1)
|
281
|
+
end
|
282
|
+
versions
|
283
|
+
end
|
284
|
+
|
243
285
|
def version_or_latest
|
244
286
|
options[:version] || "latest"
|
245
287
|
end
|
data/lib/mrsk/cli/base.rb
CHANGED
@@ -77,25 +77,35 @@ module Mrsk::Cli
|
|
77
77
|
end
|
78
78
|
|
79
79
|
def with_lock
|
80
|
-
|
81
|
-
|
82
|
-
|
80
|
+
if MRSK.holding_lock?
|
81
|
+
yield
|
82
|
+
else
|
83
|
+
acquire_lock
|
84
|
+
|
85
|
+
begin
|
86
|
+
yield
|
87
|
+
rescue
|
88
|
+
if MRSK.hold_lock_on_error?
|
89
|
+
error " \e[31mDeploy lock was not released\e[0m"
|
90
|
+
else
|
91
|
+
release_lock
|
92
|
+
end
|
93
|
+
|
94
|
+
raise
|
95
|
+
end
|
83
96
|
|
84
|
-
|
85
|
-
|
86
|
-
error " \e[31mDeploy lock was not released\e[0m" if MRSK.lock_count > 0
|
87
|
-
raise
|
97
|
+
release_lock
|
98
|
+
end
|
88
99
|
end
|
89
100
|
|
90
101
|
def acquire_lock
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
MRSK.lock_count += 1
|
102
|
+
say "Acquiring the deploy lock"
|
103
|
+
on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
|
104
|
+
|
105
|
+
MRSK.holding_lock = true
|
96
106
|
rescue SSHKit::Runner::ExecuteError => e
|
97
107
|
if e.message =~ /cannot create directory/
|
98
|
-
|
108
|
+
on(MRSK.primary_host) { execute *MRSK.lock.status }
|
99
109
|
raise LockError, "Deploy lock found"
|
100
110
|
else
|
101
111
|
raise e
|
@@ -103,10 +113,19 @@ module Mrsk::Cli
|
|
103
113
|
end
|
104
114
|
|
105
115
|
def release_lock
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
116
|
+
say "Releasing the deploy lock"
|
117
|
+
on(MRSK.primary_host) { execute *MRSK.lock.release }
|
118
|
+
|
119
|
+
MRSK.holding_lock = false
|
120
|
+
end
|
121
|
+
|
122
|
+
def hold_lock_on_error
|
123
|
+
if MRSK.hold_lock_on_error?
|
124
|
+
yield
|
125
|
+
else
|
126
|
+
MRSK.hold_lock_on_error = true
|
127
|
+
yield
|
128
|
+
MRSK.hold_lock_on_error = false
|
110
129
|
end
|
111
130
|
end
|
112
131
|
end
|
data/lib/mrsk/cli/build.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
class Mrsk::Cli::Build < Mrsk::Cli::Base
|
2
|
+
class BuildError < StandardError; end
|
3
|
+
|
2
4
|
desc "deliver", "Build app and push app image to registry then pull image on servers"
|
3
5
|
def deliver
|
4
6
|
with_lock do
|
@@ -14,7 +16,9 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
|
|
14
16
|
|
15
17
|
run_locally do
|
16
18
|
begin
|
17
|
-
|
19
|
+
if cli.verify_local_dependencies
|
20
|
+
MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
|
21
|
+
end
|
18
22
|
rescue SSHKit::Command::Failed => e
|
19
23
|
if e.message =~ /(no builder)|(no such file or directory)/
|
20
24
|
error "Missing compatible builder, so creating a new one first"
|
@@ -77,4 +81,22 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
|
|
77
81
|
puts capture(*MRSK.builder.info)
|
78
82
|
end
|
79
83
|
end
|
84
|
+
|
85
|
+
|
86
|
+
desc "", "" # Really a private method, but needed to be invoked from #push
|
87
|
+
def verify_local_dependencies
|
88
|
+
run_locally do
|
89
|
+
begin
|
90
|
+
execute *MRSK.builder.ensure_local_dependencies_installed
|
91
|
+
rescue SSHKit::Command::Failed => e
|
92
|
+
build_error = e.message =~ /command not found/ ?
|
93
|
+
"Docker is not installed locally" :
|
94
|
+
"Docker buildx plugin is not installed locally"
|
95
|
+
|
96
|
+
raise BuildError, build_error
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
true
|
101
|
+
end
|
80
102
|
end
|
data/lib/mrsk/cli/healthcheck.rb
CHANGED
@@ -1,7 +1,4 @@
|
|
1
1
|
class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base
|
2
|
-
|
3
|
-
class HealthcheckError < StandardError; end
|
4
|
-
|
5
2
|
default_command :perform
|
6
3
|
|
7
4
|
desc "perform", "Health check current app version"
|
@@ -9,38 +6,11 @@ class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base
|
|
9
6
|
on(MRSK.primary_host) do
|
10
7
|
begin
|
11
8
|
execute *MRSK.healthcheck.run
|
12
|
-
|
13
|
-
|
14
|
-
attempt = 1
|
15
|
-
max_attempts = MRSK.config.healthcheck["max_attempts"]
|
16
|
-
|
17
|
-
begin
|
18
|
-
status = capture_with_info(*MRSK.healthcheck.curl)
|
19
|
-
|
20
|
-
if status == "200"
|
21
|
-
info "#{target} succeeded with 200 OK!"
|
22
|
-
else
|
23
|
-
raise HealthcheckError, "#{target} failed with status #{status}"
|
24
|
-
end
|
25
|
-
rescue SSHKit::Command::Failed
|
26
|
-
if attempt <= max_attempts
|
27
|
-
info "#{target} failed to respond, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
|
28
|
-
sleep attempt
|
29
|
-
attempt += 1
|
30
|
-
|
31
|
-
retry
|
32
|
-
else
|
33
|
-
raise
|
34
|
-
end
|
35
|
-
end
|
36
|
-
rescue SSHKit::Command::Failed, HealthcheckError => e
|
9
|
+
Mrsk::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*MRSK.healthcheck.status) }
|
10
|
+
rescue Mrsk::Utils::HealthcheckPoller::HealthcheckError => e
|
37
11
|
error capture_with_info(*MRSK.healthcheck.logs)
|
38
|
-
|
39
|
-
|
40
|
-
raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!"
|
41
|
-
else
|
42
|
-
raise
|
43
|
-
end
|
12
|
+
error capture_with_pretty_json(*MRSK.healthcheck.container_health_log)
|
13
|
+
raise
|
44
14
|
ensure
|
45
15
|
execute *MRSK.healthcheck.stop, raise_on_non_zero_exit: false
|
46
16
|
execute *MRSK.healthcheck.remove, raise_on_non_zero_exit: false
|
data/lib/mrsk/cli/main.rb
CHANGED
@@ -17,9 +17,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
|
|
17
17
|
invoke_options = deploy_options
|
18
18
|
|
19
19
|
runtime = print_runtime do
|
20
|
-
say "Ensure curl and Docker are installed...", :magenta
|
21
|
-
invoke "mrsk:cli:server:bootstrap", [], invoke_options
|
22
|
-
|
23
20
|
say "Log into image registry...", :magenta
|
24
21
|
invoke "mrsk:cli:registry:login", [], invoke_options
|
25
22
|
|
@@ -37,7 +34,12 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
|
|
37
34
|
say "Ensure app can pass healthcheck...", :magenta
|
38
35
|
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
|
39
36
|
|
40
|
-
|
37
|
+
say "Detect stale containers...", :magenta
|
38
|
+
invoke "mrsk:cli:app:stale_containers", [], invoke_options
|
39
|
+
|
40
|
+
hold_lock_on_error do
|
41
|
+
invoke "mrsk:cli:app:boot", [], invoke_options
|
42
|
+
end
|
41
43
|
|
42
44
|
say "Prune old containers and images...", :magenta
|
43
45
|
invoke "mrsk:cli:prune:all", [], invoke_options
|
@@ -65,7 +67,12 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
|
|
65
67
|
say "Ensure app can pass healthcheck...", :magenta
|
66
68
|
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
|
67
69
|
|
68
|
-
|
70
|
+
say "Detect stale containers...", :magenta
|
71
|
+
invoke "mrsk:cli:app:stale_containers", [], invoke_options
|
72
|
+
|
73
|
+
hold_lock_on_error do
|
74
|
+
invoke "mrsk:cli:app:boot", [], invoke_options
|
75
|
+
end
|
69
76
|
end
|
70
77
|
|
71
78
|
audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
|
@@ -75,34 +82,41 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
|
|
75
82
|
desc "rollback [VERSION]", "Rollback app to VERSION"
|
76
83
|
def rollback(version)
|
77
84
|
with_lock do
|
78
|
-
|
79
|
-
|
80
|
-
if container_available?(version)
|
81
|
-
say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
|
85
|
+
invoke_options = deploy_options
|
82
86
|
|
83
|
-
|
87
|
+
hold_lock_on_error do
|
88
|
+
MRSK.config.version = version
|
84
89
|
old_version = nil
|
85
90
|
|
86
|
-
|
87
|
-
|
91
|
+
if container_available?(version)
|
92
|
+
say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
|
93
|
+
|
94
|
+
on(MRSK.hosts) do
|
95
|
+
execute *MRSK.auditor.record("Tagging #{MRSK.config.absolute_image} as the latest image"), verbosity: :debug
|
96
|
+
execute *MRSK.app.tag_current_as_latest
|
97
|
+
end
|
88
98
|
|
89
|
-
|
90
|
-
|
91
|
-
old_version = capture_with_info(*app.current_running_version).strip.presence
|
99
|
+
on(MRSK.hosts) do |host|
|
100
|
+
roles = MRSK.roles_on(host)
|
92
101
|
|
93
|
-
|
102
|
+
roles.each do |role|
|
103
|
+
app = MRSK.app(role: role)
|
104
|
+
old_version = capture_with_info(*app.current_running_version).strip.presence
|
94
105
|
|
95
|
-
|
96
|
-
sleep MRSK.config.readiness_delay
|
106
|
+
execute *app.start
|
97
107
|
|
98
|
-
|
108
|
+
if old_version
|
109
|
+
sleep MRSK.config.readiness_delay
|
110
|
+
|
111
|
+
execute *app.stop(version: old_version), raise_on_non_zero_exit: false
|
112
|
+
end
|
99
113
|
end
|
100
114
|
end
|
101
|
-
end
|
102
115
|
|
103
|
-
|
104
|
-
|
105
|
-
|
116
|
+
audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
|
117
|
+
else
|
118
|
+
say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
|
119
|
+
end
|
106
120
|
end
|
107
121
|
end
|
108
122
|
end
|
@@ -186,6 +200,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
|
|
186
200
|
end
|
187
201
|
end
|
188
202
|
|
203
|
+
desc "broadcast", "Broadcast an audit message"
|
204
|
+
option :message, aliases: "-m", type: :string, desc: "Audit mesasge", required: true
|
205
|
+
def broadcast
|
206
|
+
say "Broadcast: #{options[:message]}", :magenta
|
207
|
+
audit_broadcast options[:message]
|
208
|
+
end
|
209
|
+
|
189
210
|
desc "version", "Show MRSK version"
|
190
211
|
def version
|
191
212
|
puts Mrsk::VERSION
|
@@ -219,15 +240,24 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
|
|
219
240
|
subcommand "lock", Mrsk::Cli::Lock
|
220
241
|
|
221
242
|
private
|
222
|
-
def container_available?(version
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
243
|
+
def container_available?(version)
|
244
|
+
begin
|
245
|
+
on(MRSK.hosts) do
|
246
|
+
MRSK.roles_on(host).each do |role|
|
247
|
+
container_id = capture_with_info(*MRSK.app(role: role).container_id_for_version(version))
|
248
|
+
raise "Container not found" unless container_id.present?
|
249
|
+
end
|
250
|
+
end
|
251
|
+
rescue SSHKit::Runner::ExecuteError => e
|
252
|
+
if e.message =~ /Container not found/
|
253
|
+
say "Error looking for container version #{version}: #{e.message}"
|
254
|
+
return false
|
255
|
+
else
|
256
|
+
raise
|
257
|
+
end
|
228
258
|
end
|
229
259
|
|
230
|
-
|
260
|
+
true
|
231
261
|
end
|
232
262
|
|
233
263
|
def deploy_options
|
data/lib/mrsk/cli/prune.rb
CHANGED
@@ -7,7 +7,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
|
|
7
7
|
end
|
8
8
|
end
|
9
9
|
|
10
|
-
desc "images", "Prune
|
10
|
+
desc "images", "Prune dangling images"
|
11
11
|
def images
|
12
12
|
with_lock do
|
13
13
|
on(MRSK.hosts) do
|
@@ -17,7 +17,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
|
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
-
desc "containers", "Prune stopped containers
|
20
|
+
desc "containers", "Prune all stopped containers, except the last 5"
|
21
21
|
def containers
|
22
22
|
with_lock do
|
23
23
|
on(MRSK.hosts) do
|
data/lib/mrsk/cli/server.rb
CHANGED
@@ -1,17 +1,21 @@
|
|
1
1
|
class Mrsk::Cli::Server < Mrsk::Cli::Base
|
2
|
-
desc "bootstrap", "
|
2
|
+
desc "bootstrap", "Set up Docker to run MRSK apps"
|
3
3
|
def bootstrap
|
4
|
-
|
5
|
-
on(MRSK.hosts + MRSK.accessory_hosts) do
|
6
|
-
dependencies_to_install = Array.new.tap do |dependencies|
|
7
|
-
dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false
|
8
|
-
dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false
|
9
|
-
end
|
4
|
+
missing = []
|
10
5
|
|
11
|
-
|
12
|
-
|
6
|
+
on(MRSK.hosts | MRSK.accessory_hosts) do |host|
|
7
|
+
unless execute(*MRSK.docker.installed?, raise_on_non_zero_exit: false)
|
8
|
+
if execute(*MRSK.docker.superuser?, raise_on_non_zero_exit: false)
|
9
|
+
info "Missing Docker on #{host}. Installing…"
|
10
|
+
execute *MRSK.docker.install
|
11
|
+
else
|
12
|
+
missing << host
|
13
13
|
end
|
14
14
|
end
|
15
15
|
end
|
16
|
+
|
17
|
+
if missing.any?
|
18
|
+
raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/"
|
19
|
+
end
|
16
20
|
end
|
17
21
|
end
|
data/lib/mrsk/cli/traefik.rb
CHANGED
@@ -94,7 +94,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
|
|
94
94
|
end
|
95
95
|
end
|
96
96
|
|
97
|
-
desc "
|
97
|
+
desc "remove_image", "Remove Traefik image from servers", hide: true
|
98
98
|
def remove_image
|
99
99
|
with_lock do
|
100
100
|
on(MRSK.traefik_hosts) do
|