odysseus-core 0.3.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +485 -1
- data/LICENSE.txt +25 -7
- data/README.md +12 -3
- data/lib/odysseus/builder/client.rb +15 -15
- data/lib/odysseus/caddy/client.rb +119 -42
- data/lib/odysseus/command_redaction.rb +38 -0
- data/lib/odysseus/config/parser.rb +43 -18
- data/lib/odysseus/core/deploy_versioning.rb +42 -0
- data/lib/odysseus/core/environment.rb +63 -0
- data/lib/odysseus/core/version.rb +1 -1
- data/lib/odysseus/core/volume_namespacer.rb +0 -0
- data/lib/odysseus/core.rb +2 -2
- data/lib/odysseus/deploy_log.rb +120 -0
- data/lib/odysseus/deploy_version.rb +9 -0
- data/lib/odysseus/deployer/dependency_manager.rb +121 -0
- data/lib/odysseus/deployer/executor.rb +213 -161
- data/lib/odysseus/deployer/retention_sweeper.rb +94 -0
- data/lib/odysseus/deployer/ssh.rb +92 -18
- data/lib/odysseus/docker/client.rb +240 -48
- data/lib/odysseus/docker/labels.rb +43 -0
- data/lib/odysseus/errors.rb +10 -0
- data/lib/odysseus/git.rb +70 -0
- data/lib/odysseus/host_paths.rb +79 -0
- data/lib/odysseus/host_providers/base.rb +1 -1
- data/lib/odysseus/host_providers/static.rb +1 -1
- data/lib/odysseus/host_providers.rb +3 -4
- data/lib/odysseus/host_verifier.rb +156 -0
- data/lib/odysseus/host_versions.rb +59 -0
- data/lib/odysseus/orchestrator/{accessory_deploy.rb → dependency_deploy.rb} +84 -88
- data/lib/odysseus/orchestrator/job_deploy.rb +31 -43
- data/lib/odysseus/orchestrator/web_deploy.rb +61 -55
- data/lib/odysseus/plugins.rb +72 -0
- data/lib/odysseus/retention_plan.rb +14 -0
- data/lib/odysseus/retention_planner.rb +58 -0
- data/lib/odysseus/rollback_plan.rb +20 -0
- data/lib/odysseus/rollback_planner.rb +107 -0
- data/lib/odysseus/sails.rb +0 -0
- data/lib/odysseus/secrets/encrypted_file.rb +5 -7
- data/lib/odysseus/secrets/loader.rb +1 -3
- data/lib/odysseus/setup/docker_apt.rb +178 -0
- data/lib/odysseus/setup/escalation.rb +73 -0
- data/lib/odysseus/setup/preparer.rb +372 -0
- data/lib/odysseus/setup/public_key.rb +127 -0
- data/lib/odysseus/validators/config.rb +38 -25
- data/lib/odysseus/version_resolver.rb +80 -0
- data/lib/odysseus.rb +0 -0
- data/sig/odysseus/core.rbs +0 -0
- metadata +48 -27
- data/Rakefile +0 -12
- data/lib/odysseus/version.rb +0 -5
|
@@ -22,24 +22,56 @@ module Odysseus
|
|
|
22
22
|
@session = nil
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
+
# The user this connection authenticates as. Read by HostPaths, which
|
|
26
|
+
# puts host state under /var/lib for root and under $HOME for anyone
|
|
27
|
+
# else.
|
|
28
|
+
attr_reader :user
|
|
29
|
+
|
|
30
|
+
# Host and port a caller needs to open an equivalent second connection
|
|
31
|
+
# (Setup::Preparer's self-test, which must reconnect as a different
|
|
32
|
+
# user against the same target) without this class exposing @keys or
|
|
33
|
+
# @use_tailscale too -- those the caller already has, from config and
|
|
34
|
+
# from its own reasons for choosing them.
|
|
35
|
+
attr_reader :host, :port
|
|
36
|
+
|
|
25
37
|
# Execute remote command
|
|
38
|
+
#
|
|
39
|
+
# Standard error is kept out of the returned value so callers can parse
|
|
40
|
+
# stdout (JSON, container IDs, inspect templates) without warning text
|
|
41
|
+
# leaking in.
|
|
42
|
+
#
|
|
43
|
+
# Commands that are expected to fail should guard themselves at the shell
|
|
44
|
+
# level (`cmd || echo 'none'`) as the Docker client's predicates do.
|
|
45
|
+
#
|
|
26
46
|
# @param command [String] command to execute
|
|
27
|
-
# @return [String] command
|
|
28
|
-
# @raise [Odysseus::
|
|
47
|
+
# @return [String] command stdout
|
|
48
|
+
# @raise [Odysseus::SSHCommandError] if the command exits non-zero
|
|
29
49
|
def execute(command)
|
|
30
|
-
|
|
50
|
+
# Redacted on the way to the terminal only; the command executed
|
|
51
|
+
# below is the real one. See Odysseus::CommandRedaction for why this
|
|
52
|
+
# lives here rather than in the CLI's own redactor.
|
|
53
|
+
puts " > #{Odysseus::CommandRedaction.redact(command)}" if @verbose
|
|
31
54
|
with_connection do |session|
|
|
32
|
-
|
|
55
|
+
stdout = ''
|
|
56
|
+
stderr = ''
|
|
57
|
+
exit_status = nil
|
|
58
|
+
|
|
33
59
|
session.open_channel do |channel|
|
|
34
|
-
channel.exec(command) do |
|
|
60
|
+
channel.exec(command) do |_ch, success|
|
|
35
61
|
raise Odysseus::SSHCommandError, "Failed to execute: #{command}" unless success
|
|
36
62
|
|
|
37
|
-
channel.on_data { |_, data|
|
|
38
|
-
channel.on_extended_data { |_, _, data|
|
|
63
|
+
channel.on_data { |_, data| stdout += data }
|
|
64
|
+
channel.on_extended_data { |_, _, data| stderr += data }
|
|
65
|
+
channel.on_request('exit-status') { |_, data| exit_status = data.read_long }
|
|
39
66
|
end
|
|
40
67
|
end
|
|
41
68
|
session.loop
|
|
42
|
-
|
|
69
|
+
|
|
70
|
+
unless exit_status.nil? || exit_status.zero?
|
|
71
|
+
raise Odysseus::SSHCommandError, command_failure_message(command, exit_status, stderr, stdout)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
stdout
|
|
43
75
|
end
|
|
44
76
|
end
|
|
45
77
|
|
|
@@ -62,11 +94,16 @@ module Odysseus
|
|
|
62
94
|
end
|
|
63
95
|
|
|
64
96
|
# Upload string content to remote file
|
|
97
|
+
#
|
|
98
|
+
# The mode travels in the SCP protocol itself, so a file holding secrets
|
|
99
|
+
# is never briefly world-readable the way a write-then-chmod would be.
|
|
100
|
+
#
|
|
65
101
|
# @param content [String] content to write
|
|
66
102
|
# @param remote_path [String] remote file path
|
|
67
|
-
|
|
103
|
+
# @param mode [Integer] permissions for the remote file
|
|
104
|
+
def upload_string(content, remote_path, mode: 0o640)
|
|
68
105
|
with_connection do |session|
|
|
69
|
-
session.scp.upload!(StringIO.new(content), remote_path)
|
|
106
|
+
session.scp.upload!(StringIO.new(content), remote_path, mode: mode)
|
|
70
107
|
end
|
|
71
108
|
end
|
|
72
109
|
|
|
@@ -76,7 +113,7 @@ module Odysseus
|
|
|
76
113
|
def stream(command, &block)
|
|
77
114
|
with_connection do |session|
|
|
78
115
|
session.open_channel do |channel|
|
|
79
|
-
channel.exec(command) do |
|
|
116
|
+
channel.exec(command) do |_ch, success|
|
|
80
117
|
raise Odysseus::SSHCommandError, "Failed to execute: #{command}" unless success
|
|
81
118
|
|
|
82
119
|
channel.on_data do |_, data|
|
|
@@ -105,16 +142,34 @@ module Odysseus
|
|
|
105
142
|
|
|
106
143
|
private
|
|
107
144
|
|
|
145
|
+
# Build a diagnostic message for a failed remote command.
|
|
146
|
+
# Prefers stderr, falls back to stdout when the command only wrote there.
|
|
147
|
+
def command_failure_message(command, exit_status, stderr, stdout)
|
|
148
|
+
details = stderr.strip
|
|
149
|
+
details = stdout.strip if details.empty?
|
|
150
|
+
|
|
151
|
+
message = "Command failed on #{@host} with exit status #{exit_status}: #{command}"
|
|
152
|
+
message += "\n#{details}" unless details.empty?
|
|
153
|
+
message
|
|
154
|
+
end
|
|
155
|
+
|
|
108
156
|
def with_connection
|
|
109
157
|
connect unless connected?
|
|
158
|
+
# Set only once the session is open, so the rescue below can tell a
|
|
159
|
+
# command that died from a connection that never carried one. Ruby
|
|
160
|
+
# leaves it nil when connect raises first, which is exactly the
|
|
161
|
+
# distinction needed.
|
|
162
|
+
open = true
|
|
110
163
|
yield(@session)
|
|
111
|
-
rescue Errno::ECONNREFUSED
|
|
112
|
-
raise Odysseus::SSHConnectionError,
|
|
113
|
-
|
|
164
|
+
rescue Errno::ECONNREFUSED
|
|
165
|
+
raise Odysseus::SSHConnectionError,
|
|
166
|
+
"Connection refused to #{@host}. Is the server running and accepting SSH connections?"
|
|
167
|
+
rescue SocketError
|
|
114
168
|
raise Odysseus::SSHConnectionError, "Could not resolve hostname '#{@host}'. Check your DNS or /etc/hosts."
|
|
115
|
-
rescue Net::SSH::AuthenticationFailed
|
|
116
|
-
raise Odysseus::
|
|
117
|
-
|
|
169
|
+
rescue Net::SSH::AuthenticationFailed
|
|
170
|
+
raise Odysseus::SSHAuthenticationError,
|
|
171
|
+
"SSH authentication failed for #{@user}@#{@host}. Check your SSH keys."
|
|
172
|
+
rescue Errno::ETIMEDOUT, Net::SSH::ConnectionTimeout, Errno::EHOSTUNREACH
|
|
118
173
|
error_msg = "Connection to #{@host} timed out."
|
|
119
174
|
if @use_tailscale
|
|
120
175
|
error_msg += "\n\nThis looks like a Tailscale hostname. Please check:\n"
|
|
@@ -123,6 +178,25 @@ module Odysseus
|
|
|
123
178
|
error_msg += " 3. The host is online: tailscale ping #{@host}"
|
|
124
179
|
end
|
|
125
180
|
raise Odysseus::SSHConnectionError, error_msg
|
|
181
|
+
rescue IOError, Net::SSH::Disconnect, Errno::ECONNRESET, Errno::EPIPE => e
|
|
182
|
+
# These four arrive from both directions and mean opposite things.
|
|
183
|
+
# Mid-command, the host took the work and then vanished. While
|
|
184
|
+
# opening, it accepted the TCP connection and hung up before a command
|
|
185
|
+
# existed -- a machine still booting, an sshd not up, or a name
|
|
186
|
+
# pointing at a host that is no longer there. Saying "dropped
|
|
187
|
+
# mid-command" for the second is a false claim about what happened,
|
|
188
|
+
# and it points the reader at a flaky network instead of a host that
|
|
189
|
+
# never answered.
|
|
190
|
+
raise Odysseus::SSHConnectionError, drop_message(open, e)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# @param open [Boolean, nil] whether the session was established
|
|
194
|
+
def drop_message(open, error)
|
|
195
|
+
return "Connection to #{@host} dropped mid-command rather than failing to open: #{error.message}" if open
|
|
196
|
+
|
|
197
|
+
"#{@host} accepted the connection and then closed it before any command ran: #{error.message}. " \
|
|
198
|
+
'The host may still be booting, sshd may not be running yet, or the name may point at a ' \
|
|
199
|
+
'machine that is no longer there.'
|
|
126
200
|
end
|
|
127
201
|
|
|
128
202
|
def connect
|
|
@@ -131,7 +205,7 @@ module Odysseus
|
|
|
131
205
|
port: @port,
|
|
132
206
|
non_interactive: true,
|
|
133
207
|
verify_host_key: :never,
|
|
134
|
-
timeout: 10
|
|
208
|
+
timeout: 10 # Connection timeout in seconds
|
|
135
209
|
}
|
|
136
210
|
options[:keys] = @keys if @keys.any?
|
|
137
211
|
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
# lib/odysseus/docker/client.rb
|
|
2
2
|
|
|
3
3
|
require 'json'
|
|
4
|
+
require 'securerandom'
|
|
5
|
+
require 'shellwords'
|
|
4
6
|
|
|
5
7
|
module Odysseus
|
|
6
8
|
module Docker
|
|
7
9
|
class Client
|
|
8
10
|
HEALTHCHECK_POLL_INTERVAL = 2 # seconds
|
|
9
|
-
HEALTHCHECK_MAX_ATTEMPTS = 30 # ~60 seconds max wait
|
|
10
11
|
|
|
11
12
|
# @param ssh [Odysseus::Deployer::SSH] SSH connection to server
|
|
12
13
|
def initialize(ssh)
|
|
@@ -14,23 +15,33 @@ module Odysseus
|
|
|
14
15
|
end
|
|
15
16
|
|
|
16
17
|
# Run a new container
|
|
18
|
+
#
|
|
19
|
+
# Environment variables travel in a 0600 env file rather than on the
|
|
20
|
+
# command line, so secrets stay out of the host's process list and values
|
|
21
|
+
# containing spaces or shell metacharacters survive intact. Docker copies
|
|
22
|
+
# them into the container config at create time, so the file is removed
|
|
23
|
+
# again as soon as the container exists.
|
|
24
|
+
#
|
|
17
25
|
# @param name [String] container name
|
|
18
26
|
# @param image [String] image:tag
|
|
19
27
|
# @param options [Hash] container options
|
|
20
28
|
# @return [String] container ID
|
|
21
29
|
def run(name:, image:, options: {})
|
|
22
|
-
|
|
30
|
+
env_file = env_file_path(name, options[:env])
|
|
31
|
+
write_env_file(env_file, options[:env])
|
|
32
|
+
|
|
33
|
+
cmd = build_run_command(name: name, image: image, options: options, env_file: env_file)
|
|
23
34
|
output = @ssh.execute(cmd)
|
|
24
35
|
# Container ID is the last line (64-char hex), ignore any warnings
|
|
25
36
|
lines = output.strip.split("\n")
|
|
26
37
|
container_id = lines.last&.strip
|
|
27
38
|
|
|
28
39
|
# Validate it looks like a container ID
|
|
29
|
-
unless container_id&.match?(/\A[a-f0-9]{64}\z/)
|
|
30
|
-
raise Odysseus::DeployError, "Failed to start container: #{output}"
|
|
31
|
-
end
|
|
40
|
+
raise Odysseus::DeployError, "Failed to start container: #{output}" unless container_id&.match?(/\A[a-f0-9]{64}\z/)
|
|
32
41
|
|
|
33
42
|
container_id
|
|
43
|
+
ensure
|
|
44
|
+
remove_env_file(env_file)
|
|
34
45
|
end
|
|
35
46
|
|
|
36
47
|
# Stop a container
|
|
@@ -77,7 +88,7 @@ module Odysseus
|
|
|
77
88
|
# @param timeout [Integer] max seconds to wait
|
|
78
89
|
# @return [Boolean] true if healthy, false if timeout
|
|
79
90
|
def wait_healthy(container_id, timeout: 60)
|
|
80
|
-
attempts = [timeout / HEALTHCHECK_POLL_INTERVAL,
|
|
91
|
+
attempts = [timeout / HEALTHCHECK_POLL_INTERVAL, 1].max
|
|
81
92
|
|
|
82
93
|
attempts.times do
|
|
83
94
|
status = health_status(container_id)
|
|
@@ -100,6 +111,16 @@ module Odysseus
|
|
|
100
111
|
output.strip == 'true'
|
|
101
112
|
end
|
|
102
113
|
|
|
114
|
+
# Check if a container exists, running or stopped
|
|
115
|
+
# @param container_id [String] container ID or name
|
|
116
|
+
# @return [Boolean]
|
|
117
|
+
def container_exists?(container_id)
|
|
118
|
+
output = @ssh.execute(
|
|
119
|
+
"docker inspect --format '{{.Id}}' #{container_id} 2>/dev/null || echo ''"
|
|
120
|
+
)
|
|
121
|
+
!output.strip.empty?
|
|
122
|
+
end
|
|
123
|
+
|
|
103
124
|
# Get container IP address
|
|
104
125
|
# @param container_id [String] container ID or name
|
|
105
126
|
# @param network [String] network name (default: bridge)
|
|
@@ -126,6 +147,50 @@ module Odysseus
|
|
|
126
147
|
!output.strip.empty?
|
|
127
148
|
end
|
|
128
149
|
|
|
150
|
+
# Tags of the images present locally for a repository.
|
|
151
|
+
#
|
|
152
|
+
# docker orders these newest-created first, which is the fallback
|
|
153
|
+
# ordering a rollback uses on a host with no deploy log. Untagged
|
|
154
|
+
# (dangling) images report a tag of '<none>' and are dropped: they cannot
|
|
155
|
+
# be named in a docker run, so they are never rollback targets.
|
|
156
|
+
#
|
|
157
|
+
# @param image [String] repository name, without a tag
|
|
158
|
+
# @return [Array<String>] tags present on this host
|
|
159
|
+
def image_tags(image)
|
|
160
|
+
output = @ssh.execute(
|
|
161
|
+
"docker images #{Shellwords.escape(image)} --format '{{.Tag}}' 2>/dev/null || true"
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
output.lines.map(&:strip).reject { |tag| tag.empty? || tag == '<none>' }
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Remove one image by reference.
|
|
168
|
+
#
|
|
169
|
+
# Lets SSHCommandError through deliberately: docker refuses to remove an
|
|
170
|
+
# image a container still references, and the caller prunes one image at a
|
|
171
|
+
# time so a refusal is a logged skip rather than a failed deploy.
|
|
172
|
+
#
|
|
173
|
+
# @param image [String] repository:tag
|
|
174
|
+
# @return [String] docker's output
|
|
175
|
+
def remove_image(image)
|
|
176
|
+
@ssh.execute("docker image rm #{Shellwords.escape(image)}")
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# The versions any container on this host still references.
|
|
180
|
+
#
|
|
181
|
+
# Includes stopped containers (`all: true`): a stopped container still
|
|
182
|
+
# references its image, whether it stopped from a crash, a reboot, or by
|
|
183
|
+
# hand, and deleting that image would remove something an operator may
|
|
184
|
+
# still need. Used to protect those versions from retention.
|
|
185
|
+
#
|
|
186
|
+
# @param service_labels [Array<String>] odysseus.service values to check
|
|
187
|
+
# @return [Array<String>] distinct odysseus.version labels found
|
|
188
|
+
def versions_in_use(service_labels)
|
|
189
|
+
service_labels.flat_map { |label| list(service: label, all: true) }
|
|
190
|
+
.filter_map { |container| Odysseus::Docker::Labels.version_of(container) }
|
|
191
|
+
.uniq
|
|
192
|
+
end
|
|
193
|
+
|
|
129
194
|
# Get logs from a container
|
|
130
195
|
# @param container_id [String] container ID or name
|
|
131
196
|
# @param follow [Boolean] follow log output (streaming)
|
|
@@ -133,7 +198,7 @@ module Odysseus
|
|
|
133
198
|
# @param since [String] show logs since timestamp (e.g., '10m', '2h', '2024-01-01')
|
|
134
199
|
# @param timestamps [Boolean] show timestamps
|
|
135
200
|
# @return [String] log output (or yields lines if block given)
|
|
136
|
-
def logs(container_id, follow: false, tail: 100, since: nil, timestamps: false, &
|
|
201
|
+
def logs(container_id, follow: false, tail: 100, since: nil, timestamps: false, &)
|
|
137
202
|
parts = ['docker logs']
|
|
138
203
|
parts << '--follow' if follow
|
|
139
204
|
parts << "--tail #{tail}" if tail
|
|
@@ -144,7 +209,7 @@ module Odysseus
|
|
|
144
209
|
cmd = parts.join(' ')
|
|
145
210
|
|
|
146
211
|
if block_given?
|
|
147
|
-
@ssh.stream(cmd, &
|
|
212
|
+
@ssh.stream(cmd, &)
|
|
148
213
|
else
|
|
149
214
|
@ssh.execute(cmd)
|
|
150
215
|
end
|
|
@@ -167,32 +232,63 @@ module Odysseus
|
|
|
167
232
|
end
|
|
168
233
|
|
|
169
234
|
# Run a one-off command in a new container (doesn't persist)
|
|
235
|
+
#
|
|
236
|
+
# The environment travels in a 0600 env file, exactly as a deployed
|
|
237
|
+
# container's does: `rails db:migrate` needs the app's DATABASE_URL, and
|
|
238
|
+
# putting it on the command line would show it in the host's process list
|
|
239
|
+
# and break any value containing a space.
|
|
240
|
+
#
|
|
170
241
|
# @param image [String] image to use
|
|
171
|
-
# @param command [String] command to execute
|
|
242
|
+
# @param command [String] command to execute, as a shell command line
|
|
172
243
|
# @param options [Hash] container options (env, volumes, network, etc.)
|
|
173
244
|
# @return [String] command output
|
|
174
245
|
def run_once(image:, command:, options: {})
|
|
175
|
-
|
|
246
|
+
with_env_file(options[:env]) do |env_file|
|
|
247
|
+
parts = ['docker run --rm']
|
|
176
248
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
parts << "-e #{key}=#{value}"
|
|
180
|
-
end
|
|
249
|
+
# Environment variables (see #write_env_file — never inlined here)
|
|
250
|
+
parts << "--env-file #{Shellwords.escape(env_file)}" if env_file
|
|
181
251
|
|
|
182
|
-
|
|
183
|
-
|
|
252
|
+
# Volume mounts
|
|
253
|
+
options[:volumes]&.each { |v| parts << "-v #{v}" }
|
|
184
254
|
|
|
185
|
-
|
|
186
|
-
|
|
255
|
+
# Network
|
|
256
|
+
parts << "--network #{options[:network]}" if options[:network]
|
|
187
257
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
258
|
+
# Interactive/TTY
|
|
259
|
+
parts << '-i' if options[:interactive]
|
|
260
|
+
parts << '-t' if options[:tty]
|
|
191
261
|
|
|
192
|
-
|
|
193
|
-
|
|
262
|
+
parts << Shellwords.escape(image)
|
|
263
|
+
# Not escaped: the command is a command line, and `rake db:migrate`
|
|
264
|
+
# has to reach docker as two arguments. build_run_command treats
|
|
265
|
+
# options[:cmd] the same way.
|
|
266
|
+
parts << command
|
|
194
267
|
|
|
195
|
-
|
|
268
|
+
@ssh.execute(parts.join(' '))
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Hold an env file open on the host for the duration of a block.
|
|
273
|
+
#
|
|
274
|
+
# For runs Odysseus does not execute itself: `app shell` and `app console`
|
|
275
|
+
# need an interactive TTY, so the CLI builds its own `ssh -t ... docker
|
|
276
|
+
# run` and passes the yielded path as --env-file. The file goes away
|
|
277
|
+
# afterwards whether the block returned or raised, and remove_env_file
|
|
278
|
+
# never masks the block's own failure.
|
|
279
|
+
#
|
|
280
|
+
# Yields nil, having written nothing, when there is no environment, so
|
|
281
|
+
# the caller has one code path either way.
|
|
282
|
+
#
|
|
283
|
+
# @param env [Hash, nil] environment variables
|
|
284
|
+
# @yieldparam path [String, nil] path to the env file on the host
|
|
285
|
+
# @return [Object] whatever the block returned
|
|
286
|
+
def with_env_file(env)
|
|
287
|
+
path = env_file_path(one_off_env_name, env)
|
|
288
|
+
write_env_file(path, env)
|
|
289
|
+
yield path
|
|
290
|
+
ensure
|
|
291
|
+
remove_env_file(path)
|
|
196
292
|
end
|
|
197
293
|
|
|
198
294
|
# Cleanup old stopped containers, keeping only the last N
|
|
@@ -278,35 +374,121 @@ module Odysseus
|
|
|
278
374
|
|
|
279
375
|
private
|
|
280
376
|
|
|
281
|
-
|
|
377
|
+
# Where this connection's env files go. Derived rather than constant
|
|
378
|
+
# because a deploy user cannot write — or chmod — the system directory.
|
|
379
|
+
def host_paths
|
|
380
|
+
@host_paths ||= Odysseus::HostPaths.new(@ssh)
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
# Where a container's env file goes, or nil when there is nothing to
|
|
384
|
+
# write. Settled before the write rather than returned by it: scp creates
|
|
385
|
+
# the remote file and then streams into it, so an upload that dies partway
|
|
386
|
+
# has already left part of a file of secrets on the host, and a caller
|
|
387
|
+
# that learned the path from the write's return value has nothing to
|
|
388
|
+
# remove — the file stays under a name nobody is going to look for.
|
|
389
|
+
#
|
|
390
|
+
# @return [String, nil] path to the env file, nil when there is nothing to write
|
|
391
|
+
def env_file_path(name, env)
|
|
392
|
+
return nil if env.nil? || env.empty?
|
|
393
|
+
|
|
394
|
+
"#{host_paths.env_dir}/#{name}.env"
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
# Write the container's environment to a private file on the host.
|
|
398
|
+
#
|
|
399
|
+
# The directory is made 0700 before anything is written into it. The file
|
|
400
|
+
# itself is uploaded 0600, so this is a second guard rather than the only
|
|
401
|
+
# one — but it is the guard that has to hold for a file left behind by a
|
|
402
|
+
# session that died, and `mkdir -p` on its own leaves the directory 0755.
|
|
403
|
+
def write_env_file(path, env)
|
|
404
|
+
return unless path
|
|
405
|
+
|
|
406
|
+
dir = host_paths.env_dir
|
|
407
|
+
@ssh.execute("mkdir -p #{Shellwords.escape(dir)} && chmod 700 #{Shellwords.escape(dir)}")
|
|
408
|
+
@ssh.upload_string(format_env_file(env), path, mode: 0o600)
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
# The name a one-off run's env file is written under.
|
|
412
|
+
#
|
|
413
|
+
# write_env_file names the file after the container, and a one-off has no
|
|
414
|
+
# container name. '@' is not a character Docker allows in one
|
|
415
|
+
# ([a-zA-Z0-9][a-zA-Z0-9_.-]*), so no deployed container's env file can
|
|
416
|
+
# ever live at this path — overwriting a running container's env file, or
|
|
417
|
+
# deleting it on the way out, would be a live incident. The random suffix
|
|
418
|
+
# keeps two one-off runs on the same host from sharing a file.
|
|
419
|
+
def one_off_env_name
|
|
420
|
+
"one-off@#{SecureRandom.hex(8)}"
|
|
421
|
+
end
|
|
422
|
+
|
|
423
|
+
# docker --env-file takes one KEY=VALUE per line and cannot represent a
|
|
424
|
+
# value containing a newline, so refuse rather than truncate a secret.
|
|
425
|
+
def format_env_file(env)
|
|
426
|
+
lines = env.map do |key, value|
|
|
427
|
+
value = value.to_s
|
|
428
|
+
if value.include?("\n")
|
|
429
|
+
raise Odysseus::DeployError,
|
|
430
|
+
"Environment variable #{key} contains a newline, which a Docker env file cannot represent"
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
"#{key}=#{value}"
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
"#{lines.join("\n")}\n"
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
# Remove the env file, reconnecting once if the connection it was written
|
|
440
|
+
# over has died in the meantime.
|
|
441
|
+
#
|
|
442
|
+
# This runs from an ensure, and on the interactive paths the connection
|
|
443
|
+
# has been held open — idle, and with nothing pumping it — for as long as
|
|
444
|
+
# the user's session lasted. An idle NAT or firewall timeout, sshd's
|
|
445
|
+
# ClientAlive limit or a Tailscale relay change all leave it dead by the
|
|
446
|
+
# time the session ends, and a dead connection raises IOError,
|
|
447
|
+
# Net::SSH::Disconnect, Errno::EPIPE or Errno::ECONNRESET — none of them
|
|
448
|
+
# an Odysseus::SSHError, which is all this used to rescue. The cleanup's
|
|
449
|
+
# own failure then escaped the ensure and replaced whatever the block was
|
|
450
|
+
# already raising, so `app shell`'s exit status arrived as a backtrace.
|
|
451
|
+
#
|
|
452
|
+
# Closing the session is what makes the second attempt a new one: SSH
|
|
453
|
+
# connects lazily and only when it has no live session. If that fails too
|
|
454
|
+
# the file is left behind — 0600 in a 0700 directory — and nothing is
|
|
455
|
+
# raised: the caller came for the block's outcome, not this one's.
|
|
456
|
+
def remove_env_file(path)
|
|
457
|
+
return unless path
|
|
458
|
+
|
|
459
|
+
@ssh.execute("rm -f #{Shellwords.escape(path)}")
|
|
460
|
+
rescue StandardError
|
|
461
|
+
remove_env_file_on_a_new_connection(path)
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
def remove_env_file_on_a_new_connection(path)
|
|
465
|
+
@ssh.close
|
|
466
|
+
@ssh.execute("rm -f #{Shellwords.escape(path)}")
|
|
467
|
+
rescue StandardError
|
|
468
|
+
nil
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
def build_run_command(name:, image:, options:, env_file: nil)
|
|
282
472
|
parts = ['docker run -d']
|
|
283
473
|
|
|
284
474
|
# Container name
|
|
285
475
|
parts << "--name #{name}"
|
|
286
476
|
|
|
287
|
-
# Labels for tracking
|
|
288
|
-
|
|
289
|
-
parts << "--label odysseus.
|
|
477
|
+
# Labels for tracking. Values are quoted: they carry refs and timestamps
|
|
478
|
+
# supplied by the app's repository, not just internal identifiers.
|
|
479
|
+
parts << "--label #{Shellwords.escape("odysseus.service=#{options[:service] || name}")}"
|
|
480
|
+
parts << "--label #{Shellwords.escape("odysseus.version=#{options[:version]}")}" if options[:version]
|
|
290
481
|
|
|
291
482
|
# Additional custom labels
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
parts << "--label #{key}=#{value}"
|
|
295
|
-
end
|
|
483
|
+
options[:labels]&.each do |key, value|
|
|
484
|
+
parts << "--label #{Shellwords.escape("#{key}=#{value}")}"
|
|
296
485
|
end
|
|
297
486
|
|
|
298
487
|
# Port mappings
|
|
299
|
-
|
|
300
|
-
options[:ports].each { |p| parts << "-p #{p}" }
|
|
301
|
-
end
|
|
488
|
+
options[:ports]&.each { |p| parts << "-p #{p}" }
|
|
302
489
|
|
|
303
|
-
# Environment variables
|
|
304
|
-
if
|
|
305
|
-
options[:env].each do |key, value|
|
|
306
|
-
# Don't log secret values
|
|
307
|
-
parts << "-e #{key}=#{value}"
|
|
308
|
-
end
|
|
309
|
-
end
|
|
490
|
+
# Environment variables (see #write_env_file — never inlined here)
|
|
491
|
+
parts << "--env-file #{Shellwords.escape(env_file)}" if env_file
|
|
310
492
|
|
|
311
493
|
# Memory limits
|
|
312
494
|
parts << "--memory #{options[:memory]}" if options[:memory]
|
|
@@ -319,7 +501,15 @@ module Odysseus
|
|
|
319
501
|
# Health check (use image's HEALTHCHECK by default)
|
|
320
502
|
if options[:healthcheck]
|
|
321
503
|
hc = options[:healthcheck]
|
|
322
|
-
|
|
504
|
+
# Escaped, not hand-quoted. A cmd containing a single quote --
|
|
505
|
+
# `--execute='SELECT 1'`, which is how you write a one-shot query
|
|
506
|
+
# for most database clients -- closed the quote early, and the
|
|
507
|
+
# remainder word-split into docker's argument list: the tail landed
|
|
508
|
+
# where the image name goes, and docker reported it could not find
|
|
509
|
+
# the image '1:latest'. Shellwords makes it one word whatever it
|
|
510
|
+
# contains; docker runs it through a shell at the other end, so the
|
|
511
|
+
# inner quoting still means what it says.
|
|
512
|
+
parts << "--health-cmd #{Shellwords.escape(hc[:cmd])}" if hc[:cmd]
|
|
323
513
|
parts << "--health-interval #{hc[:interval]}s" if hc[:interval]
|
|
324
514
|
parts << "--health-timeout #{hc[:timeout]}s" if hc[:timeout]
|
|
325
515
|
parts << "--health-retries #{hc[:retries]}" if hc[:retries]
|
|
@@ -328,10 +518,9 @@ module Odysseus
|
|
|
328
518
|
# Network
|
|
329
519
|
parts << "--network #{options[:network]}" if options[:network]
|
|
330
520
|
|
|
331
|
-
# Volume mounts
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
end
|
|
521
|
+
# Volume mounts. Escaped for the same reason as the healthcheck above:
|
|
522
|
+
# a host path with a space in it is otherwise two arguments.
|
|
523
|
+
options[:volumes]&.each { |v| parts << "-v #{Shellwords.escape(v)}" }
|
|
335
524
|
|
|
336
525
|
# Restart policy
|
|
337
526
|
parts << "--restart #{options[:restart] || 'unless-stopped'}"
|
|
@@ -339,7 +528,10 @@ module Odysseus
|
|
|
339
528
|
# Image
|
|
340
529
|
parts << image
|
|
341
530
|
|
|
342
|
-
# Command (if provided)
|
|
531
|
+
# Command (if provided). Deliberately NOT escaped, unlike everything
|
|
532
|
+
# above: `start-single-node --insecure` has to reach the container as
|
|
533
|
+
# three arguments, and escaping it would hand docker one literal
|
|
534
|
+
# string containing spaces. Do not "fix" this to match its neighbours.
|
|
343
535
|
parts << options[:cmd] if options[:cmd]
|
|
344
536
|
|
|
345
537
|
parts.join(' ')
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# lib/odysseus/docker/labels.rb
|
|
2
|
+
|
|
3
|
+
module Odysseus
|
|
4
|
+
module Docker
|
|
5
|
+
# docker ps --format '{{json .}}' reports labels as one flat string:
|
|
6
|
+
# "k=v,k2=v2". This turns that back into a hash.
|
|
7
|
+
module Labels
|
|
8
|
+
VERSION_KEY = 'odysseus.version'.freeze
|
|
9
|
+
|
|
10
|
+
# @param raw [String, nil] the Labels field of a docker ps entry
|
|
11
|
+
# @return [Hash{String => String}]
|
|
12
|
+
def self.parse(raw)
|
|
13
|
+
return {} if raw.nil? || raw.empty?
|
|
14
|
+
|
|
15
|
+
raw.split(',').each_with_object({}) do |pair, acc|
|
|
16
|
+
key, value = pair.split('=', 2)
|
|
17
|
+
acc[key] = value.to_s unless key.nil? || key.empty?
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @param container [Hash] a docker ps entry
|
|
22
|
+
# @return [String, nil] the deployed version, when labelled
|
|
23
|
+
def self.version_of(container)
|
|
24
|
+
parse(container['Labels'])[VERSION_KEY]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# The odysseus.service label value carried by a role's containers.
|
|
28
|
+
#
|
|
29
|
+
# WebDeploy labels the web role with the bare service name; JobDeploy
|
|
30
|
+
# labels every other role "<service>-<role>". Both conventions predate
|
|
31
|
+
# this method, which exists so that reading containers back cannot
|
|
32
|
+
# disagree with writing them. docker ps filters on an exact label match,
|
|
33
|
+
# so a wrong value here silently reports nothing running.
|
|
34
|
+
#
|
|
35
|
+
# @param service [String] the service name from deploy.yml
|
|
36
|
+
# @param role [Symbol, String] the server role
|
|
37
|
+
# @return [String]
|
|
38
|
+
def self.service_for(service:, role:)
|
|
39
|
+
role.to_sym == :web ? service.to_s : "#{service}-#{role}"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
data/lib/odysseus/errors.rb
CHANGED
|
@@ -9,10 +9,18 @@ module Odysseus
|
|
|
9
9
|
class ConfigMissingKeyError < ConfigError; end
|
|
10
10
|
|
|
11
11
|
class DeployError < Error; end
|
|
12
|
+
class RollbackError < DeployError; end
|
|
12
13
|
class SSHError < DeployError; end
|
|
13
14
|
class SSHConnectionError < SSHError; end
|
|
15
|
+
# Distinct from SSHConnectionError: the host was reached and refused
|
|
16
|
+
# this identity. Only a caller that knows which identities are
|
|
17
|
+
# available can advise on that, so it needs to be catchable on its own.
|
|
18
|
+
class SSHAuthenticationError < SSHError; end
|
|
14
19
|
class SSHCommandError < SSHError; end
|
|
15
20
|
|
|
21
|
+
class ProxyError < Error; end
|
|
22
|
+
class ProxyApiError < ProxyError; end
|
|
23
|
+
|
|
16
24
|
class RegistryError < Error; end
|
|
17
25
|
class RegistryPushError < RegistryError; end
|
|
18
26
|
class RegistryAuthError < RegistryError; end
|
|
@@ -24,4 +32,6 @@ module Odysseus
|
|
|
24
32
|
class GeneratorError < Error; end
|
|
25
33
|
class DockerComposeGenerationError < GeneratorError; end
|
|
26
34
|
class CaddyGenerationError < GeneratorError; end
|
|
35
|
+
|
|
36
|
+
class SetupError < Error; end
|
|
27
37
|
end
|