odysseus-core 0.3.2 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +485 -1
  3. data/LICENSE.txt +25 -7
  4. data/README.md +12 -3
  5. data/lib/odysseus/builder/client.rb +15 -15
  6. data/lib/odysseus/caddy/client.rb +119 -42
  7. data/lib/odysseus/command_redaction.rb +38 -0
  8. data/lib/odysseus/config/parser.rb +43 -18
  9. data/lib/odysseus/core/deploy_versioning.rb +42 -0
  10. data/lib/odysseus/core/environment.rb +63 -0
  11. data/lib/odysseus/core/version.rb +1 -1
  12. data/lib/odysseus/core/volume_namespacer.rb +0 -0
  13. data/lib/odysseus/core.rb +2 -2
  14. data/lib/odysseus/deploy_log.rb +120 -0
  15. data/lib/odysseus/deploy_version.rb +9 -0
  16. data/lib/odysseus/deployer/dependency_manager.rb +121 -0
  17. data/lib/odysseus/deployer/executor.rb +213 -161
  18. data/lib/odysseus/deployer/retention_sweeper.rb +94 -0
  19. data/lib/odysseus/deployer/ssh.rb +92 -18
  20. data/lib/odysseus/docker/client.rb +240 -48
  21. data/lib/odysseus/docker/labels.rb +43 -0
  22. data/lib/odysseus/errors.rb +10 -0
  23. data/lib/odysseus/git.rb +70 -0
  24. data/lib/odysseus/host_paths.rb +79 -0
  25. data/lib/odysseus/host_providers/base.rb +1 -1
  26. data/lib/odysseus/host_providers/static.rb +1 -1
  27. data/lib/odysseus/host_providers.rb +3 -4
  28. data/lib/odysseus/host_verifier.rb +156 -0
  29. data/lib/odysseus/host_versions.rb +59 -0
  30. data/lib/odysseus/orchestrator/{accessory_deploy.rb → dependency_deploy.rb} +84 -88
  31. data/lib/odysseus/orchestrator/job_deploy.rb +31 -43
  32. data/lib/odysseus/orchestrator/web_deploy.rb +61 -55
  33. data/lib/odysseus/plugins.rb +72 -0
  34. data/lib/odysseus/retention_plan.rb +14 -0
  35. data/lib/odysseus/retention_planner.rb +58 -0
  36. data/lib/odysseus/rollback_plan.rb +20 -0
  37. data/lib/odysseus/rollback_planner.rb +107 -0
  38. data/lib/odysseus/sails.rb +0 -0
  39. data/lib/odysseus/secrets/encrypted_file.rb +5 -7
  40. data/lib/odysseus/secrets/loader.rb +1 -3
  41. data/lib/odysseus/setup/docker_apt.rb +178 -0
  42. data/lib/odysseus/setup/escalation.rb +73 -0
  43. data/lib/odysseus/setup/preparer.rb +372 -0
  44. data/lib/odysseus/setup/public_key.rb +127 -0
  45. data/lib/odysseus/validators/config.rb +38 -25
  46. data/lib/odysseus/version_resolver.rb +80 -0
  47. data/lib/odysseus.rb +0 -0
  48. data/sig/odysseus/core.rbs +0 -0
  49. metadata +48 -27
  50. data/Rakefile +0 -12
  51. data/lib/odysseus/version.rb +0 -5
@@ -0,0 +1,70 @@
1
+ # lib/odysseus/git.rb
2
+
3
+ require 'open3'
4
+
5
+ module Odysseus
6
+ # Answers questions about a local git work tree. Knows nothing about deploys:
7
+ # the rules that turn these answers into a version live in VersionResolver.
8
+ class Git
9
+ # @param dir [String] directory inside the work tree
10
+ def initialize(dir)
11
+ @dir = dir
12
+ end
13
+
14
+ # @return [Boolean] true when dir is inside a git work tree
15
+ def repository?
16
+ _, status = capture('rev-parse', '--git-dir')
17
+ status.success?
18
+ end
19
+
20
+ # @param length [Integer] characters of the object name to return
21
+ # @return [String, nil] abbreviated commit sha of HEAD
22
+ def head_sha(length: 12)
23
+ out, status = capture('rev-parse', "--short=#{length}", 'HEAD')
24
+ status.success? && !out.empty? ? out : nil
25
+ end
26
+
27
+ # Tracked modifications only. Untracked files are reported separately: they
28
+ # are usually local noise, and blocking on them would make deploys hostile.
29
+ # @return [Boolean]
30
+ def uncommitted_changes?
31
+ out, status = capture('status', '--porcelain', '--untracked-files=no')
32
+ status.success? && !out.empty?
33
+ end
34
+
35
+ # @return [Boolean] true when the work tree has files git is not tracking
36
+ def untracked_files?
37
+ out, status = capture('ls-files', '--others', '--exclude-standard')
38
+ status.success? && !out.empty?
39
+ end
40
+
41
+ # @return [String, nil] branch name, or 'HEAD' when detached
42
+ def ref
43
+ out, status = capture('rev-parse', '--abbrev-ref', 'HEAD')
44
+ status.success? && !out.empty? ? out : nil
45
+ end
46
+
47
+ # @return [String, nil] user.email as git resolves it for this repository
48
+ def committer_email
49
+ out, status = capture('config', 'user.email')
50
+ status.success? && !out.empty? ? out : nil
51
+ end
52
+
53
+ private
54
+
55
+ def capture(*)
56
+ stdout, _stderr, status = Open3.capture3('git', *, chdir: @dir)
57
+ [stdout.strip, status]
58
+ rescue Errno::ENOENT
59
+ # git is not installed; treat as "not a repository" rather than crashing.
60
+ ['', FailedStatus.new]
61
+ end
62
+
63
+ # Stands in for a Process::Status when git could not be executed at all.
64
+ class FailedStatus
65
+ def success?
66
+ false
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,79 @@
1
+ # lib/odysseus/host_paths.rb
2
+
3
+ module Odysseus
4
+ # Where odysseus keeps its state on a target host.
5
+ #
6
+ # Root writes /var/lib/odysseus, which is where every install has always
7
+ # written and where existing hosts still have their deploy history. Any other
8
+ # user cannot create that directory, so their state goes under their own
9
+ # home. One class answers this so the deploy log, the env files, Caddy's
10
+ # data directory and — later — the deploy lock cannot disagree about where
11
+ # they live.
12
+ class HostPaths
13
+ SYSTEM_BASE = '/var/lib/odysseus'.freeze
14
+ USER_DIRNAME = '.odysseus'.freeze
15
+ ROOT = 'root'.freeze
16
+
17
+ # @param ssh [Odysseus::Deployer::SSH] connection whose user decides the base
18
+ def initialize(ssh)
19
+ @ssh = ssh
20
+ @base = nil
21
+ end
22
+
23
+ # @return [String] the directory all odysseus state lives under
24
+ def base
25
+ @base ||= @ssh.user == ROOT ? SYSTEM_BASE : File.join(home, USER_DIRNAME)
26
+ end
27
+
28
+ # @param service [String] the service: value from deploy.yml
29
+ # @return [String] that service's state directory
30
+ def service_dir(service)
31
+ File.join(base, service)
32
+ end
33
+
34
+ # @return [String] where env files are written for the length of a docker run
35
+ def env_dir
36
+ File.join(base, 'env')
37
+ end
38
+
39
+ # Caddy's /data mount: issued certificates, written by the container as
40
+ # root. For a root connection this resolves to exactly the path every
41
+ # install has always used, so existing certificates are found unmoved. A
42
+ # non-root connection gets a directory it can actually create — the
43
+ # earlier design left this fixed at the root path to protect those
44
+ # certificates, but that protection only ever applied to root: deriving it
45
+ # like every other path here protects root identically while letting a
46
+ # non-root deploy user create its own directory instead of failing at
47
+ # `mkdir`.
48
+ #
49
+ # Caddy is one container shared by every service on a host, so this only
50
+ # works cleanly with the one-deploy-user-per-host shape this class already
51
+ # assumes: two different deploy users on the same host would disagree
52
+ # about where Caddy's data lives, and whichever one first starts the
53
+ # container wins, since #ensure_running never recreates one already
54
+ # running.
55
+ # @return [String]
56
+ def caddy_dir
57
+ File.join(base, 'caddy')
58
+ end
59
+
60
+ # Where a root install wrote, whoever is connected now. Used to read the
61
+ # deploy history of a host that has since moved to a deploy user.
62
+ # @return [String]
63
+ def legacy_base
64
+ SYSTEM_BASE
65
+ end
66
+
67
+ private
68
+
69
+ # Asked once per connection and cached. A host that reports nothing is an
70
+ # error rather than a path of "/.odysseus", which would be unwritable and
71
+ # would only be noticed later, as a failed deploy.
72
+ def home
73
+ value = @ssh.execute('echo $HOME').to_s.strip
74
+ raise Odysseus::DeployError, "Could not determine the home directory of #{@ssh.user}" if value.empty?
75
+
76
+ value
77
+ end
78
+ end
79
+ end
@@ -14,7 +14,7 @@ module Odysseus
14
14
  # Resolve and return list of hosts
15
15
  # @return [Array<String>] list of hostnames or IPs
16
16
  def resolve
17
- raise NotImplementedError, "Subclasses must implement #resolve"
17
+ raise NotImplementedError, 'Subclasses must implement #resolve'
18
18
  end
19
19
 
20
20
  # Provider name for display/logging
@@ -17,7 +17,7 @@ module Odysseus
17
17
  end
18
18
 
19
19
  def name
20
- "static"
20
+ 'static'
21
21
  end
22
22
  end
23
23
  end
@@ -18,7 +18,7 @@ module Odysseus
18
18
  providers[:aws_asg].new(role_config[:aws])
19
19
  elsif role_config[:aws]
20
20
  raise Odysseus::ConfigError,
21
- "AWS ASG host provider not available — is the odysseus-sail-aws-asg gem loaded?"
21
+ 'AWS ASG host provider not available — is the odysseus-sail-aws-asg gem loaded?'
22
22
  elsif role_config[:hosts]
23
23
  Static.new(hosts: role_config[:hosts])
24
24
  else
@@ -38,9 +38,8 @@ module Odysseus
38
38
  # @param name [Symbol] provider name
39
39
  # @param klass [Class] provider class (must inherit from Base)
40
40
  def register(name, klass)
41
- unless klass < Base
42
- raise ArgumentError, "Provider must inherit from Odysseus::HostProviders::Base"
43
- end
41
+ raise ArgumentError, 'Provider must inherit from Odysseus::HostProviders::Base' unless klass < Base
42
+
44
43
  providers[name] = klass
45
44
  end
46
45
  end
@@ -0,0 +1,156 @@
1
+ # lib/odysseus/host_verifier.rb
2
+
3
+ require 'shellwords'
4
+
5
+ module Odysseus
6
+ # Read-only diagnosis of whether one host is ready for odysseus to deploy to
7
+ # it AS THE USER THE CONFIG NAMES.
8
+ #
9
+ # That last part is the point. Verifying as root would pass on a host the
10
+ # deploy user cannot use, which is the failure this exists to catch — a deploy
11
+ # that dies at the first container start because a directory is not writable.
12
+ #
13
+ # Nothing here writes to the host. A check that would need to write in order
14
+ # to learn something reports that it cannot tell instead.
15
+ #
16
+ # Caddy's data directory is deliberately absent from these checks: it does not
17
+ # exist until the first deploy starts Caddy, so a correctly configured host
18
+ # that has not deployed yet would be reported broken.
19
+ class HostVerifier
20
+ # Named explicitly rather than computed, so supporting a new LTS is a
21
+ # deliberate edit with a tested host behind it, not something that becomes
22
+ # true on a date.
23
+ SUPPORTED_UBUNTU = %w[24.04 26.04].freeze
24
+ ROOT = 'root'.freeze
25
+
26
+ # @param check [Symbol] which check this is
27
+ # @param status [Symbol] :ok, :warn or :fail
28
+ # @param detail [String] one line a reader can act on
29
+ Result = Data.define(:check, :status, :detail)
30
+
31
+ def initialize(ssh:, config:)
32
+ @ssh = ssh
33
+ @config = config
34
+ end
35
+
36
+ # @return [Array<Result>] one per check, in a stable order
37
+ def verify
38
+ [distro, docker, docker_group, state_dir, deploy_log]
39
+ end
40
+
41
+ private
42
+
43
+ def root?
44
+ @ssh.user == ROOT
45
+ end
46
+
47
+ def host_paths
48
+ @host_paths ||= Odysseus::HostPaths.new(@ssh)
49
+ end
50
+
51
+ def distro
52
+ os = read_os_release
53
+ id = os['ID']
54
+ version = os['VERSION_ID']
55
+
56
+ if id == 'ubuntu' && SUPPORTED_UBUNTU.include?(version)
57
+ Result.new(check: :distro, status: :ok, detail: "ubuntu #{version}")
58
+ else
59
+ # A warning, not a failure: odysseus deploys to any host with Docker.
60
+ # SUPPORTED_UBUNTU is what odysseus is tested against, not a
61
+ # requirement.
62
+ Result.new(
63
+ check: :distro, status: :warn,
64
+ detail: "#{id || 'unknown'} #{version}".strip +
65
+ " — not ubuntu #{SUPPORTED_UBUNTU.join(' or ')}; deploys work anywhere Docker does"
66
+ )
67
+ end
68
+ end
69
+
70
+ def docker
71
+ output = @ssh.execute("docker info --format '{{.ServerVersion}}' 2>&1 || true")
72
+ version = output.to_s.strip
73
+
74
+ if version.match?(/\A\d+\./)
75
+ Result.new(check: :docker, status: :ok, detail: "docker #{version}")
76
+ else
77
+ Result.new(
78
+ check: :docker, status: :fail,
79
+ detail: "the docker daemon did not answer as #{@ssh.user}: #{version.lines.first.to_s.strip}"
80
+ )
81
+ end
82
+ end
83
+
84
+ def docker_group
85
+ # root does not need the group, and saying it is missing would be a false
86
+ # alarm on every root install — still the default.
87
+ return Result.new(check: :docker_group, status: :ok, detail: 'not needed for root') if root?
88
+
89
+ groups = @ssh.execute("id -nG #{Shellwords.escape(@ssh.user)} 2>/dev/null || true").to_s.split
90
+
91
+ if groups.include?('docker')
92
+ Result.new(check: :docker_group, status: :ok, detail: "#{@ssh.user} is in the docker group")
93
+ else
94
+ Result.new(
95
+ check: :docker_group, status: :fail,
96
+ detail: "#{@ssh.user} is not in the docker group (has: #{groups.join(' ')})"
97
+ )
98
+ end
99
+ end
100
+
101
+ def state_dir
102
+ dir = host_paths.base
103
+ # Check the nearest existing ancestor: the directory itself may legitimately
104
+ # not exist yet on a host that has never deployed, and creating it to find
105
+ # out is exactly what this command must not do.
106
+ probe = Shellwords.escape(dir)
107
+ output = @ssh.execute(
108
+ "d=#{probe}; while [ ! -e \"$d\" ] && [ \"$d\" != / ]; do d=$(dirname \"$d\"); done; " \
109
+ 'if test -w "$d"; then echo writable; else echo "not writable:$d"; fi'
110
+ ).to_s.strip
111
+
112
+ if output == 'writable'
113
+ Result.new(check: :state_dir, status: :ok, detail: "#{dir} is writable")
114
+ else
115
+ Result.new(
116
+ check: :state_dir, status: :fail,
117
+ detail: "#{dir} is not writable by #{@ssh.user} (#{output.split(':').last} is not)"
118
+ )
119
+ end
120
+ end
121
+
122
+ def deploy_log
123
+ path = File.join(host_paths.service_dir(@config[:service]), Odysseus::DeployLog::FILENAME)
124
+ legacy = File.join(host_paths.legacy_base, @config[:service], Odysseus::DeployLog::FILENAME)
125
+
126
+ # For root, path and legacy are always the same string: HostPaths#base
127
+ # resolves to legacy_base for root, so this already covers root without
128
+ # a separate root? check — one that could never change the outcome.
129
+ return Result.new(check: :deploy_log, status: :ok, detail: path) if legacy == path
130
+
131
+ present = @ssh.execute("test -e #{Shellwords.escape(legacy)} && echo present || echo absent").to_s.strip
132
+
133
+ if present == 'present'
134
+ # Losing this is silent — rollback just offers fewer versions — so it is
135
+ # worth a warning rather than a note.
136
+ Result.new(
137
+ check: :deploy_log, status: :warn,
138
+ detail: "#{path}; root-era history still at #{legacy}, readable but not writable by #{@ssh.user}"
139
+ )
140
+ else
141
+ Result.new(check: :deploy_log, status: :ok, detail: path)
142
+ end
143
+ end
144
+
145
+ def read_os_release
146
+ raw = @ssh.execute('cat /etc/os-release 2>/dev/null || true').to_s
147
+
148
+ raw.lines.each_with_object({}) do |line, acc|
149
+ key, value = line.strip.split('=', 2)
150
+ next if key.nil? || value.nil?
151
+
152
+ acc[key] = value.delete('"')
153
+ end
154
+ end
155
+ end
156
+ end
@@ -0,0 +1,59 @@
1
+ # lib/odysseus/host_versions.rb
2
+
3
+ module Odysseus
4
+ # What one host knows about one service's versions: what is serving now, what
5
+ # it could serve (images present), and what it has served (its deploy log).
6
+ #
7
+ # Hosts are authoritative for rollback. A developer's checkout and the git
8
+ # notes history can both drift from what a host can actually run — a rebuilt
9
+ # host, a pruned image, a change made by hand — so the target is chosen from
10
+ # this, never from the repository.
11
+ HostVersions = Data.define(:host, :current, :available, :history) do
12
+ # Read one host's state. Caller owns the connection and closes it.
13
+ #
14
+ # @param host [String] host name, for reporting
15
+ # @param ssh [Odysseus::Deployer::SSH] open connection to that host
16
+ # @param service [String] the service name from deploy.yml
17
+ # @param image [String] the repository from deploy.yml, without a tag
18
+ # @param roles [Array<Symbol>] the roles this host serves, in config order.
19
+ # current scans them in this order and stops at the first that is
20
+ # serving, so a host whose web role is down but whose jobs role is up
21
+ # still reports something rather than nil.
22
+ # @return [HostVersions]
23
+ def self.read(host:, ssh:, service:, image:, roles:)
24
+ docker = Odysseus::Docker::Client.new(ssh)
25
+
26
+ new(
27
+ host: host,
28
+ current: current_version(docker, service, roles),
29
+ available: docker.image_tags(image),
30
+ history: Odysseus::DeployLog.new(ssh: ssh, service: service).entries
31
+ )
32
+ end
33
+
34
+ # The version label of the first running container of the first role, in
35
+ # role order, that is actually serving. WebDeploy and JobDeploy label
36
+ # their containers differently (see Docker::Labels.service_for), so each
37
+ # role must be queried under its own label or a non-web role reports
38
+ # nothing running.
39
+ def self.current_version(docker, service, roles)
40
+ roles.each do |role|
41
+ label = Odysseus::Docker::Labels.service_for(service: service, role: role)
42
+ version = docker.list(service: label)
43
+ .filter_map { |container| Odysseus::Docker::Labels.version_of(container) }
44
+ .first
45
+ return version if version
46
+ end
47
+
48
+ nil
49
+ end
50
+
51
+ private_class_method :current_version
52
+
53
+ # @param version [String]
54
+ # @return [Boolean] whether this host has an image tagged that way
55
+ def available?(version)
56
+ available.include?(version)
57
+ end
58
+ end
59
+ end