empeira 0.1.0.pre.alpha.6 → 0.1.0.pre.alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/config/defaults.yaml +1 -0
- data/docs/architecture.md +18 -0
- data/docs/configuration.md +5 -1
- data/docs/nodes.md +9 -3
- data/docs/proxy.md +52 -0
- data/lib/empeira/build.json +3 -3
- data/lib/empeira/configuration/schema.rb +2 -1
- data/lib/empeira/control_plane/controller.rb +1 -2
- data/lib/empeira/control_plane/health.rb +11 -0
- data/lib/empeira/control_plane/plan.rb +1 -1
- data/lib/empeira/control_plane/startup_failure.rb +43 -0
- data/lib/empeira/infrastructure/cleanup.rb +6 -0
- data/lib/empeira/infrastructure/session_lock.rb +60 -0
- data/lib/empeira/network/proxy_policy.rb +1 -1
- data/lib/empeira/node/container_lifecycle.rb +41 -23
- data/lib/empeira/node/session_lock.rb +19 -0
- data/lib/empeira/runtime/service_inspection.rb +5 -1
- data/lib/empeira/vm/session_lock.rb +2 -46
- data/lib/empeira.rb +3 -0
- data/resources/nodes/management/guest_agent.py +4 -0
- metadata +4 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3f0b0c2558d0799e27a8985ab84de156e8de49741ec078965338d0c1638ab0d3
|
|
4
|
+
data.tar.gz: 5607b7c52a5fb130b7a8534318aecb89c0093709ea7d9b65fcccef86da745fab
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5a0eaa2c61a33c12d49632cfa4690f9e369bbae26a1d4cf4f9b1c5ae0d3178d6aed0f14f28721d1a83e1332319b02cf5a899f8f31287b88816b2e2e7ba3e8366
|
|
7
|
+
data.tar.gz: ed8ab5306f5948ed1f81c8e1b00121cb6360490e35daf915b95c441a42174552320b08fcc37a193eb2e1011b694eb1f85932739df3ee0d2274b182fded0b6957
|
data/config/defaults.yaml
CHANGED
data/docs/architecture.md
CHANGED
|
@@ -93,6 +93,24 @@ system SSH. Guest commands use argument arrays and owned process groups; stdout
|
|
|
93
93
|
and stderr stream in bounded frames without truncation. Timeouts, interrupts and
|
|
94
94
|
host disconnects terminate those groups. Binary uploads are SHA-256 verified,
|
|
95
95
|
staged privately and atomically installed in the real guest filesystem.
|
|
96
|
+
The management supervisor and workers retain `umask 077`, private 0700 staging and
|
|
97
|
+
explicit upload modes. Only executed commands (including Puppet and its children)
|
|
98
|
+
and PTY shells start with `umask 022`. Previously the worker's private mask leaked
|
|
99
|
+
into these children, causing ordinary file/directory creation to yield 0600/0700;
|
|
100
|
+
APT could not read public keys created by commands that omitted an explicit mode.
|
|
101
|
+
Commands still run as numeric root, and explicit private modes remain private.
|
|
102
|
+
Empeira's native APT acquisition already copies its own signing key with mode 0644
|
|
103
|
+
under `/var/tmp`; it does not create project repository keys under
|
|
104
|
+
`/usr/share/keyrings` or `/etc/apt/keyrings`.
|
|
105
|
+
|
|
106
|
+
This correction applies to newly seeded VMs. Recreate disposable existing VMs to
|
|
107
|
+
receive the updated adapter; `up` does not replace seeded management code or repair
|
|
108
|
+
existing files. No directory-wide chmod or post-Puppet keyring rewrite is performed.
|
|
109
|
+
A catalog explicitly setting mode 0600, or a tool using private temporary files,
|
|
110
|
+
still needs correction at its owning resource. Project-managed public APT keys
|
|
111
|
+
should specify root:root and 0644, with searchable parent directories. The reported
|
|
112
|
+
Docker/Elastic/project key names are absent from Empeira's source; their exact
|
|
113
|
+
catalog resources require inspection in the affected project.
|
|
96
114
|
|
|
97
115
|
The QEMU monitor verifies workspace/node/instance identity before management.
|
|
98
116
|
A 0700 host directory and 0600 Unix socket restrict access; recorded device/inode
|
data/docs/configuration.md
CHANGED
|
@@ -159,6 +159,7 @@ release version. A partial project mapping may omit it and inherit the default.
|
|
|
159
159
|
| `network.egress.nodes` | Node-only direct TCP host/port grants, `[]`; arrays replace |
|
|
160
160
|
| `network.egress.infrastructure` | Infrastructure-only direct TCP host/port grants, `[]`; arrays replace |
|
|
161
161
|
| `network.redirects` | Exact IPv4/protocol/port endpoints redirected to enabled internal services, `[]` |
|
|
162
|
+
| `proxy.memory` | Positive integer MiB, `512` per Squid container; shared regular proxy and separate temporary bootstrap proxy each use this limit; does not enable either proxy |
|
|
162
163
|
| `proxy.nodes.enabled` | Boolean, `false`; enables normal node HTTP/HTTPS proxy access |
|
|
163
164
|
| `proxy.nodes.global` | DNS destination array, `[]` |
|
|
164
165
|
| `proxy.nodes.rules` | Additive node hostname-glob rules with `hosts` and `allow` arrays, `[]` |
|
|
@@ -970,7 +971,10 @@ The default control plane has six containers, or seven with the proxy, plus the
|
|
|
970
971
|
runtime's peer-network adapter. Requested limits
|
|
971
972
|
are 1536 MiB for the server (768 MiB JVM heap), 768 MiB for PuppetDB (256 MiB heap),
|
|
972
973
|
256 MiB for PostgreSQL, 64 MiB for DNS, 96 MiB for the database relay, 64 MiB
|
|
973
|
-
for the gateway and
|
|
974
|
+
for the gateway and 512 MiB for the regular Squid proxy (`proxy.memory`). A temporary
|
|
975
|
+
bootstrap Squid container has its own additional limit of the same size while
|
|
976
|
+
running. These are per-container limits, not reservations or measured peaks.
|
|
977
|
+
The control-plane memory ceiling is about
|
|
974
978
|
2.7 GiB without the proxy, excluding the peer adapter and nodes. The server gets two CPUs and each
|
|
975
979
|
other service one. Actual idle use is lower and varies by provider and platform.
|
|
976
980
|
`up` automatically pulls missing remote images and builds missing recipe images.
|
data/docs/nodes.md
CHANGED
|
@@ -296,12 +296,18 @@ Empeira uses the private privileged VirtIO channel internally to install the VM
|
|
|
296
296
|
enroll certificates and run Puppet. User-facing SSH opens an ordinary interactive
|
|
297
297
|
login without the internal provisioning command or automatic sudo.
|
|
298
298
|
|
|
299
|
-
VM exec, shell, SSH and serial sessions release the workspace
|
|
300
|
-
running-state checks. Parallel sessions and read-only status remain available. A
|
|
299
|
+
Container SSH and VM exec, shell, SSH and serial sessions release the workspace
|
|
300
|
+
mutation lock after ownership and running-state checks. Parallel sessions and read-only status remain available. A
|
|
301
301
|
shared instance guard prevents `node stop`, `node destroy` or restart from replacing
|
|
302
|
-
|
|
302
|
+
that node instance while a session is attached; detach first and retry the conflicting operation.
|
|
303
303
|
Other nodes and workspace commands can proceed. Instance lock files stay stable;
|
|
304
304
|
their open descriptors are released on detach, disconnect, errors and signals.
|
|
305
|
+
Container SSH preparation remains inside the workspace transaction and pins the
|
|
306
|
+
observed container ID before releasing that lock. Concurrent SSH sessions to the
|
|
307
|
+
same or different nodes are allowed. Container `exec` and `shell` retain their
|
|
308
|
+
existing workspace transaction. Mutations still use the exclusive workspace lock;
|
|
309
|
+
`down` continues to require node removal, and destroy retains attached instances
|
|
310
|
+
for a retry after sessions detach.
|
|
305
311
|
|
|
306
312
|
## Root diagnostic commands and interactive shell
|
|
307
313
|
|
data/docs/proxy.md
CHANGED
|
@@ -6,6 +6,7 @@ the node scope makes its global and hostname rules available to nodes:
|
|
|
6
6
|
|
|
7
7
|
```yaml
|
|
8
8
|
proxy:
|
|
9
|
+
memory: 512 # MiB per regular or temporary bootstrap Squid container
|
|
9
10
|
nodes:
|
|
10
11
|
enabled: true
|
|
11
12
|
global:
|
|
@@ -48,6 +49,57 @@ The diagnostic resolves desired policy even for a hostname that does not exist.
|
|
|
48
49
|
It reports when the normal proxy is disabled and does not claim that live
|
|
49
50
|
infrastructure has already been reconciled.
|
|
50
51
|
|
|
52
|
+
## Memory budget and startup diagnostics
|
|
53
|
+
|
|
54
|
+
`proxy.memory` is a positive integer MiB limit per managed Squid container,
|
|
55
|
+
defaulting to 512. Node access and infrastructure access share one regular proxy;
|
|
56
|
+
the same limit applies when either or both scopes are enabled. The temporary,
|
|
57
|
+
authenticated bootstrap proxy inherits this value as its own container limit.
|
|
58
|
+
If both containers run simultaneously, each has a separate 512 MiB limit by
|
|
59
|
+
default (1024 MiB combined), not one shared 512 MiB budget. These limits are
|
|
60
|
+
neither memory reservations nor measured peak consumption.
|
|
61
|
+
|
|
62
|
+
Existing configurations inherit the default. Setting `proxy.memory` alone does
|
|
63
|
+
not enable a regular proxy or start a bootstrap proxy. Node and infrastructure
|
|
64
|
+
access keep independent activation switches and destination grants. Bootstrap
|
|
65
|
+
keeps its separate temporary activation, authentication and destination grants.
|
|
66
|
+
Docker and Podman use the same native
|
|
67
|
+
`--memory` argument, including rootless Podman. The engine host (or its Linux VM on
|
|
68
|
+
macOS) still needs enough available memory and working cgroup memory controls.
|
|
69
|
+
|
|
70
|
+
The former fixed 128 MiB limit was below the reported killed process's approximately
|
|
71
|
+
139 MiB resident footprint (anonymous plus file RSS), before accounting for other
|
|
72
|
+
cgroup charges. The 512 MiB default provides headroom for startup, concurrent
|
|
73
|
+
requests, DNS and allocator overhead. It is a conservative budget, not a measured
|
|
74
|
+
maximum or a guarantee for arbitrary workloads. Validate actual peak usage with
|
|
75
|
+
native container stats and host cgroup/OOM diagnostics in the affected environment.
|
|
76
|
+
|
|
77
|
+
Generated policy keeps `cache deny all`, sets `cache_mem 16 MB`, and explicitly uses
|
|
78
|
+
`workers 1` together with `-N` for one foreground main process. An explicit
|
|
79
|
+
`workers 0` causes Squid 7.7 to exit during startup with
|
|
80
|
+
`FATAL: check failed: theProcessCount > 0`; `-N` already selects foreground mode.
|
|
81
|
+
No disk cache is configured. Squid's [cache_mem documentation](https://www.squid-cache.org/Doc/config/cache_mem/)
|
|
82
|
+
describes an object-memory target, not a cap on process RSS; in-flight requests can
|
|
83
|
+
exceed it. The [workers setting](https://www.squid-cache.org/Doc/config/workers/)
|
|
84
|
+
avoids multiplying main-process memory. These settings preserve DNS, source-bound
|
|
85
|
+
client rules and the HTTP/HTTPS destination policy.
|
|
86
|
+
|
|
87
|
+
If the regular proxy exits during startup or its readiness check, diagnostics
|
|
88
|
+
include the native exit code, `OOMKilled` observation and actual inspected memory
|
|
89
|
+
limit in bytes/MiB when available. Only `OOMKilled=true` confirms an OOM kill. Exit 137 alone does not:
|
|
90
|
+
external SIGKILL can produce the same exit. Missing observations are explicitly
|
|
91
|
+
reported as unavailable, and a runtime-reported zero limit is shown as unlimited.
|
|
92
|
+
For confirmed proxy OOM, increase `proxy.memory`, check engine-host capacity, and
|
|
93
|
+
run `empeira up`. Native logs remain available via the displayed runtime command.
|
|
94
|
+
The temporary bootstrap proxy uses its own endpoint verification and readiness
|
|
95
|
+
probe; the new regular-proxy OOM diagnostics do not apply to that separate path.
|
|
96
|
+
|
|
97
|
+
A memory change replaces only the regular proxy on the next `up`; CA/database
|
|
98
|
+
storage, nodes and unrelated services retain their identities. Unchanged runs
|
|
99
|
+
reuse the proxy. Generated policy changes use the existing readiness/reload checkpoint.
|
|
100
|
+
Replacing or reloading the proxy may interrupt active proxy connections.
|
|
101
|
+
The next temporary bootstrap proxy receives the configured limit when created.
|
|
102
|
+
|
|
51
103
|
## Matching and destinations
|
|
52
104
|
|
|
53
105
|
Defaults are `proxy.nodes.global: []` and `proxy.nodes.rules: []`. Every node receives
|
data/lib/empeira/build.json
CHANGED
|
@@ -22,7 +22,8 @@ module Empeira
|
|
|
22
22
|
'puppetdb' => { 'enabled' => :boolean, 'memory' => :positive_integer, 'runtime' => :puppetdb_runtime },
|
|
23
23
|
'network' => { 'egress' => { 'nodes' => :network_egress, 'infrastructure' => :network_egress },
|
|
24
24
|
'redirects' => :network_redirects },
|
|
25
|
-
'proxy' => { '
|
|
25
|
+
'proxy' => { 'memory' => :positive_integer,
|
|
26
|
+
'nodes' => { 'enabled' => :boolean, 'global' => :allowlist, 'rules' => :proxy_rules },
|
|
26
27
|
'infrastructure' => { 'enabled' => :boolean, 'allow' => :allowlist } },
|
|
27
28
|
'vm' => { 'disk' => :vm_disk, 'interfaces' => :vm_interfaces,
|
|
28
29
|
'console' => { 'root_password' => :optional_string } },
|
|
@@ -528,8 +528,7 @@ module Empeira
|
|
|
528
528
|
def require_started!(key, resource)
|
|
529
529
|
return if resource.fetch('state') == 'running'
|
|
530
530
|
|
|
531
|
-
raise
|
|
532
|
-
"inspect #{@context.container_engine} logs #{resource.fetch('id')}"
|
|
531
|
+
raise StartupFailure.new(key: key, resource: resource, engine: @context.container_engine)
|
|
533
532
|
end
|
|
534
533
|
|
|
535
534
|
def update_hosts(reload_proxy: false)
|
|
@@ -14,6 +14,8 @@ module Empeira
|
|
|
14
14
|
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @plan.config.dig('server', 'timeout')
|
|
15
15
|
loop do
|
|
16
16
|
return if ready?(key, resources)
|
|
17
|
+
|
|
18
|
+
verify_proxy_running!(resources) if key == 'proxy'
|
|
17
19
|
if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
18
20
|
raise Error,
|
|
19
21
|
"#{key} did not become ready within server.timeout; inspect empeira status and owned service logs\n" \
|
|
@@ -52,6 +54,15 @@ module Empeira
|
|
|
52
54
|
|
|
53
55
|
private
|
|
54
56
|
|
|
57
|
+
def verify_proxy_running!(resources)
|
|
58
|
+
resource = @runtime.inspect_service(@plan.definitions.fetch('proxy'),
|
|
59
|
+
expected_id: resources.fetch('proxy').fetch('id'))
|
|
60
|
+
raise Error, 'Managed service proxy disappeared during readiness' unless resource
|
|
61
|
+
return if resource.fetch('state') == 'running'
|
|
62
|
+
|
|
63
|
+
raise StartupFailure.new(key: 'proxy', resource: resource, engine: @plan.context.container_engine)
|
|
64
|
+
end
|
|
65
|
+
|
|
55
66
|
def puppetdb_ready?(server)
|
|
56
67
|
path = '/pdb/query/v4/nodes'
|
|
57
68
|
check(server, [*http_curl, "http://puppetdb.empeira.internal:8080#{path}"]) &&
|
|
@@ -330,7 +330,7 @@ module Empeira
|
|
|
330
330
|
configured = config.dig('images', 'proxy')
|
|
331
331
|
{
|
|
332
332
|
**Images::Configuration.artifact(configured, registry: config.dig('images', 'registry')),
|
|
333
|
-
memory:
|
|
333
|
+
memory: config.dig('proxy', 'memory'), runtime_ip: subnet && Network::Gateway.address(subnet, 'proxy'),
|
|
334
334
|
sysctls: { 'net.ipv4.ip_forward' => '0', 'net.ipv6.conf.all.disable_ipv6' => '1',
|
|
335
335
|
'net.ipv6.conf.default.disable_ipv6' => '1' }, entrypoint: '/usr/sbin/squid-gnutls',
|
|
336
336
|
command: ['-N', '-f', '/empeira-proxy/squid.conf'],
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Empeira
|
|
4
|
+
module ControlPlane
|
|
5
|
+
# Report only native observations; SIGKILL alone does not establish an OOM kill.
|
|
6
|
+
class StartupFailure < Error
|
|
7
|
+
def initialize(key:, resource:, engine:)
|
|
8
|
+
@resource = resource
|
|
9
|
+
message = "Managed service #{key} is #{resource.fetch('state')} after start; " \
|
|
10
|
+
"inspect #{engine} logs #{resource.fetch('id')}; " \
|
|
11
|
+
"#{exit_diagnostic}; #{oom_diagnostic}; #{memory_diagnostic}"
|
|
12
|
+
if key == 'proxy' && resource['oom_killed'] == true
|
|
13
|
+
message += '; increase proxy.memory (MiB) in .empeira.yaml and run empeira up; ' \
|
|
14
|
+
'check available memory in the container engine host/VM'
|
|
15
|
+
end
|
|
16
|
+
super(Execution::Diagnostics.safe_text(message))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
def exit_diagnostic
|
|
22
|
+
code = @resource['exit_code']
|
|
23
|
+
code.is_a?(Integer) ? "exit code #{code}" : 'exit code unavailable'
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def oom_diagnostic
|
|
27
|
+
case @resource['oom_killed']
|
|
28
|
+
when true then 'confirmed OOM kill (OOMKilled=true)'
|
|
29
|
+
when false then 'OOMKilled=false'
|
|
30
|
+
else 'OOM status unavailable'
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def memory_diagnostic
|
|
35
|
+
bytes = @resource.dig('host_config', 'Memory')
|
|
36
|
+
return 'memory limit unavailable' unless bytes.is_a?(Integer) && bytes >= 0
|
|
37
|
+
return 'memory limit unlimited (runtime reports 0)' if bytes.zero?
|
|
38
|
+
|
|
39
|
+
"memory limit #{bytes} bytes (#{format('%g', bytes.fdiv(1024 * 1024))} MiB)"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -107,6 +107,12 @@ module Empeira
|
|
|
107
107
|
end
|
|
108
108
|
|
|
109
109
|
def remove_container(record)
|
|
110
|
+
return remove_container_instance(record) unless record['id']
|
|
111
|
+
|
|
112
|
+
Node::SessionLock.new(context: @context, record: record).exclusive { remove_container_instance(record) }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def remove_container_instance(record)
|
|
110
116
|
name = record.fetch('hostname')
|
|
111
117
|
path = @store.directory.join('containers', name)
|
|
112
118
|
OwnedFiles.verify_removable!(path, root: @store.directory)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Empeira
|
|
4
|
+
module Infrastructure
|
|
5
|
+
# Pin one node instance during access without retaining the workspace mutation lock.
|
|
6
|
+
class SessionLock
|
|
7
|
+
def initialize(path:, subject:)
|
|
8
|
+
@path = path
|
|
9
|
+
@subject = subject
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def acquire(shared:)
|
|
13
|
+
lock = open_lock
|
|
14
|
+
mode = shared ? File::LOCK_SH : File::LOCK_EX
|
|
15
|
+
unless lock.flock(mode | File::LOCK_NB)
|
|
16
|
+
raise Infrastructure::Locked,
|
|
17
|
+
"#{@subject} instance has active access or lifecycle operations; detach and retry"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
lock.close_on_exec = true
|
|
21
|
+
acquired = true
|
|
22
|
+
lock
|
|
23
|
+
ensure
|
|
24
|
+
lock&.close unless acquired
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def exclusive
|
|
28
|
+
lock = acquire(shared: false)
|
|
29
|
+
yield
|
|
30
|
+
ensure
|
|
31
|
+
lock&.close
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def open_lock
|
|
37
|
+
if @path.dirname.symlink? || @path.symlink?
|
|
38
|
+
raise Providers::OwnershipError,
|
|
39
|
+
"#{@subject} session lock path is unsafe"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# A successful open passes descriptor ownership to acquire.
|
|
43
|
+
# rubocop:disable-next Style/FileOpen
|
|
44
|
+
lock = File.open(@path, File::RDWR | File::CREAT | File::NOFOLLOW, 0o600)
|
|
45
|
+
verify_lock!(lock.stat)
|
|
46
|
+
|
|
47
|
+
opened = true
|
|
48
|
+
lock
|
|
49
|
+
ensure
|
|
50
|
+
lock&.close unless opened
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def verify_lock!(metadata)
|
|
54
|
+
return if metadata.file? && metadata.uid == Process.uid && metadata.mode.nobits?(0o077)
|
|
55
|
+
|
|
56
|
+
raise Providers::OwnershipError, "#{@subject} session lock ownership is invalid"
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -27,7 +27,7 @@ module Empeira
|
|
|
27
27
|
def configuration
|
|
28
28
|
[
|
|
29
29
|
'dns_nameservers @EMPEIRA_DNS@', 'http_port 3128',
|
|
30
|
-
'visible_hostname empeira-proxy', 'cache deny all',
|
|
30
|
+
'visible_hostname empeira-proxy', 'cache deny all', 'cache_mem 16 MB', 'workers 1',
|
|
31
31
|
'access_log none', 'cache_log /dev/null', 'pid_filename none',
|
|
32
32
|
client_rule, 'http_access deny !clients',
|
|
33
33
|
'request_header_access Proxy-Authorization deny all',
|
|
@@ -23,12 +23,15 @@ module Empeira
|
|
|
23
23
|
|
|
24
24
|
def stop(name:)
|
|
25
25
|
mutate do
|
|
26
|
-
|
|
26
|
+
record = fetch(name)
|
|
27
|
+
resource = observed(record)
|
|
27
28
|
raise Error, 'Node is missing; inspect node list' unless resource
|
|
28
29
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
SessionLock.new(context: context, record: record).exclusive do
|
|
31
|
+
changed = resource['state'] == 'running'
|
|
32
|
+
@runtime.stop_service(resource) if changed
|
|
33
|
+
lifecycle_result(name.downcase, :stopped, changed: changed)
|
|
34
|
+
end
|
|
32
35
|
end
|
|
33
36
|
end
|
|
34
37
|
|
|
@@ -73,27 +76,19 @@ module Empeira
|
|
|
73
76
|
end
|
|
74
77
|
end
|
|
75
78
|
|
|
76
|
-
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- Verify the owned endpoint before selecting a guest tunnel.
|
|
77
79
|
def ssh(name:, user: nil, identity: nil, port: nil)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
resource = running(
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
proxy = if port != 22
|
|
87
|
-
@runtime.ssh_proxy_command(resource, port: port)
|
|
88
|
-
elsif record['ssh_transport'] == 'tunnel'
|
|
89
|
-
@runtime.ssh_proxy_command(resource)
|
|
90
|
-
end
|
|
91
|
-
record = record.merge('ssh_port' => port) if port != 22
|
|
92
|
-
UserSSH.new(runner: @runner, credentials: ssh_credentials(record.fetch('hostname')),
|
|
93
|
-
proxy_command: proxy, home: context.locations.home).session(
|
|
94
|
-
record, user: user, identity: identity
|
|
95
|
-
)
|
|
80
|
+
lock = nil
|
|
81
|
+
client, record = mutate do
|
|
82
|
+
current = fetch(name)
|
|
83
|
+
resource = running(current)
|
|
84
|
+
lock = SessionLock.new(context: context, record: current).acquire(shared: true)
|
|
85
|
+
prepare_ssh(current, resource, port)
|
|
96
86
|
end
|
|
87
|
+
Execution::Terminal.with_interrupts do
|
|
88
|
+
@progress.streaming { client.session(record, user: user, identity: identity) }
|
|
89
|
+
end
|
|
90
|
+
ensure
|
|
91
|
+
lock&.close
|
|
97
92
|
end
|
|
98
93
|
|
|
99
94
|
def logs(name:)
|
|
@@ -107,6 +102,22 @@ module Empeira
|
|
|
107
102
|
|
|
108
103
|
private
|
|
109
104
|
|
|
105
|
+
def prepare_ssh(record, resource, port)
|
|
106
|
+
port = Configuration::SSHPreferences.port!(port || 22)
|
|
107
|
+
unless record['ssh_host'] && record['ssh_port']
|
|
108
|
+
raise Error, 'Node has no managed SSH endpoint; destroy and recreate it with the current Empeira image'
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
proxy = if port != 22
|
|
112
|
+
@runtime.ssh_proxy_command(resource, port: port)
|
|
113
|
+
elsif record['ssh_transport'] == 'tunnel'
|
|
114
|
+
@runtime.ssh_proxy_command(resource)
|
|
115
|
+
end
|
|
116
|
+
client = UserSSH.new(runner: @runner, credentials: ssh_credentials(record.fetch('hostname')),
|
|
117
|
+
proxy_command: proxy, home: context.locations.home)
|
|
118
|
+
[client, port == 22 ? record : record.merge('ssh_port' => port)]
|
|
119
|
+
end
|
|
120
|
+
|
|
110
121
|
def require_provisioned!(record)
|
|
111
122
|
return if record['provisioned']
|
|
112
123
|
|
|
@@ -117,6 +128,13 @@ module Empeira
|
|
|
117
128
|
record = @nodes[name]
|
|
118
129
|
return unless record
|
|
119
130
|
|
|
131
|
+
SessionLock.new(context: context, record: record).exclusive do
|
|
132
|
+
remove_node_instance(name, record, certificates: certificates)
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# rubocop:disable-next Naming/PredicateMethod -- Removes an owned instance and persists its absence.
|
|
137
|
+
def remove_node_instance(name, record, certificates:)
|
|
120
138
|
observed(record)
|
|
121
139
|
if certificates && record['certificate_key']
|
|
122
140
|
Certificates.new(runtime: @runtime, server: ready_server).clean(record)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
|
|
5
|
+
module Empeira
|
|
6
|
+
module Node
|
|
7
|
+
# Native container IDs pin access to an observed instance, never just its reusable name.
|
|
8
|
+
class SessionLock < Infrastructure::SessionLock
|
|
9
|
+
def initialize(context:, record:)
|
|
10
|
+
id = record.fetch('id')
|
|
11
|
+
raise Providers::OwnershipError, 'Container session requires a recorded instance ID' unless
|
|
12
|
+
id.is_a?(String) && !id.empty?
|
|
13
|
+
|
|
14
|
+
directory = context.locations.workspace(context.workspace)
|
|
15
|
+
super(path: directory.join("container-session-#{Digest::SHA256.hexdigest(id)}.lock"), subject: 'Container')
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -147,7 +147,11 @@ module Empeira
|
|
|
147
147
|
'networks' => data.fetch('NetworkSettings').fetch('Networks'),
|
|
148
148
|
'host_config' => data['HostConfig'], 'image_id' => data['Image'],
|
|
149
149
|
'mounts' => data.fetch('Mounts'), 'dns' => data.dig('HostConfig', 'Dns') || [],
|
|
150
|
-
**port_observations(data) }
|
|
150
|
+
**port_observations(data), **exit_observations(data) }
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def exit_observations(data)
|
|
154
|
+
{ 'exit_code' => data.dig('State', 'ExitCode'), 'oom_killed' => data.dig('State', 'OOMKilled') }
|
|
151
155
|
end
|
|
152
156
|
|
|
153
157
|
def port_observations(data)
|
|
@@ -2,54 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
module Empeira
|
|
4
4
|
module VM
|
|
5
|
-
|
|
6
|
-
class SessionLock
|
|
5
|
+
class SessionLock < Infrastructure::SessionLock
|
|
7
6
|
def initialize(context:, record:)
|
|
8
7
|
directory = context.locations.workspace(context.workspace)
|
|
9
|
-
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def acquire(shared:)
|
|
13
|
-
lock = open_lock
|
|
14
|
-
mode = shared ? File::LOCK_SH : File::LOCK_EX
|
|
15
|
-
unless lock.flock(mode | File::LOCK_NB)
|
|
16
|
-
raise Infrastructure::Locked, 'VM instance has active access or lifecycle operations; detach and retry'
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
lock.close_on_exec = true
|
|
20
|
-
acquired = true
|
|
21
|
-
lock
|
|
22
|
-
ensure
|
|
23
|
-
lock&.close unless acquired
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def exclusive
|
|
27
|
-
lock = acquire(shared: false)
|
|
28
|
-
yield
|
|
29
|
-
ensure
|
|
30
|
-
lock&.close
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
private
|
|
34
|
-
|
|
35
|
-
def open_lock
|
|
36
|
-
raise Providers::OwnershipError, 'VM session lock path is unsafe' if @path.dirname.symlink? || @path.symlink?
|
|
37
|
-
|
|
38
|
-
# A successful open passes descriptor ownership to acquire.
|
|
39
|
-
# rubocop:disable-next Style/FileOpen
|
|
40
|
-
lock = File.open(@path, File::RDWR | File::CREAT, 0o600)
|
|
41
|
-
verify_lock!(lock.stat)
|
|
42
|
-
|
|
43
|
-
opened = true
|
|
44
|
-
lock
|
|
45
|
-
ensure
|
|
46
|
-
lock&.close unless opened
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
def verify_lock!(metadata)
|
|
50
|
-
return if metadata.file? && metadata.uid == Process.uid && metadata.mode.nobits?(0o077)
|
|
51
|
-
|
|
52
|
-
raise Providers::OwnershipError, 'VM session lock ownership is invalid'
|
|
8
|
+
super(path: directory.join("vm-session-#{record.fetch('peer').fetch('token')}.lock"), subject: 'VM')
|
|
53
9
|
end
|
|
54
10
|
end
|
|
55
11
|
end
|
data/lib/empeira.rb
CHANGED
|
@@ -94,7 +94,9 @@ require_relative 'empeira/vm/image_cache'
|
|
|
94
94
|
require_relative 'empeira/vm/disk'
|
|
95
95
|
require_relative 'empeira/vm/root_disk'
|
|
96
96
|
require_relative 'empeira/vm/qemu_runtime'
|
|
97
|
+
require_relative 'empeira/infrastructure/session_lock'
|
|
97
98
|
require_relative 'empeira/vm/session_lock'
|
|
99
|
+
require_relative 'empeira/node/session_lock'
|
|
98
100
|
require_relative 'empeira/vm/removal'
|
|
99
101
|
require_relative 'empeira/node/ssh_credentials'
|
|
100
102
|
require_relative 'empeira/node/ssh_client'
|
|
@@ -190,6 +192,7 @@ require_relative 'empeira/infrastructure/status'
|
|
|
190
192
|
require_relative 'empeira/control_plane/health'
|
|
191
193
|
require_relative 'empeira/control_plane/storage'
|
|
192
194
|
require_relative 'empeira/control_plane/discovery'
|
|
195
|
+
require_relative 'empeira/control_plane/startup_failure'
|
|
193
196
|
require_relative 'empeira/control_plane/controller'
|
|
194
197
|
require_relative 'empeira/infrastructure/service'
|
|
195
198
|
require_relative 'empeira/updates/images'
|
|
@@ -159,6 +159,8 @@ def start_exec(message):
|
|
|
159
159
|
try:
|
|
160
160
|
process = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
161
161
|
stderr=subprocess.PIPE, start_new_session=True,
|
|
162
|
+
# Workers are single-threaded; support guest Python 3.6 as well.
|
|
163
|
+
preexec_fn=lambda: os.umask(0o022),
|
|
162
164
|
env={**{key: value for key, value in os.environ.items() if key != 'RUNTIME_DIRECTORY'}, 'PATH': '/opt/puppetlabs/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'})
|
|
163
165
|
except OSError as error:
|
|
164
166
|
send({'type': 'stderr', 'data': base64.b64encode((os.strerror(error.errno) + '\n').encode()).decode()})
|
|
@@ -211,6 +213,8 @@ def start_shell(message):
|
|
|
211
213
|
pid, master = pty.fork()
|
|
212
214
|
if pid == 0:
|
|
213
215
|
try:
|
|
216
|
+
# Keep management staging private without changing ordinary guest file creation.
|
|
217
|
+
os.umask(0o022)
|
|
214
218
|
fcntl.ioctl(0, termios.TIOCSWINSZ, size)
|
|
215
219
|
try:
|
|
216
220
|
home = pwd.getpwuid(0).pw_dir
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: empeira
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.0.pre.alpha.
|
|
4
|
+
version: 0.1.0.pre.alpha.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Christian Meißner (c8m6)
|
|
@@ -108,6 +108,7 @@ files:
|
|
|
108
108
|
- lib/empeira/control_plane/files.rb
|
|
109
109
|
- lib/empeira/control_plane/health.rb
|
|
110
110
|
- lib/empeira/control_plane/plan.rb
|
|
111
|
+
- lib/empeira/control_plane/startup_failure.rb
|
|
111
112
|
- lib/empeira/control_plane/storage.rb
|
|
112
113
|
- lib/empeira/errors.rb
|
|
113
114
|
- lib/empeira/execution/console.rb
|
|
@@ -128,6 +129,7 @@ files:
|
|
|
128
129
|
- lib/empeira/infrastructure/owned_files.rb
|
|
129
130
|
- lib/empeira/infrastructure/recovery_inventory.rb
|
|
130
131
|
- lib/empeira/infrastructure/service.rb
|
|
132
|
+
- lib/empeira/infrastructure/session_lock.rb
|
|
131
133
|
- lib/empeira/infrastructure/status.rb
|
|
132
134
|
- lib/empeira/infrastructure/store.rb
|
|
133
135
|
- lib/empeira/modules/events.rb
|
|
@@ -192,6 +194,7 @@ files:
|
|
|
192
194
|
- lib/empeira/node/run_request.rb
|
|
193
195
|
- lib/empeira/node/runtime_proxy.rb
|
|
194
196
|
- lib/empeira/node/service.rb
|
|
197
|
+
- lib/empeira/node/session_lock.rb
|
|
195
198
|
- lib/empeira/node/ssh_client.rb
|
|
196
199
|
- lib/empeira/node/ssh_credentials.rb
|
|
197
200
|
- lib/empeira/node/ssh_endpoint.rb
|