gowalk-cicd 1.0.193 → 1.0.195
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend-action/action.yml +2 -0
- package/backend-action/health_rollout.py +167 -0
- package/backend-action/health_rollout_credit.py +39 -0
- package/backend-action/health_rollout_engine.py +84 -0
- package/backend-action/health_rollout_spec.py +134 -0
- package/backend-action/install_runtime.py +5 -1
- package/backend-action/memory_agent.py +5 -2
- package/backend-action/network_host.py +1 -1
- package/backend-action/runtime_healthcheck.py +129 -0
- package/backend-action/runtime_image_health.py +50 -0
- package/backend-action/runtime_launch.py +3 -0
- package/backend-action/runtime_model.py +3 -0
- package/backend-action/runtime_projection.py +1 -1
- package/docs/BACKEND_HEALTHCHECKS.md +88 -0
- package/docs/BACKEND_MEMORY.md +6 -3
- package/docs/BACKEND_SSH_GATE.md +1 -1
- package/package.json +2 -1
|
@@ -222,6 +222,8 @@ runs:
|
|
|
222
222
|
"${{ github.action_path }}/runtime_env.py" \
|
|
223
223
|
"${{ github.action_path }}/runtime_env_parse.py" \
|
|
224
224
|
"${{ github.action_path }}/runtime_model.py" \
|
|
225
|
+
"${{ github.action_path }}/runtime_healthcheck.py" \
|
|
226
|
+
"${{ github.action_path }}/runtime_image_health.py" \
|
|
225
227
|
"${{ github.action_path }}/runtime_host.py" \
|
|
226
228
|
"${{ github.action_path }}/native_profile.py" \
|
|
227
229
|
"${{ github.action_path }}/runtime_launch.py" \
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Operator-only, exact-container health-cache adoption with a retained-container rollback."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
import health_rollout_credit as credit
|
|
11
|
+
from health_rollout_engine import Engine, MARKER, healthy, probe, unshared
|
|
12
|
+
from health_rollout_spec import ROLES, receipt, scope, specification, unchanged, verified
|
|
13
|
+
from runtime_host import host_enabled, start_lock
|
|
14
|
+
from runtime_launch import START_LOCK
|
|
15
|
+
from runtime_readback import profile
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def allowed(app, service, identifier, image):
|
|
19
|
+
if (not re.fullmatch('[a-z0-9][a-z0-9-]{0,79}', app) or service not in ROLES
|
|
20
|
+
or not re.fullmatch('[a-f0-9]{64}', identifier) or not re.fullmatch('sha256:[a-f0-9]{64}', image)):
|
|
21
|
+
raise ValueError('rollout_scope_invalid')
|
|
22
|
+
host_enabled()
|
|
23
|
+
document = profile(app)
|
|
24
|
+
if document is not None:
|
|
25
|
+
value = json.loads(document)
|
|
26
|
+
if value.get('disabled') or service in value.get('remove_services', []):
|
|
27
|
+
raise ValueError('runtime_disabled')
|
|
28
|
+
if Path('/opt/gowalk-backend-native', app + '.json').exists():
|
|
29
|
+
raise ValueError('native_runtime_unsupported')
|
|
30
|
+
return document
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def completed(engine, app, service, identifier, image, dry_run):
|
|
34
|
+
filters = quote(json.dumps({'label': [MARKER + '=' + identifier]}), safe='')
|
|
35
|
+
matches = engine.request('GET', '/containers/json?all=1&filters=' + filters)
|
|
36
|
+
if len(matches) != 1:
|
|
37
|
+
raise ValueError('original_container_missing')
|
|
38
|
+
current = engine.inspect(matches[0]['Id'])
|
|
39
|
+
scope(current, app, service, image)
|
|
40
|
+
if not current['State']['Running'] or current['State'].get('Health', {}).get('Status') != 'healthy':
|
|
41
|
+
raise ValueError('replacement_health_unconfirmed')
|
|
42
|
+
if not dry_run:
|
|
43
|
+
probe(engine, current['Id'], current['Config']['Healthcheck']['Test'])
|
|
44
|
+
return receipt(app, service, identifier, current['Id'], 'already_applied')
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def candidate(engine, old, body, name, staged, reservation):
|
|
48
|
+
if old['Name'].lstrip('/') != staged:
|
|
49
|
+
engine.request('POST', '/containers/' + old['Id'] + '/rename?name=' + quote(staged, safe=''))
|
|
50
|
+
current = engine.inspect(old['Id'])
|
|
51
|
+
unchanged(old, current)
|
|
52
|
+
if current['State']['Running']:
|
|
53
|
+
engine.request('POST', '/containers/' + old['Id'] + '/stop?t=10')
|
|
54
|
+
replacement = engine.inspect(name, missing=True)
|
|
55
|
+
if replacement is None:
|
|
56
|
+
result = engine.request('POST', '/containers/create?name=' + quote(name, safe=''), body)
|
|
57
|
+
replacement = engine.inspect(result['Id'])
|
|
58
|
+
verified(replacement, body, old)
|
|
59
|
+
credit.retain(replacement['Id'], reservation)
|
|
60
|
+
if not replacement['State']['Running']:
|
|
61
|
+
engine.request('POST', '/containers/' + replacement['Id'] + '/start')
|
|
62
|
+
return replacement['Id']
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def confirmed(engine, method, path, proof):
|
|
66
|
+
try:
|
|
67
|
+
engine.request(method, path)
|
|
68
|
+
except Exception:
|
|
69
|
+
if not proof():
|
|
70
|
+
raise
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def rollback(engine, old, name, reservation):
|
|
74
|
+
replacement = engine.inspect(name, missing=True)
|
|
75
|
+
if replacement is not None and replacement['Id'] != old['Id']:
|
|
76
|
+
if (replacement['Config'].get('Labels', {}).get(MARKER) != old['Id']
|
|
77
|
+
or replacement['Image'] != old['Image']):
|
|
78
|
+
raise ValueError('rollback_name_conflict')
|
|
79
|
+
confirmed(engine, 'DELETE', '/containers/' + replacement['Id'] + '?force=1&v=0',
|
|
80
|
+
lambda: engine.inspect(replacement['Id'], missing=True) is None)
|
|
81
|
+
retained = engine.inspect(old['Id'])
|
|
82
|
+
unchanged(old, retained, reservation_changes=True)
|
|
83
|
+
if retained['Name'].lstrip('/') != name:
|
|
84
|
+
confirmed(engine, 'POST', '/containers/' + old['Id'] + '/rename?name=' + quote(name, safe=''),
|
|
85
|
+
lambda: engine.inspect(old['Id'])['Name'].lstrip('/') == name)
|
|
86
|
+
if not retained['State']['Running']:
|
|
87
|
+
confirmed(engine, 'POST', '/containers/' + old['Id'] + '/start',
|
|
88
|
+
lambda: engine.inspect(old['Id'])['State']['Running'])
|
|
89
|
+
healthy(engine, old['Id'], engine.deadline)
|
|
90
|
+
probe(engine, old['Id'], old['Config']['Healthcheck']['Test'])
|
|
91
|
+
if replacement is not None and replacement['Id'] != old['Id']:
|
|
92
|
+
credit.forget(replacement['Id'], reservation)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def apply(engine, app, service, old, body, name, staged, original_profile, reservation):
|
|
96
|
+
try:
|
|
97
|
+
replacement = candidate(engine, old, body, name, staged, reservation)
|
|
98
|
+
current = healthy(engine, replacement, min(time.monotonic() + 40, engine.deadline - 25))
|
|
99
|
+
verified(current, body, old)
|
|
100
|
+
probe(engine, replacement, old['Config']['Healthcheck']['Test'])
|
|
101
|
+
host_enabled()
|
|
102
|
+
if profile(app) != original_profile:
|
|
103
|
+
raise ValueError('runtime_profile_changed')
|
|
104
|
+
unchanged(old, engine.inspect(old['Id']))
|
|
105
|
+
try:
|
|
106
|
+
engine.request('DELETE', '/containers/' + old['Id'] + '?v=0')
|
|
107
|
+
except Exception:
|
|
108
|
+
if engine.inspect(old['Id'], missing=True) is not None:
|
|
109
|
+
raise
|
|
110
|
+
return receipt(app, service, old['Id'], replacement)
|
|
111
|
+
except Exception:
|
|
112
|
+
try:
|
|
113
|
+
rollback(engine, old, name, reservation)
|
|
114
|
+
except Exception:
|
|
115
|
+
raise ValueError('rollback_unconfirmed') from None
|
|
116
|
+
raise ValueError('rollout_failed_restored') from None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def rollout(app, service, identifier, image, dry_run=False):
|
|
120
|
+
original_profile = allowed(app, service, identifier, image)
|
|
121
|
+
with start_lock(START_LOCK):
|
|
122
|
+
if allowed(app, service, identifier, image) != original_profile:
|
|
123
|
+
raise ValueError('runtime_profile_changed')
|
|
124
|
+
engine = Engine()
|
|
125
|
+
old = engine.inspect(identifier, missing=True)
|
|
126
|
+
if old is None:
|
|
127
|
+
return completed(engine, app, service, identifier, image, dry_run)
|
|
128
|
+
if old['Id'] != identifier:
|
|
129
|
+
raise ValueError('runtime_changed')
|
|
130
|
+
scope(old, app, service, image)
|
|
131
|
+
body = specification(old)
|
|
132
|
+
reservation = credit.original(old)
|
|
133
|
+
suffix = '.gowalk-health-' + identifier[:12]
|
|
134
|
+
current_name = old['Name'].lstrip('/')
|
|
135
|
+
recovering = current_name.endswith(suffix)
|
|
136
|
+
name = current_name[:-len(suffix)] if recovering else current_name
|
|
137
|
+
if not recovering and (not old['State']['Running']
|
|
138
|
+
or old['State'].get('Health', {}).get('Status') != 'healthy'):
|
|
139
|
+
raise ValueError('original_health_unconfirmed')
|
|
140
|
+
unshared(engine, old)
|
|
141
|
+
unchanged(old, engine.inspect(identifier))
|
|
142
|
+
if dry_run:
|
|
143
|
+
return receipt(app, service, identifier, status='recovery_required' if recovering else 'eligible')
|
|
144
|
+
return apply(engine, app, service, old, body, name, name + suffix, original_profile, reservation)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def main(argv=None):
|
|
148
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
149
|
+
for name in ('project', 'service', 'container_id', 'image_id'):
|
|
150
|
+
parser.add_argument(name)
|
|
151
|
+
parser.add_argument('--dry-run', action='store_true')
|
|
152
|
+
args = parser.parse_args(argv)
|
|
153
|
+
try:
|
|
154
|
+
if os.geteuid() != 0:
|
|
155
|
+
raise ValueError('operator_root_required')
|
|
156
|
+
result = rollout(args.project, args.service, args.container_id, args.image_id, args.dry_run)
|
|
157
|
+
except Exception as error:
|
|
158
|
+
code = str(error) if isinstance(error, ValueError) and re.fullmatch('[a-z_]+', str(error)) else 'unclassified'
|
|
159
|
+
result = receipt(args.project, args.service, args.container_id, status=code)
|
|
160
|
+
print(json.dumps(result, separators=(',', ':')))
|
|
161
|
+
return 1
|
|
162
|
+
print(json.dumps(result, separators=(',', ':')))
|
|
163
|
+
return 0
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
if __name__ == '__main__':
|
|
167
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Preserve the original container's trusted redeployment admission credit across health adoption."""
|
|
2
|
+
import stat
|
|
3
|
+
|
|
4
|
+
from memory_agent import DEFAULT, record
|
|
5
|
+
from runtime_host import AGENT_STATE, ROOT_UID
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def read(identifier, default=None):
|
|
9
|
+
path = AGENT_STATE / identifier
|
|
10
|
+
try:
|
|
11
|
+
info = path.lstat()
|
|
12
|
+
except FileNotFoundError:
|
|
13
|
+
return default
|
|
14
|
+
if (not stat.S_ISREG(info.st_mode) or info.st_uid != ROOT_UID or info.st_mode & 0o022 or info.st_size > 64):
|
|
15
|
+
raise ValueError('reservation_record_invalid')
|
|
16
|
+
try:
|
|
17
|
+
value = int(path.read_text())
|
|
18
|
+
except (OSError, ValueError):
|
|
19
|
+
raise ValueError('reservation_record_invalid') from None
|
|
20
|
+
if value <= 0:
|
|
21
|
+
raise ValueError('reservation_record_invalid')
|
|
22
|
+
return value
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def original(row):
|
|
26
|
+
return read(row['Id'], row['HostConfig'].get('MemoryReservation') or DEFAULT)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def retain(identifier, amount):
|
|
30
|
+
record(identifier, amount)
|
|
31
|
+
if read(identifier) != amount:
|
|
32
|
+
raise ValueError('reservation_record_changed')
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def forget(identifier, amount):
|
|
36
|
+
if read(identifier) is not None:
|
|
37
|
+
if read(identifier) != amount:
|
|
38
|
+
raise ValueError('reservation_record_changed')
|
|
39
|
+
(AGENT_STATE / identifier).unlink()
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Private, bounded Docker transport for the operator's selected health-check rollout."""
|
|
2
|
+
import http.client
|
|
3
|
+
import json
|
|
4
|
+
import socket
|
|
5
|
+
import subprocess
|
|
6
|
+
import time
|
|
7
|
+
from urllib.parse import quote
|
|
8
|
+
|
|
9
|
+
MARKER = 'io.gowalk.backend.health-original'
|
|
10
|
+
SOCKET = '/var/run/docker.sock'
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Engine:
|
|
14
|
+
def __init__(self):
|
|
15
|
+
self.deadline = time.monotonic() + 90
|
|
16
|
+
|
|
17
|
+
def request(self, method, path, body=None, missing=False, raw=False):
|
|
18
|
+
remaining = self.deadline - time.monotonic()
|
|
19
|
+
if remaining <= 0:
|
|
20
|
+
raise ValueError('rollout_deadline')
|
|
21
|
+
connection = http.client.HTTPConnection('localhost', timeout=min(10, remaining))
|
|
22
|
+
connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
23
|
+
connection.sock.settimeout(min(10, remaining))
|
|
24
|
+
try:
|
|
25
|
+
connection.sock.connect(SOCKET)
|
|
26
|
+
data = None if body is None else json.dumps(body).encode()
|
|
27
|
+
connection.request(method, '/v1.45' + path, data, {'Content-Type': 'application/json'})
|
|
28
|
+
response = connection.getresponse()
|
|
29
|
+
content = response.read(2 * 1024 * 1024 + 1)
|
|
30
|
+
if missing and response.status == 404:
|
|
31
|
+
return None
|
|
32
|
+
if len(content) > 2 * 1024 * 1024 or response.status not in (200, 201, 204, 304):
|
|
33
|
+
raise ValueError('engine_operation_failed')
|
|
34
|
+
return content if raw else json.loads(content) if content else None
|
|
35
|
+
finally:
|
|
36
|
+
connection.close()
|
|
37
|
+
|
|
38
|
+
def inspect(self, identifier, missing=False):
|
|
39
|
+
return self.request('GET', '/containers/' + quote(identifier, safe='') + '/json', missing=missing)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def unshared(engine, original):
|
|
43
|
+
identifiers = [row['Id'] for row in engine.request('GET', '/containers/json?all=1')]
|
|
44
|
+
if len(identifiers) > 1024:
|
|
45
|
+
raise ValueError('container_inventory_limit')
|
|
46
|
+
template = ('{"id":{{json .Id}},"network":{{json .HostConfig.NetworkMode}},'
|
|
47
|
+
'"pid":{{json .HostConfig.PidMode}},"ipc":{{json .HostConfig.IpcMode}},'
|
|
48
|
+
'"volumes":{{json .HostConfig.VolumesFrom}},"links":{{json .HostConfig.Links}}}')
|
|
49
|
+
result = subprocess.run(['docker', 'inspect', '--format', template, *identifiers],
|
|
50
|
+
capture_output=True, text=True, timeout=15, check=True)
|
|
51
|
+
if len(result.stdout) > 1024 * 1024:
|
|
52
|
+
raise ValueError('container_inventory_limit')
|
|
53
|
+
references = {original['Id'], original['Id'][:12], original['Name'].lstrip('/')}
|
|
54
|
+
for line in result.stdout.splitlines():
|
|
55
|
+
row = json.loads(line)
|
|
56
|
+
if row['id'] == original['Id']:
|
|
57
|
+
continue
|
|
58
|
+
names = [row[field][10:] if row[field].startswith('container:') else row[field]
|
|
59
|
+
for field in ('network', 'pid', 'ipc')]
|
|
60
|
+
names += [value.split(':')[0].lstrip('/') for field in ('volumes', 'links') for value in row[field] or []]
|
|
61
|
+
if references.intersection(names):
|
|
62
|
+
raise ValueError('container_has_consumers')
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def probe(engine, identifier, test):
|
|
66
|
+
command = test[1:] if test[0] == 'CMD' else ['/bin/sh', '-c', test[1]]
|
|
67
|
+
created = engine.request('POST', '/containers/' + identifier + '/exec',
|
|
68
|
+
{'AttachStdout': False, 'AttachStderr': False, 'Cmd': command})
|
|
69
|
+
engine.request('POST', '/exec/' + created['Id'] + '/start', {'Detach': False, 'Tty': False}, raw=True)
|
|
70
|
+
result = engine.request('GET', '/exec/' + created['Id'] + '/json')
|
|
71
|
+
if result['Running'] or result['ExitCode'] != 0:
|
|
72
|
+
raise ValueError('original_probe_failed')
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def healthy(engine, identifier, deadline):
|
|
76
|
+
while time.monotonic() < deadline:
|
|
77
|
+
current = engine.inspect(identifier)
|
|
78
|
+
status = current['State'].get('Health', {}).get('Status')
|
|
79
|
+
if current['State']['Running'] and status == 'healthy':
|
|
80
|
+
return current
|
|
81
|
+
if not current['State']['Running'] or status == 'unhealthy':
|
|
82
|
+
raise ValueError('container_health_failed')
|
|
83
|
+
time.sleep(1)
|
|
84
|
+
raise ValueError('container_health_timeout')
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Copy an eligible running API's exact private Docker configuration; change only its health command."""
|
|
2
|
+
import copy
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from health_rollout_engine import MARKER
|
|
6
|
+
from runtime_healthcheck import cache_healthcheck
|
|
7
|
+
|
|
8
|
+
ROLES = frozenset(('api', 'app', 'backend', 'web'))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def scope(row, app, service, image):
|
|
12
|
+
labels = row['Config'].get('Labels') or {}
|
|
13
|
+
if (labels.get('com.docker.compose.project') != app
|
|
14
|
+
or labels.get('com.docker.compose.service') != service
|
|
15
|
+
or labels.get('com.docker.compose.project.working_dir') != '/opt/gowalk-backends/' + app
|
|
16
|
+
or row['Image'] != image or service not in ROLES
|
|
17
|
+
or labels.get('com.docker.compose.oneoff', 'false').lower() != 'false'):
|
|
18
|
+
raise ValueError('runtime_scope_changed')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def mounts(row):
|
|
22
|
+
host = row['HostConfig']
|
|
23
|
+
declared = {}
|
|
24
|
+
for binding in host.get('Binds') or []:
|
|
25
|
+
parts = binding.split(':')
|
|
26
|
+
if len(parts) not in (2, 3):
|
|
27
|
+
raise ValueError('mount_unsupported')
|
|
28
|
+
declared[parts[1]] = parts[0]
|
|
29
|
+
for binding in host.get('Mounts') or []:
|
|
30
|
+
if binding.get('Type') not in ('bind', 'volume') or not binding.get('Source'):
|
|
31
|
+
raise ValueError('mount_unsupported')
|
|
32
|
+
declared[binding['Target']] = binding['Source']
|
|
33
|
+
for mounted in row.get('Mounts') or []:
|
|
34
|
+
kind, target = mounted['Type'], mounted['Destination']
|
|
35
|
+
if kind == 'tmpfs' and target in (host.get('Tmpfs') or {}):
|
|
36
|
+
continue
|
|
37
|
+
source = mounted.get('Name') if kind == 'volume' else mounted.get('Source')
|
|
38
|
+
if (kind not in ('bind', 'volume') or declared.get(target) != source
|
|
39
|
+
or kind == 'volume' and re.fullmatch('[a-f0-9]{64}', source or '')):
|
|
40
|
+
raise ValueError('mount_unsupported')
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def networks(row):
|
|
44
|
+
result = {}
|
|
45
|
+
for name, endpoint in row['NetworkSettings']['Networks'].items():
|
|
46
|
+
if endpoint.get('IPAMConfig') or endpoint.get('Links'):
|
|
47
|
+
raise ValueError('network_unsupported')
|
|
48
|
+
result[name] = {key: copy.deepcopy(endpoint[key]) for key in ('Aliases', 'DriverOpts') if endpoint.get(key)}
|
|
49
|
+
if not result:
|
|
50
|
+
raise ValueError('network_unsupported')
|
|
51
|
+
return {'EndpointsConfig': result}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def published_ports(row):
|
|
55
|
+
declared = row['HostConfig'].get('PortBindings') or {}
|
|
56
|
+
if row['State']['Running']:
|
|
57
|
+
actual = {key: value for key, value in row['NetworkSettings'].get('Ports', {}).items() if value}
|
|
58
|
+
if declared != actual:
|
|
59
|
+
raise ValueError('dynamic_ports_unsupported')
|
|
60
|
+
return
|
|
61
|
+
# Docker clears live bindings on stop; a retained original still holds its exact explicit host ports.
|
|
62
|
+
for bindings in declared.values():
|
|
63
|
+
if not isinstance(bindings, list) or not bindings:
|
|
64
|
+
raise ValueError('dynamic_ports_unsupported')
|
|
65
|
+
for binding in bindings:
|
|
66
|
+
port = binding.get('HostPort') if isinstance(binding, dict) else None
|
|
67
|
+
if not isinstance(port, str) or not re.fullmatch('[0-9]+', port) or not 1 <= int(port) <= 65535:
|
|
68
|
+
raise ValueError('dynamic_ports_unsupported')
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def specification(row):
|
|
72
|
+
host = row['HostConfig']
|
|
73
|
+
if (not host.get('ReadonlyRootfs') or host.get('AutoRemove') or host.get('Privileged')
|
|
74
|
+
or host.get('VolumesFrom') or host.get('Links') or host.get('ContainerIDFile')
|
|
75
|
+
or host.get('PublishAllPorts') or host.get('PidMode') not in ('', 'private')
|
|
76
|
+
or host.get('IpcMode') not in ('', 'private', 'shareable')
|
|
77
|
+
or host.get('UTSMode') or host.get('NetworkMode') in ('host', 'none')
|
|
78
|
+
or host.get('NetworkMode', '').startswith('container:')):
|
|
79
|
+
raise ValueError('runtime_unsupported')
|
|
80
|
+
if row['Config'].get('Shell') not in (None, [], ['/bin/sh', '-c']):
|
|
81
|
+
raise ValueError('custom_shell_unsupported')
|
|
82
|
+
if (row['Config'].get('StopTimeout') or 10) > 10:
|
|
83
|
+
raise ValueError('stop_timeout_unsupported')
|
|
84
|
+
published_ports(row)
|
|
85
|
+
mounts(row)
|
|
86
|
+
health = copy.deepcopy(row['Config'].get('Healthcheck') or {})
|
|
87
|
+
if (health.get('StartPeriod', 0) < 30000000000
|
|
88
|
+
or (health.get('StartInterval') or 5000000000) > 5000000000
|
|
89
|
+
or (health.get('Timeout') or 30000000000) > 5000000000):
|
|
90
|
+
raise ValueError('health_startup_unsupported')
|
|
91
|
+
service = {'healthcheck': {'test': health.get('Test')},
|
|
92
|
+
'tmpfs': [name + ':' + options for name, options in (host.get('Tmpfs') or {}).items()],
|
|
93
|
+
'environment': dict(value.split('=', 1) for value in row['Config'].get('Env') or [])}
|
|
94
|
+
if not cache_healthcheck(service):
|
|
95
|
+
raise ValueError('healthcheck_not_eligible')
|
|
96
|
+
body = copy.deepcopy(row['Config'])
|
|
97
|
+
body['Image'] = row['Image']
|
|
98
|
+
body['Healthcheck']['Test'] = service['healthcheck']['test']
|
|
99
|
+
body.setdefault('Labels', {})[MARKER] = row['Id']
|
|
100
|
+
body['HostConfig'] = copy.deepcopy(host)
|
|
101
|
+
body['NetworkingConfig'] = networks(row)
|
|
102
|
+
return body
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def unchanged(before, after, reservation_changes=False):
|
|
106
|
+
hosts = [dict(row['HostConfig']) for row in (before, after)]
|
|
107
|
+
if reservation_changes:
|
|
108
|
+
for host in hosts:
|
|
109
|
+
host.pop('MemoryReservation', None)
|
|
110
|
+
if hosts[0] != hosts[1] or any(before[field] != after[field] for field in ('Id', 'Image', 'Config')):
|
|
111
|
+
raise ValueError('runtime_changed')
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def verified(row, body, original):
|
|
115
|
+
config = {key: value for key, value in body.items() if key not in ('HostConfig', 'NetworkingConfig')}
|
|
116
|
+
if (row['Config'] != config or row['Image'] != original['Image'] or row['HostConfig'] != body['HostConfig']):
|
|
117
|
+
raise ValueError('replacement_config_changed')
|
|
118
|
+
actual = row['NetworkSettings']['Networks']
|
|
119
|
+
expected = body['NetworkingConfig']['EndpointsConfig']
|
|
120
|
+
if set(actual) != set(expected):
|
|
121
|
+
raise ValueError('replacement_network_changed')
|
|
122
|
+
for name, endpoint in expected.items():
|
|
123
|
+
if (not set(endpoint.get('Aliases') or []).issubset(actual[name].get('Aliases') or [])
|
|
124
|
+
or (endpoint.get('DriverOpts') or {}) != (actual[name].get('DriverOpts') or {})):
|
|
125
|
+
raise ValueError('replacement_network_changed')
|
|
126
|
+
before = sorted(original['Mounts'], key=lambda mount: mount['Destination'])
|
|
127
|
+
after = sorted(row['Mounts'], key=lambda mount: mount['Destination'])
|
|
128
|
+
if before != after:
|
|
129
|
+
raise ValueError('replacement_mount_changed')
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def receipt(app, service, old, new='', status='applied'):
|
|
133
|
+
return {'schema': 'gowalk-cicd/backend-health-rollout.v1', 'project': app, 'service': service,
|
|
134
|
+
'original_id': old, 'container_id': new, 'status': status}
|
|
@@ -17,7 +17,11 @@ AUTHORIZED_KEYS = Path('/root/.ssh/authorized_keys')
|
|
|
17
17
|
ROOT_UID = 0
|
|
18
18
|
REQUIRED = (
|
|
19
19
|
'ssh_gate.py', 'native_profile.py', 'native_deploy.py', 'runtime_host.py', 'runtime_launch.py',
|
|
20
|
-
'runtime_model.py', '
|
|
20
|
+
'runtime_model.py', 'runtime_healthcheck.py', 'runtime_image_health.py', 'runtime_readback.py',
|
|
21
|
+
'runtime_projection.py',
|
|
22
|
+
'identity_diagnostics.py',
|
|
23
|
+
'health_rollout.py', 'health_rollout_engine.py', 'health_rollout_spec.py', 'health_rollout_credit.py',
|
|
24
|
+
'memory_agent.py',
|
|
21
25
|
'runtime_env.py', 'runtime_env_parse.py', 'edge_inputs.py', 'edge_vhost.py', 'edge_health.py',
|
|
22
26
|
'configure_vhost.py', 'vhost_diagnostics.py', 'remote_deploy.sh', 'deploy_evidence.sh',
|
|
23
27
|
'ensure_certificate.sh', 'select_certbot_account.sh',
|
|
@@ -175,8 +175,11 @@ def run(dry_run, now=None):
|
|
|
175
175
|
ids, containers = discover()
|
|
176
176
|
now = time.time() if now is None else now
|
|
177
177
|
try:
|
|
178
|
-
|
|
179
|
-
|
|
178
|
+
memory = meminfo()
|
|
179
|
+
# Admission leaves 30% of RAM for host headroom; avoid evicting useful pages while it is available.
|
|
180
|
+
zswap = (memory["MemTotal"] > 0 and 0 <= memory["MemAvailable"] * 10 < memory["MemTotal"] * 3
|
|
181
|
+
and memory.get("SwapTotal", 0) > 0 and ZSWAP.read_text().strip() == "Y")
|
|
182
|
+
except (OSError, KeyError, ValueError, TypeError):
|
|
180
183
|
zswap = False
|
|
181
184
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
182
185
|
entries = list(pool.map(lambda container: handle(container, now, zswap, dry_run), containers))
|
|
@@ -138,7 +138,7 @@ def vpn_settings(containers, app, service, bridge):
|
|
|
138
138
|
if row['id'] != identifiers[0] or row['project'] != app or row['service'] != service:
|
|
139
139
|
raise NetworkFailure('network_scope_mismatch')
|
|
140
140
|
added = set(row['cap_add'] or [])
|
|
141
|
-
allowed = {'NET_ADMIN', 'NET_BIND_SERVICE', 'CHOWN', 'SETUID', 'SETGID'}
|
|
141
|
+
allowed = {'NET_ADMIN', 'NET_BIND_SERVICE', 'CHOWN', 'SETUID', 'SETGID', 'SETPCAP'}
|
|
142
142
|
safe = (row['privileged'] is False and row['network_mode'] == bridge['name']
|
|
143
143
|
and 'NET_ADMIN' in added and added.issubset(allowed)
|
|
144
144
|
and 'ALL' in (row['cap_drop'] or [])
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Cache recognized stdlib-only Python probes inside their existing bounded private tmpfs.
|
|
2
|
+
|
|
3
|
+
Only the health process receives cache settings. CPython writes its own interpreter-tagged
|
|
4
|
+
bytecode and validates source metadata; recreation discards the container's tmpfs cache.
|
|
5
|
+
Probe code, health timing, application environment and filesystem permissions stay intact.
|
|
6
|
+
"""
|
|
7
|
+
import ast
|
|
8
|
+
import re
|
|
9
|
+
import shlex
|
|
10
|
+
|
|
11
|
+
CACHE_ROOT = '/tmp/.gowalk-healthcheck-pycache'
|
|
12
|
+
MODULES = {'urllib.request', 'urllib.error', 'json', 'sys', '__future__'}
|
|
13
|
+
DYNAMIC = {'__import__', 'eval', 'exec', 'compile', 'getattr', 'setattr', 'import_module', 'load_module'}
|
|
14
|
+
BOOTSTRAP = """import os, stat, sys
|
|
15
|
+
if hasattr(sys, 'pycache_prefix') and sys.pycache_prefix is None:
|
|
16
|
+
try:
|
|
17
|
+
cache = {cache}
|
|
18
|
+
parent = os.lstat(os.path.dirname(cache))
|
|
19
|
+
safe_parent = (stat.S_ISDIR(parent.st_mode) and parent.st_uid in (0, os.geteuid())
|
|
20
|
+
and (parent.st_mode & stat.S_ISVTX or not parent.st_mode & 0o022))
|
|
21
|
+
if safe_parent:
|
|
22
|
+
try:
|
|
23
|
+
os.mkdir(cache, 0o700)
|
|
24
|
+
except FileExistsError:
|
|
25
|
+
pass
|
|
26
|
+
info = os.lstat(cache)
|
|
27
|
+
if (stat.S_ISDIR(info.st_mode) and info.st_uid == os.geteuid()
|
|
28
|
+
and stat.S_IMODE(info.st_mode) == 0o700 and os.access(cache, os.W_OK | os.X_OK)):
|
|
29
|
+
sys.pycache_prefix = cache
|
|
30
|
+
sys.dont_write_bytecode = False
|
|
31
|
+
except OSError:
|
|
32
|
+
pass
|
|
33
|
+
exec(compile(sys.argv.pop(1), '<string>', 'exec', dont_inherit=True))"""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def stdlib_probe(source):
|
|
37
|
+
if not isinstance(source, str) or len(source) > 4096:
|
|
38
|
+
return False
|
|
39
|
+
try:
|
|
40
|
+
tree = ast.parse(source)
|
|
41
|
+
except (SyntaxError, ValueError, RecursionError):
|
|
42
|
+
return False
|
|
43
|
+
modules = set()
|
|
44
|
+
for node in ast.walk(tree):
|
|
45
|
+
if isinstance(node, ast.Import):
|
|
46
|
+
modules.update(alias.name for alias in node.names)
|
|
47
|
+
elif isinstance(node, ast.ImportFrom):
|
|
48
|
+
if node.level or any(alias.name == '*' for alias in node.names):
|
|
49
|
+
return False
|
|
50
|
+
modules.add(node.module)
|
|
51
|
+
elif isinstance(node, ast.Name) and node.id in DYNAMIC:
|
|
52
|
+
return False
|
|
53
|
+
elif isinstance(node, ast.Attribute) and (node.attr in DYNAMIC or node.attr.startswith('__')):
|
|
54
|
+
return False
|
|
55
|
+
return 'urllib.request' in modules and modules <= MODULES
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def transform_healthcheck(test):
|
|
59
|
+
if not isinstance(test, list) or not all(isinstance(word, str) for word in test):
|
|
60
|
+
return None
|
|
61
|
+
if test[:1] == ['CMD']:
|
|
62
|
+
arguments = test[1:]
|
|
63
|
+
elif test[:1] == ['CMD-SHELL'] and len(test) == 2:
|
|
64
|
+
try:
|
|
65
|
+
arguments = shlex.split(test[1])
|
|
66
|
+
except ValueError:
|
|
67
|
+
return None
|
|
68
|
+
if arguments[3:] == ['||', 'exit', '1']:
|
|
69
|
+
arguments = arguments[:3]
|
|
70
|
+
else:
|
|
71
|
+
return None
|
|
72
|
+
if len(arguments) != 3 or arguments[1] != '-c':
|
|
73
|
+
return None
|
|
74
|
+
if not re.fullmatch(r'(?:/(?:[a-zA-Z0-9_.-]+/)*)?python(?:3(?:[.][0-9]+)?)?', arguments[0]):
|
|
75
|
+
return None
|
|
76
|
+
if not stdlib_probe(arguments[2]):
|
|
77
|
+
return None
|
|
78
|
+
bootstrap = BOOTSTRAP.format(cache=repr(CACHE_ROOT))
|
|
79
|
+
if test[0] == 'CMD':
|
|
80
|
+
return ['CMD', arguments[0], '-c', bootstrap, arguments[2]]
|
|
81
|
+
prefix = re.match(r'(\s*' + re.escape(arguments[0]) + r'\s+)-c(?=\s)', test[1])
|
|
82
|
+
if prefix is None:
|
|
83
|
+
return None
|
|
84
|
+
return ['CMD-SHELL', prefix[1] + '-c ' + shlex.quote(bootstrap) + test[1][prefix.end():]]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def bounded_options(options):
|
|
88
|
+
fields = options.split(',')
|
|
89
|
+
if 'ro' in fields:
|
|
90
|
+
return False
|
|
91
|
+
sizes = [field[5:] for field in fields if field.startswith('size=')]
|
|
92
|
+
if len(sizes) != 1:
|
|
93
|
+
return False
|
|
94
|
+
match = re.fullmatch(r'([0-9]+)([kKmMgG]?)', sizes[0])
|
|
95
|
+
if not match:
|
|
96
|
+
return False
|
|
97
|
+
value = int(match[1]) * {'': 1, 'k': 1024, 'm': 1024 ** 2, 'g': 1024 ** 3}[match[2].lower()]
|
|
98
|
+
return value >= 16 * 1024 * 1024
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def bounded_tmpfs(service):
|
|
102
|
+
mounts = service.get('tmpfs', [])
|
|
103
|
+
if isinstance(mounts, str):
|
|
104
|
+
mounts = [mounts]
|
|
105
|
+
matches = [mount.partition(':')[2] for mount in mounts
|
|
106
|
+
if isinstance(mount, str) and mount.partition(':')[0] == '/tmp']
|
|
107
|
+
for volume in service.get('volumes', []):
|
|
108
|
+
if not isinstance(volume, dict) or volume.get('target') != '/tmp':
|
|
109
|
+
continue
|
|
110
|
+
if volume.get('type') != 'tmpfs' or volume.get('read_only'):
|
|
111
|
+
return False
|
|
112
|
+
matches.append('size=' + str(volume.get('tmpfs', {}).get('size', '')))
|
|
113
|
+
return len(matches) == 1 and bounded_options(matches[0])
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def cache_healthcheck(service):
|
|
117
|
+
health = service.get('healthcheck', {})
|
|
118
|
+
if health.get('disable') or not bounded_tmpfs(service):
|
|
119
|
+
return False
|
|
120
|
+
environment = service.get('environment', {})
|
|
121
|
+
if isinstance(environment, list):
|
|
122
|
+
environment = {item.split('=', 1)[0]: None for item in environment if isinstance(item, str)}
|
|
123
|
+
if 'PYTHONPYCACHEPREFIX' in environment:
|
|
124
|
+
return False
|
|
125
|
+
updated = transform_healthcheck(health.get('test'))
|
|
126
|
+
if updated is None:
|
|
127
|
+
return False
|
|
128
|
+
health['test'] = updated
|
|
129
|
+
return True
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Resolve inherited health probes only from freshly built local images and pin their exact identity."""
|
|
2
|
+
import copy
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
from runtime_healthcheck import bounded_tmpfs, cache_healthcheck
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def eligible(service):
|
|
11
|
+
health = service.get('healthcheck', {})
|
|
12
|
+
policies = (None, 'never', 'build', 'missing', 'if_not_present')
|
|
13
|
+
return (bool(service.get('build')) and service.get('pull_policy') in policies
|
|
14
|
+
and not health.get('disable') and 'test' not in health and bounded_tmpfs(service))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def image_health(command, name, service):
|
|
18
|
+
reference = service.get('image')
|
|
19
|
+
if not reference:
|
|
20
|
+
result = subprocess.run(command + ['config', '--images', name],
|
|
21
|
+
capture_output=True, text=True, timeout=30, check=True)
|
|
22
|
+
references = result.stdout.split()
|
|
23
|
+
if len(references) != 1:
|
|
24
|
+
raise ValueError('runtime_built_image_ambiguous')
|
|
25
|
+
reference = references[0]
|
|
26
|
+
template = '{"Id":{{json .Id}},"Healthcheck":{{json .Config.Healthcheck}}}'
|
|
27
|
+
result = subprocess.run(['docker', 'image', 'inspect', '--format', template, '--', reference],
|
|
28
|
+
capture_output=True, text=True, timeout=30, check=True)
|
|
29
|
+
image = json.loads(result.stdout)
|
|
30
|
+
if not isinstance(image, dict) or not re.fullmatch('sha256:[a-f0-9]{64}', str(image.get('Id'))):
|
|
31
|
+
raise ValueError('runtime_built_image_invalid')
|
|
32
|
+
return image
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def inherit_healthchecks(model, command):
|
|
36
|
+
changed = False
|
|
37
|
+
for name, service in model['services'].items():
|
|
38
|
+
if not eligible(service):
|
|
39
|
+
continue
|
|
40
|
+
image = image_health(command, name, service)
|
|
41
|
+
health = image.get('Healthcheck') or {}
|
|
42
|
+
if not isinstance(health, dict) or not health.get('Test'):
|
|
43
|
+
continue
|
|
44
|
+
candidate = copy.deepcopy(service)
|
|
45
|
+
candidate.setdefault('healthcheck', {})['test'] = health['Test']
|
|
46
|
+
if cache_healthcheck(candidate):
|
|
47
|
+
service['healthcheck'] = candidate['healthcheck']
|
|
48
|
+
service.update(image=image['Id'], pull_policy='never')
|
|
49
|
+
changed = True
|
|
50
|
+
return changed
|
|
@@ -8,6 +8,7 @@ import tempfile
|
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
|
|
10
10
|
from runtime_host import admit, host_facts, running_containers, start_lock
|
|
11
|
+
from runtime_image_health import inherit_healthchecks
|
|
11
12
|
from runtime_model import configure
|
|
12
13
|
|
|
13
14
|
PROFILE_ROOT = Path("/opt/gowalk-backend-runtime")
|
|
@@ -43,6 +44,8 @@ def launch(app, arguments):
|
|
|
43
44
|
if profile_snapshot(app) != original_profile:
|
|
44
45
|
raise ValueError('runtime_profile_changed')
|
|
45
46
|
admit(app, model, host_facts(), running_containers())
|
|
47
|
+
if inherit_healthchecks(model, command):
|
|
48
|
+
Path(filename).write_text(json.dumps(model))
|
|
46
49
|
subprocess.run(command + ["up", "-d", "--no-build", "--remove-orphans"], check=True)
|
|
47
50
|
finally:
|
|
48
51
|
Path(filename).unlink()
|
|
@@ -3,6 +3,8 @@ import copy
|
|
|
3
3
|
import math
|
|
4
4
|
import re
|
|
5
5
|
|
|
6
|
+
from runtime_healthcheck import cache_healthcheck
|
|
7
|
+
|
|
6
8
|
MIB = 1024 * 1024
|
|
7
9
|
SCHEMA = "gowalk-cicd/backend-runtime.v1"
|
|
8
10
|
SERVICE_FIELDS = {"environment", "command", "entrypoint", "mem_limit", "mem_reservation", "cpus", "pids_limit"}
|
|
@@ -98,6 +100,7 @@ def configure(document, profile=None):
|
|
|
98
100
|
raise ValueError("compose_services_required")
|
|
99
101
|
for service in model["services"].values():
|
|
100
102
|
bound_service(service)
|
|
103
|
+
cache_healthcheck(service)
|
|
101
104
|
return model
|
|
102
105
|
|
|
103
106
|
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# Python healthcheck bytecode cache
|
|
2
|
+
|
|
3
|
+
`runtime_healthcheck.py` reduces repeated Python healthcheck import compilation. Runtime launch
|
|
4
|
+
automatically wraps recognized `python -c` probes whose static imports are limited to
|
|
5
|
+
`urllib.request`, `urllib.error`, `json`, `sys` and `__future__`. Application imports, script-file
|
|
6
|
+
probes, dynamic imports, unknown interpreter flags and unrecognized shell commands stay unchanged.
|
|
7
|
+
Both direct `CMD` and simple `CMD-SHELL` probes work, including a terminal `|| exit 1`.
|
|
8
|
+
|
|
9
|
+
The wrapper uses the original Python executable, passes the original source as an unchanged
|
|
10
|
+
argument, and restores `sys.argv` before compiling it as `<string>` with `dont_inherit=True`.
|
|
11
|
+
It preserves shell quoting and variable expansion, exit status and the original health timing.
|
|
12
|
+
It uses no additional executable, changes no application environment variable, and preserves
|
|
13
|
+
existing cache settings inherited from either Compose or the image. Python before 3.8 simply
|
|
14
|
+
keeps its original bytecode behavior.
|
|
15
|
+
|
|
16
|
+
Only an existing writable `/tmp` tmpfs with an explicit size of at least 16 MiB qualifies.
|
|
17
|
+
The cache lives at `/tmp/.gowalk-healthcheck-pycache` inside that same container and mount;
|
|
18
|
+
the wrapper neither creates a new mount nor changes its size, ownership or permissions.
|
|
19
|
+
The cache directory is created with mode 0700. An existing directory must be owned by the probe's
|
|
20
|
+
effective UID, private, writable and not a symlink. Its parent must belong to that UID or root and
|
|
21
|
+
be sticky or not group/other-writable. Unsafe paths keep the original uncached probe; the wrapper
|
|
22
|
+
never changes permissions, deletes an existing path or trusts another user's cache.
|
|
23
|
+
The measured Python 3.12 urllib probe cache used 1.5 MiB. CPython generates its own bytecode
|
|
24
|
+
for the running interpreter and source, validates normal cache tags and source metadata,
|
|
25
|
+
and discards this ephemeral cache on container recreation. Cache write failures retain
|
|
26
|
+
normal Python import behavior. Startup imports before the wrapper remain unchanged.
|
|
27
|
+
|
|
28
|
+
The deployed application, dependencies, credentials, volumes, read-only root filesystem,
|
|
29
|
+
resource limits and probe cadence are preserved. Install the integrity-verified published
|
|
30
|
+
host runtime to make this transform durable for future deployments; installing the helper
|
|
31
|
+
alone does not recreate existing containers. See [host runtime installation](BACKEND_SSH_GATE.md).
|
|
32
|
+
|
|
33
|
+
For build services inheriting their Dockerfile healthcheck, `runtime_image_health.py` reads only the
|
|
34
|
+
local image ID and healthcheck after a successful build, under the existing host start lock. It
|
|
35
|
+
wraps only a missing Compose test, retaining explicit timing and disabled/custom checks. The private
|
|
36
|
+
launch model pins the transformed service to that inspected image ID with pulling disabled, so a
|
|
37
|
+
moving tag cannot substitute another image's healthcheck. Explicit registry-refresh policies and
|
|
38
|
+
non-build services retain their existing image/pull behavior. No pre-build image healthcheck is reused.
|
|
39
|
+
|
|
40
|
+
## Adopt an existing API
|
|
41
|
+
|
|
42
|
+
The released root-only `health_rollout.py` is an explicit operator maintenance command, outside the
|
|
43
|
+
panel's forced SSH command vocabulary. Select one exact Compose project, API service, current full
|
|
44
|
+
container ID and immutable image ID. Inspect eligibility before applying the same selection:
|
|
45
|
+
|
|
46
|
+
```sh
|
|
47
|
+
python3 -B /opt/gowalk-cicd-runtime/<version>/backend-action/health_rollout.py \
|
|
48
|
+
<project> <api|app|backend|web> <64-character-container-id> sha256:<64-character-image-id> --dry-run
|
|
49
|
+
python3 -B /opt/gowalk-cicd-runtime/<version>/backend-action/health_rollout.py \
|
|
50
|
+
<project> <api|app|backend|web> <64-character-container-id> sha256:<64-character-image-id>
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Dry-run checks identity, ownership labels, host/profile freezes, current health and supported runtime
|
|
54
|
+
shape without running a probe or changing a container. Its receipt contains only project/service,
|
|
55
|
+
original/replacement IDs and a fixed status. Private configuration stays in process memory.
|
|
56
|
+
The command never resolves current app source, rebuilds, pulls, runs Compose, changes ingress,
|
|
57
|
+
recreates sibling services, or removes volumes/images. Future ordinary deploys apply the same released
|
|
58
|
+
runtime transform; no app source or private profile override is needed.
|
|
59
|
+
|
|
60
|
+
Only running healthy APIs with read-only root filesystems, supported named volumes/binds and the
|
|
61
|
+
recognized bounded-tmpfs health probe qualify. Writable root filesystems, anonymous/undeclared mounts,
|
|
62
|
+
workers/datastores/one-off containers, auto-remove, host/shared namespaces, namespace/volume/link consumers,
|
|
63
|
+
static IP assignments, dynamic published ports and custom shells refuse. Existing health timing must
|
|
64
|
+
have at least a 30-second startup grace, startup checks at most five seconds apart and a timeout at most
|
|
65
|
+
five seconds. Stop timeout must be at most ten seconds. These preserve health cadence while permitting
|
|
66
|
+
bounded adoption and rollback. Unsupported configurations keep their existing runtime unchanged.
|
|
67
|
+
|
|
68
|
+
The helper holds the same host start lock as ordinary deployment for a 90-second transaction budget.
|
|
69
|
+
It rechecks exact container/image/configuration and retains the original container under a deterministic
|
|
70
|
+
`.gowalk-health-<old-id-prefix>` name. The replacement uses the existing image ID and copies private env,
|
|
71
|
+
command, entrypoint, user, limits, mounts, published ports and actual network aliases; only its health
|
|
72
|
+
command and the `io.gowalk.backend.health-original` maintenance label change. Its trusted memory-agent
|
|
73
|
+
admission-credit record is copied to the replacement ID before start, preserving future redeploy credit.
|
|
74
|
+
Malformed original records refuse before mutation. The existing memory timer may still adjust the old
|
|
75
|
+
reservation; a race causes restoration that preserves that latest value.
|
|
76
|
+
|
|
77
|
+
The replacement must pass exact configuration/mount/network readback, Docker health and the original
|
|
78
|
+
HTTP health command before the old container is removed without volumes. Failures remove only the
|
|
79
|
+
marker-bound replacement, restore/start the original and verify its original health before returning
|
|
80
|
+
`rollout_failed_restored`; the replacement's admission record is then removed. A lost reply is confirmed
|
|
81
|
+
by identity/state readback. `rollback_unconfirmed` means recovery needs another exact readback/replay;
|
|
82
|
+
retained containers and records stay available, and unrelated containers are never removed to clear a name.
|
|
83
|
+
|
|
84
|
+
Replay with the same original ID resumes an interrupted retained-container transaction. After successful
|
|
85
|
+
cleanup it finds the marker-bound healthy replacement and returns `already_applied`. It refuses absent,
|
|
86
|
+
ambiguous or changed runtime evidence. Start with one canary, verify application health externally and
|
|
87
|
+
measure load, then use bounded sequential selections for the remaining eligible services. Never turn this
|
|
88
|
+
maintenance command into a blanket fleet restart.
|
package/docs/BACKEND_MEMORY.md
CHANGED
|
@@ -10,9 +10,11 @@ that started at least ten minutes ago, it first measures the working set (`memor
|
|
|
10
10
|
container's limit (`docker update --memory-reservation`). App containers rise at once and fall by 8 MiB or more.
|
|
11
11
|
Datastores (PostgreSQL, PostGIS, TimescaleDB, Redis, Valkey, KeyDB, MySQL, MariaDB, MongoDB or Memcached by
|
|
12
12
|
image, service or command, or service `db`/`database`) only rise, and only with memory beyond their file cache;
|
|
13
|
-
-
|
|
13
|
+
- only when host `MemAvailable` is below 30% of `MemTotal` (the headroom outside admission's 70% reservation cap),
|
|
14
|
+
while swap is present, zswap is enabled (`/sys/module/zswap/parameters/enabled` is `Y`) and `memory.pressure`
|
|
14
15
|
`some avg60` is below 0.5, writes a quarter of an app container's anonymous memory to `memory.reclaim`,
|
|
15
|
-
compressing cold pages into zswap. Datastores are never reclaimed.
|
|
16
|
+
compressing cold pages into zswap. Datastores are never reclaimed. With ample host memory, reservation
|
|
17
|
+
measurement and updates continue without proactive page eviction; reclaim resumes when headroom falls.
|
|
16
18
|
|
|
17
19
|
Before its first change to a container, the agent records the reservation admission counted (bytes; 64 MiB when
|
|
18
20
|
unset) in `/var/lib/gowalk-backend-memory/<full container id>` (root-owned, 0644, atomic, fsynced, never
|
|
@@ -76,7 +78,8 @@ record, prune, `docker update` or reclaim). A real run prints a `{"name","id","f
|
|
|
76
78
|
per changed container and a line per error, then the `gowalk-cicd/backend-memory.v1` summary that
|
|
77
79
|
`journalctl _SYSTEMD_UNIT=gowalk-backend-memory.service -n 1 -o cat` shows: `reserved_mib` is the
|
|
78
80
|
admission-equivalent sum after the run, `headroom_mib` what remains of `cap_mib`, `released_mib` what the run
|
|
79
|
-
returned, `zswap` whether reclaim was allowed
|
|
81
|
+
returned, `zswap` whether reclaim was allowed by host headroom and swap availability, `partial` the reclaims
|
|
82
|
+
completed only in part and `native_mib` null
|
|
80
83
|
(natives excluded) when native discovery failed. Only unavailable Docker exits 1.
|
|
81
84
|
|
|
82
85
|
Roll back with `systemctl disable --now gowalk-backend-memory.timer && systemctl stop gowalk-backend-memory.service`
|
package/docs/BACKEND_SSH_GATE.md
CHANGED
|
@@ -130,7 +130,7 @@ socket observations before returning. A changed or ambiguous observation refuses
|
|
|
130
130
|
- `namespace` reports local rootful Linux Docker, existing host forwarding, read-only XFRM/netfilter
|
|
131
131
|
availability and the selected bridge's observed masquerade rule. Existing VPN settings are observed only
|
|
132
132
|
for this app's exact service. `isolated_net_admin` requires that bridge, no privileged mode, dropping ALL,
|
|
133
|
-
only NET_ADMIN plus optional NET_BIND_SERVICE/CHOWN/SETUID/SETGID for bootstrap and privilege dropping,
|
|
133
|
+
only NET_ADMIN plus optional NET_BIND_SERVICE/CHOWN/SETUID/SETGID/SETPCAP for bootstrap and privilege dropping,
|
|
134
134
|
and namespace-only forwarding/redirect settings. This observes grants, not that a daemon actually dropped privileges.
|
|
135
135
|
An absent VPN reports `absent`; these observations do not test creating a namespace or applying capabilities.
|
|
136
136
|
- `firewall` reports observed INPUT/FORWARD policies and explicit UDP ACCEPT rules. Effective ingress
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gowalk-cicd",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.195",
|
|
4
4
|
"description": "Zero-config GitHub Actions delivery for iOS TestFlight and Android Google Play.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
"docs/BACKEND_SSH_GATE.md",
|
|
43
43
|
"docs/BACKEND_NATIVE.md",
|
|
44
44
|
"docs/BACKEND_MEMORY.md",
|
|
45
|
+
"docs/BACKEND_HEALTHCHECKS.md",
|
|
45
46
|
"docs/CONFIGURATION.md",
|
|
46
47
|
"docs/CI_PERFORMANCE.md",
|
|
47
48
|
"docs/LOCAL_VALIDATION.md"
|