gowalk-cicd 1.0.200 → 1.0.202
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend-action/health_rollout.py +5 -2
- package/backend-action/health_rollout_engine.py +23 -9
- package/backend-action/health_rollout_spec.py +4 -4
- package/backend-action/health_rollout_timing.py +34 -0
- package/backend-action/install_runtime.py +1 -0
- package/docs/BACKEND_HEALTHCHECKS.md +14 -5
- package/package.json +1 -1
- package/src/config/build-steps.mjs +7 -3
- package/templates/history.json.gz +0 -0
|
@@ -39,6 +39,7 @@ def completed(engine, app, service, identifier, image, dry_run):
|
|
|
39
39
|
scope(current, app, service, image)
|
|
40
40
|
if not current['State']['Running'] or current['State'].get('Health', {}).get('Status') != 'healthy':
|
|
41
41
|
raise ValueError('replacement_health_unconfirmed')
|
|
42
|
+
engine.configure(current['Config'])
|
|
42
43
|
if not dry_run:
|
|
43
44
|
probe(engine, current['Id'], current['Config']['Healthcheck']['Test'])
|
|
44
45
|
return receipt(app, service, identifier, current['Id'], 'already_applied')
|
|
@@ -71,6 +72,7 @@ def confirmed(engine, method, path, proof):
|
|
|
71
72
|
|
|
72
73
|
|
|
73
74
|
def rollback(engine, old, name, reservation):
|
|
75
|
+
engine.recover()
|
|
74
76
|
if engine.pending_stop is not None:
|
|
75
77
|
engine.stop(old['Id'])
|
|
76
78
|
replacement = engine.inspect(name, missing=True)
|
|
@@ -88,7 +90,7 @@ def rollback(engine, old, name, reservation):
|
|
|
88
90
|
if not retained['State']['Running']:
|
|
89
91
|
confirmed(engine, 'POST', '/containers/' + old['Id'] + '/start',
|
|
90
92
|
lambda: engine.inspect(old['Id'])['State']['Running'])
|
|
91
|
-
healthy(engine, old['Id'], engine.deadline)
|
|
93
|
+
healthy(engine, old['Id'], min(time.monotonic() + engine.timing.health_seconds, engine.deadline))
|
|
92
94
|
probe(engine, old['Id'], old['Config']['Healthcheck']['Test'])
|
|
93
95
|
if replacement is not None and replacement['Id'] != old['Id']:
|
|
94
96
|
credit.forget(replacement['Id'], reservation)
|
|
@@ -97,7 +99,7 @@ def rollback(engine, old, name, reservation):
|
|
|
97
99
|
def apply(engine, app, service, old, body, name, staged, original_profile, reservation):
|
|
98
100
|
try:
|
|
99
101
|
replacement = candidate(engine, old, body, name, staged, reservation)
|
|
100
|
-
current = healthy(engine, replacement, min(time.monotonic() +
|
|
102
|
+
current = healthy(engine, replacement, min(time.monotonic() + engine.timing.health_seconds, engine.deadline))
|
|
101
103
|
verified(current, body, old)
|
|
102
104
|
probe(engine, replacement, old['Config']['Healthcheck']['Test'])
|
|
103
105
|
host_enabled()
|
|
@@ -131,6 +133,7 @@ def rollout(app, service, identifier, image, dry_run=False):
|
|
|
131
133
|
raise ValueError('runtime_changed')
|
|
132
134
|
scope(old, app, service, image)
|
|
133
135
|
body = specification(old)
|
|
136
|
+
engine.configure(old['Config'])
|
|
134
137
|
reservation = credit.original(old)
|
|
135
138
|
suffix = '.gowalk-health-' + identifier[:12]
|
|
136
139
|
current_name = old['Name'].lstrip('/')
|
|
@@ -6,15 +6,26 @@ import subprocess
|
|
|
6
6
|
import time
|
|
7
7
|
from urllib.parse import quote
|
|
8
8
|
|
|
9
|
+
from health_rollout_timing import Timing
|
|
10
|
+
|
|
9
11
|
MARKER = 'io.gowalk.backend.health-original'
|
|
10
12
|
SOCKET = '/var/run/docker.sock'
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class Engine:
|
|
14
16
|
def __init__(self):
|
|
15
|
-
self.
|
|
17
|
+
self.started = time.monotonic()
|
|
18
|
+
self.configure({'Healthcheck': {'Timeout': 5000000000}})
|
|
16
19
|
self.pending_stop = None
|
|
17
20
|
|
|
21
|
+
def configure(self, config):
|
|
22
|
+
self.timing = Timing(config)
|
|
23
|
+
self.deadline = self.started + self.timing.forward_seconds
|
|
24
|
+
self.recovery_deadline = self.deadline + self.timing.rollback_seconds
|
|
25
|
+
|
|
26
|
+
def recover(self):
|
|
27
|
+
self.deadline = self.recovery_deadline
|
|
28
|
+
|
|
18
29
|
def request(self, method, path, body=None, missing=False, raw=False, timeout=10):
|
|
19
30
|
remaining = self.deadline - time.monotonic()
|
|
20
31
|
if remaining <= 0:
|
|
@@ -36,14 +47,16 @@ class Engine:
|
|
|
36
47
|
finally:
|
|
37
48
|
connection.close()
|
|
38
49
|
|
|
39
|
-
def inspect(self, identifier, missing=False):
|
|
40
|
-
return self.request('GET', '/containers/' + quote(identifier, safe='') + '/json',
|
|
50
|
+
def inspect(self, identifier, missing=False, timeout=10):
|
|
51
|
+
return self.request('GET', '/containers/' + quote(identifier, safe='') + '/json',
|
|
52
|
+
missing=missing, timeout=timeout)
|
|
41
53
|
|
|
42
54
|
def stop(self, identifier):
|
|
43
55
|
# Docker continues an accepted stop after a lost HTTP reply. Keep that uncertainty
|
|
44
56
|
# until the same-ID stop completes and its stopped state is positively observed.
|
|
45
57
|
self.pending_stop = identifier
|
|
46
|
-
self.request('POST', '/containers/' + identifier + '/stop?t=
|
|
58
|
+
self.request('POST', '/containers/' + identifier + '/stop?t=' + str(self.timing.stop_seconds),
|
|
59
|
+
timeout=self.timing.stop_transport)
|
|
47
60
|
if self.inspect(identifier)['State']['Running']:
|
|
48
61
|
raise ValueError('container_stop_unconfirmed')
|
|
49
62
|
self.pending_stop = None
|
|
@@ -76,11 +89,12 @@ def probe(engine, identifier, test):
|
|
|
76
89
|
command = test[1:] if test[0] == 'CMD' else ['/bin/sh', '-c', test[1]]
|
|
77
90
|
created = engine.request('POST', '/containers/' + identifier + '/exec',
|
|
78
91
|
{'AttachStdout': False, 'AttachStderr': False, 'Cmd': command})
|
|
79
|
-
|
|
80
|
-
|
|
92
|
+
deadline = min(time.monotonic() + engine.timing.probe_seconds, engine.deadline)
|
|
93
|
+
engine.request('POST', '/exec/' + created['Id'] + '/start', {'Detach': False, 'Tty': False},
|
|
94
|
+
raw=True, timeout=max(0, deadline - time.monotonic()))
|
|
81
95
|
while time.monotonic() < deadline:
|
|
82
96
|
result = engine.request('GET', '/exec/' + created['Id'] + '/json',
|
|
83
|
-
timeout=max(0
|
|
97
|
+
timeout=max(0, deadline - time.monotonic()))
|
|
84
98
|
if not result['Running']:
|
|
85
99
|
if result['ExitCode'] != 0:
|
|
86
100
|
raise ValueError('original_probe_failed')
|
|
@@ -91,11 +105,11 @@ def probe(engine, identifier, test):
|
|
|
91
105
|
|
|
92
106
|
def healthy(engine, identifier, deadline):
|
|
93
107
|
while time.monotonic() < deadline:
|
|
94
|
-
current = engine.inspect(identifier)
|
|
108
|
+
current = engine.inspect(identifier, timeout=min(10, max(0, deadline - time.monotonic())))
|
|
95
109
|
status = current['State'].get('Health', {}).get('Status')
|
|
96
110
|
if current['State']['Running'] and status == 'healthy':
|
|
97
111
|
return current
|
|
98
112
|
if not current['State']['Running'] or status == 'unhealthy':
|
|
99
113
|
raise ValueError('container_health_failed')
|
|
100
|
-
time.sleep(1)
|
|
114
|
+
time.sleep(min(1, max(0, deadline - time.monotonic())))
|
|
101
115
|
raise ValueError('container_health_timeout')
|
|
@@ -3,6 +3,7 @@ import copy
|
|
|
3
3
|
import re
|
|
4
4
|
|
|
5
5
|
from health_rollout_engine import MARKER
|
|
6
|
+
from health_rollout_timing import probe_seconds, stop_seconds
|
|
6
7
|
from runtime_healthcheck import cache_healthcheck
|
|
7
8
|
|
|
8
9
|
ROLES = frozenset(('api', 'app', 'backend', 'web'))
|
|
@@ -79,15 +80,14 @@ def specification(row):
|
|
|
79
80
|
raise ValueError('runtime_unsupported')
|
|
80
81
|
if row['Config'].get('Shell') not in (None, [], ['/bin/sh', '-c']):
|
|
81
82
|
raise ValueError('custom_shell_unsupported')
|
|
82
|
-
|
|
83
|
-
raise ValueError('stop_timeout_unsupported')
|
|
83
|
+
stop_seconds(row['Config'])
|
|
84
84
|
published_ports(row)
|
|
85
85
|
mounts(row)
|
|
86
86
|
health = copy.deepcopy(row['Config'].get('Healthcheck') or {})
|
|
87
87
|
if (health.get('StartPeriod', 0) < 30000000000
|
|
88
|
-
or (health.get('StartInterval') or 5000000000) > 5000000000
|
|
89
|
-
or (health.get('Timeout') or 30000000000) > 5000000000):
|
|
88
|
+
or (health.get('StartInterval') or 5000000000) > 5000000000):
|
|
90
89
|
raise ValueError('health_startup_unsupported')
|
|
90
|
+
probe_seconds(row['Config'])
|
|
91
91
|
service = {'healthcheck': {'test': health.get('Test')},
|
|
92
92
|
'tmpfs': [name + ':' + options for name, options in (host.get('Tmpfs') or {}).items()],
|
|
93
93
|
'environment': dict(value.split('=', 1) for value in row['Config'].get('Env') or [])}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Preserve supported Docker deadlines and reserve a separate, finite recovery window."""
|
|
2
|
+
|
|
3
|
+
REQUEST_SECONDS = 10
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def stop_seconds(config):
|
|
7
|
+
value = config.get('StopTimeout')
|
|
8
|
+
value = 10 if value is None else value
|
|
9
|
+
if type(value) is not int or not 0 <= value <= 35:
|
|
10
|
+
raise ValueError('stop_timeout_unsupported')
|
|
11
|
+
return value
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def probe_seconds(config):
|
|
15
|
+
value = (config.get('Healthcheck') or {}).get('Timeout') or 30000000000
|
|
16
|
+
if type(value) is not int or not 0 < value <= 10000000000:
|
|
17
|
+
raise ValueError('health_startup_unsupported')
|
|
18
|
+
return value / 1000000000
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Timing:
|
|
22
|
+
def __init__(self, config):
|
|
23
|
+
self.stop_seconds = stop_seconds(config)
|
|
24
|
+
self.probe_seconds = probe_seconds(config)
|
|
25
|
+
self.stop_transport = self.stop_seconds + REQUEST_SECONDS
|
|
26
|
+
self.health_seconds = 35 + self.probe_seconds
|
|
27
|
+
# Preflight: three reads and the bounded 15s consumer inventory. Control
|
|
28
|
+
# allowances include positive state readback after a lost mutation reply.
|
|
29
|
+
self.forward_seconds = (45 + 11 * REQUEST_SECONDS + self.stop_transport
|
|
30
|
+
+ self.health_seconds + self.probe_seconds)
|
|
31
|
+
# Recovery can settle one outstanding stop, remove the marked candidate,
|
|
32
|
+
# restore/start the original, then confirm Docker and original-probe health.
|
|
33
|
+
self.rollback_seconds = (10 * REQUEST_SECONDS + self.stop_transport
|
|
34
|
+
+ self.health_seconds + self.probe_seconds)
|
|
@@ -21,6 +21,7 @@ REQUIRED = (
|
|
|
21
21
|
'runtime_projection.py',
|
|
22
22
|
'identity_diagnostics.py',
|
|
23
23
|
'health_rollout.py', 'health_rollout_engine.py', 'health_rollout_spec.py', 'health_rollout_credit.py',
|
|
24
|
+
'health_rollout_timing.py',
|
|
24
25
|
'memory_agent.py',
|
|
25
26
|
'runtime_env.py', 'runtime_env_parse.py', 'edge_inputs.py', 'edge_vhost.py', 'edge_health.py',
|
|
26
27
|
'configure_vhost.py', 'vhost_diagnostics.py', 'remote_deploy.sh', 'deploy_evidence.sh',
|
|
@@ -66,10 +66,15 @@ recognized bounded-tmpfs health probe qualify. Writable root filesystems, anonym
|
|
|
66
66
|
workers/datastores/one-off containers, auto-remove, host/shared namespaces, namespace/volume/link consumers,
|
|
67
67
|
static IP assignments, dynamic published ports and custom shells refuse. Existing health timing must
|
|
68
68
|
have at least a 30-second startup grace, startup checks at most five seconds apart and a timeout at most
|
|
69
|
-
|
|
69
|
+
ten seconds. Stop timeout must be finite and at most 35 seconds. The configured values are preserved,
|
|
70
|
+
including an explicit zero-second stop grace; an omitted stop timeout uses Docker's ten-second default.
|
|
71
|
+
These preserve health cadence while permitting
|
|
70
72
|
bounded adoption and rollback. Unsupported configurations keep their existing runtime unchanged.
|
|
71
73
|
|
|
72
|
-
The helper holds the same host start lock as ordinary deployment for a
|
|
74
|
+
The helper holds the same host start lock as ordinary deployment for a bounded transaction. Its timing
|
|
75
|
+
policy derives separate forward and recovery windows from the selected container's configured deadlines;
|
|
76
|
+
the forward phase cannot consume recovery's allowance. The maximum total window is 455 seconds (35-second
|
|
77
|
+
stop grace and ten-second probe). Successful operations return immediately, without waiting out that window.
|
|
73
78
|
It rechecks exact container/image/configuration and retains the original container under a deterministic
|
|
74
79
|
`.gowalk-health-<old-id-prefix>` name. The replacement uses the existing image ID and copies private env,
|
|
75
80
|
command, entrypoint, user, limits, mounts, published ports and actual network aliases; only its health
|
|
@@ -86,10 +91,14 @@ by identity/state readback. `rollback_unconfirmed` means recovery needs another
|
|
|
86
91
|
retained containers and records stay available, and unrelated containers are never removed to clear a name.
|
|
87
92
|
|
|
88
93
|
Docker may report the default `OomKillDisable` as either null or false; these alone compare equivalently,
|
|
89
|
-
while an enabled OOM-kill override must match exactly.
|
|
90
|
-
|
|
94
|
+
while an enabled OOM-kill override must match exactly. Each stop request uses the configured grace with
|
|
95
|
+
an additional ten-second transport allowance. An uncertain stop must settle on
|
|
91
96
|
that exact container before rollback can restart it or certify its health. Original-probe verification
|
|
92
|
-
waits for the exec's confirmed exit status
|
|
97
|
+
waits for the exec's confirmed exit status within the configured probe timeout, including exec-start time.
|
|
98
|
+
Docker-health polling has a separate window of 35 seconds plus that probe timeout. Each ordinary Docker
|
|
99
|
+
request is limited to ten seconds and the remaining phase/health/probe budget. The forward allowance
|
|
100
|
+
includes 45 seconds for preflight and eleven control requests; recovery reserves ten control requests,
|
|
101
|
+
another stop, Docker-health polling and an original probe. Recovery never extends its original deadline.
|
|
93
102
|
|
|
94
103
|
Replay with the same original ID resumes an interrupted retained-container transaction. After successful
|
|
95
104
|
cleanup it finds the marker-bound healthy replacement and returns `already_applied`. It refuses absent,
|
package/package.json
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
// Reviewed build-only recipes. Names alone never authorize candidate execution.
|
|
2
|
-
const RUBY =
|
|
2
|
+
const RUBY = new Set([
|
|
3
|
+
'ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b',
|
|
4
|
+
'ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8',
|
|
5
|
+
]);
|
|
3
6
|
// The optional grep drops beta and release-candidate Xcodes, whose builds App Store Connect refuses.
|
|
4
7
|
const XCODE = /^XCODE_PATH=\$\(ls -d \/Applications\/Xcode_\d+\*\.app (?:\| grep -viE 'beta\|release_candidate\|_rc' )?\| sort -V \| tail -1\)\n/;
|
|
5
8
|
const XCODE_END = 'sudo xcode-select -s "$XCODE_PATH/Contents/Developer"\nxcodebuild -version\nswift --version\n';
|
|
@@ -13,9 +16,10 @@ const SDK = /^\s*"\$ANDROID_HOME\/cmdline-tools\/latest\/bin\/sdkmanager" 'platf
|
|
|
13
16
|
export function reviewedBuildStep(step) {
|
|
14
17
|
const allowed = ['name', 'id', 'uses', 'with', 'run', 'if', 'shell'];
|
|
15
18
|
if (!step || Object.keys(step).some(key => !allowed.includes(key))) return false;
|
|
16
|
-
if (step.uses
|
|
19
|
+
if (RUBY.has(step.uses) && !step.run) {
|
|
17
20
|
return Object.entries(step.with || {}).every(([key, value]) =>
|
|
18
|
-
|
|
21
|
+
key === 'bundler-cache' ? value === false
|
|
22
|
+
: ['ruby-version', 'bundler'].includes(key) && /^\d+(?:\.\d+){0,2}$/.test(String(value)));
|
|
19
23
|
}
|
|
20
24
|
if (step.uses || step.with || step.shell && step.shell !== 'bash' || typeof step.run !== 'string') return false;
|
|
21
25
|
return XCODE.test(step.run) && step.run.replace(XCODE, '') === XCODE_END
|
|
Binary file
|