@p10i/create-nestbox 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +0 -4
- package/CHANGELOG.md +7 -0
- package/INSTALL.md +2 -2
- package/README.md +1 -1
- package/bin/create-nestbox.js +2 -25
- package/docker/control/Dockerfile +3 -2
- package/docker/control/host_bridge.py +205 -0
- package/docker/control/server.py +113 -54
- package/docker-compose.yaml +0 -9
- package/home/configs/nginx/nginx.conf +0 -17
- package/home/configs/opencode/control-mcp.js +1 -1
- package/home/configs/opencode/instructions.md +1 -2
- package/host-runner.mjs +134 -112
- package/package.json +1 -1
- package/scripts/check-package.js +1 -0
- package/tests/control.mjs +74 -13
- package/tests/creator.mjs +3 -3
- package/tests/host-runner.mjs +137 -65
- package/tests/nestbox.ps1 +4 -9
- package/tests/nestbox.sh +6 -7
package/.env.example
CHANGED
|
@@ -8,10 +8,6 @@ COMPOSE_PROJECT_NAME=nestbox-change-me
|
|
|
8
8
|
# directory, or .. when this installation is <workspace>/.nestbox.
|
|
9
9
|
WORKSPACE_PATH=.
|
|
10
10
|
|
|
11
|
-
# The native runner uses the shared WEB_PORT gateway. The creator generates a
|
|
12
|
-
# private token file outside the workspace and mounts it only into control.
|
|
13
|
-
NESTBOX_HOST_TOKEN_FILE=
|
|
14
|
-
|
|
15
11
|
# Override for rootless or nonstandard Docker Engine installations.
|
|
16
12
|
DOCKER_SOCKET_PATH=/var/run/docker.sock
|
|
17
13
|
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.5.0
|
|
4
|
+
|
|
5
|
+
- Replace the public token-authenticated host API with a persistent `docker compose exec -T control` stdio bridge.
|
|
6
|
+
- Bind the bridge API to container loopback and add reconnect-safe runner sessions with idempotent result replay.
|
|
7
|
+
- Remove host tokens, the Nginx host proxy, and all `4089` exposure while retaining one Nginx-published host port.
|
|
8
|
+
- Existing `0.3.x` and `0.4.x` installations may remove `NESTBOX_HOST_TOKEN_FILE` and its referenced token file after upgrading.
|
|
9
|
+
|
|
3
10
|
## 0.4.0
|
|
4
11
|
|
|
5
12
|
- Route the authenticated host runner through `/_nestbox/host/` on the existing Nginx gateway.
|
package/INSTALL.md
CHANGED
|
@@ -101,8 +101,8 @@ Configure the host bridge in the workspace root before Compose starts:
|
|
|
101
101
|
- For `.nestbox` layout, create `package.json` from `.nestbox/package.host.json` when absent, or merge only its `host` and `test:host` scripts into the existing manifest.
|
|
102
102
|
- For direct layout, use `node host-runner.mjs` and `node tests/host-runner.mjs` for those scripts. A source clone's publisher-oriented `package.json` may be replaced by this installation manifest after preserving any intentional project metadata.
|
|
103
103
|
- Never overwrite a conflicting script. Stop and ask for a different script name.
|
|
104
|
-
-
|
|
105
|
-
-
|
|
104
|
+
- Start `npm run host --` after `control` is healthy. The runner opens a persistent `docker compose exec -T control` stdio bridge from the installation directory. The image-resident helper sends heartbeats and job results through the server's container-loopback API without a host listener, token, published port, or shared queue files.
|
|
105
|
+
- If the runner reports a stale control image, rebuild and recreate `control` with `docker compose build control` and `docker compose up -d control`.
|
|
106
106
|
- The workspace manifest is mounted read-only in runtime containers. Confirmed changes are written only by the native host runner.
|
|
107
107
|
|
|
108
108
|
Update the new `home/configs/opencode/instance.md` with:
|
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@ Nestbox runs with Docker Compose using Nginx, PHP-FPM, Rollup, OpenCode, and a p
|
|
|
12
12
|
|
|
13
13
|
The `control` service is the only container with `/var/run/docker.sock`. Its internal-only API permits exec only when caller, target, project, and root-user labels allow it. OpenCode reaches the API through the bundled control MCP and can target PHP-FPM and Rollup by default. Trusted PHP pages can use `nestbox_control_exec()` and are restricted to PHP-FPM self-exec.
|
|
14
14
|
|
|
15
|
-
Native host commands are limited to scripts already declared in the workspace `package.json`. Keep `npm run host --` running from the workspace when host scripts are needed. The native runner
|
|
15
|
+
Native host commands are limited to scripts already declared in the workspace `package.json`. Keep `npm run host --` running from the workspace when host scripts are needed. The native runner maintains a `docker compose exec -T control` stdio bridge; its image-resident helper talks only to the control server on container loopback. No host API, runner token, second published port, or shared filesystem queue is used. MCP can request script additions, edits, or deletions, but the host runner applies them only after the user supplies a short-lived code displayed exclusively in the host terminal.
|
|
16
16
|
|
|
17
17
|
Control emits bounded structured events to ordinary container logs. Use `docker compose logs control`; no telemetry collector, database, or observability dependency is installed. Docker socket access is root-equivalent on the host, and host scripts execute in the trusted writable workspace, so neither mechanism is a sandbox.
|
|
18
18
|
|
package/bin/create-nestbox.js
CHANGED
|
@@ -1,22 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { spawnSync } from 'node:child_process';
|
|
3
|
-
import { randomBytes, randomUUID } from 'node:crypto';
|
|
4
|
-
import { homedir } from 'node:os';
|
|
5
3
|
import { createInterface } from 'node:readline/promises';
|
|
6
4
|
import { stdin as input, stdout as output } from 'node:process';
|
|
7
|
-
import { dirname,
|
|
5
|
+
import { dirname, join, resolve } from 'node:path';
|
|
8
6
|
import { fileURLToPath } from 'node:url';
|
|
9
|
-
import { cp, mkdir, readdir, readFile,
|
|
7
|
+
import { cp, mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
10
8
|
import { constants } from 'node:fs';
|
|
11
9
|
|
|
12
10
|
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
13
11
|
const packageJson = JSON.parse(await readFile(join(packageRoot, 'package.json'), 'utf8'));
|
|
14
12
|
|
|
15
|
-
function containsPath(root, target) {
|
|
16
|
-
const child = relative(resolve(root), resolve(target));
|
|
17
|
-
return child === '' || (!child.startsWith(`..${sep}`) && child !== '..' && !isAbsolute(child));
|
|
18
|
-
}
|
|
19
|
-
|
|
20
13
|
const copyEntries = [
|
|
21
14
|
'docker',
|
|
22
15
|
'home',
|
|
@@ -227,22 +220,6 @@ async function configureHostPackage(installDir, workspaceDir, layout) {
|
|
|
227
220
|
private: true,
|
|
228
221
|
scripts
|
|
229
222
|
}, null, 2)}\n`, 'utf8');
|
|
230
|
-
const envPath = join(installDir, '.env');
|
|
231
|
-
const token = randomBytes(32).toString('hex');
|
|
232
|
-
const stateBase = process.env.NESTBOX_CREATOR_STATE_ROOT
|
|
233
|
-
|| process.env.LOCALAPPDATA
|
|
234
|
-
|| process.env.XDG_STATE_HOME
|
|
235
|
-
|| (process.platform === 'darwin' ? join(homedir(), 'Library', 'Application Support') : join(homedir(), '.local', 'state'));
|
|
236
|
-
const tokenDirectory = join(stateBase, 'nestbox', 'host-control');
|
|
237
|
-
await mkdir(tokenDirectory, { recursive: true });
|
|
238
|
-
const tokenPath = join(await realpath(tokenDirectory), `${randomUUID()}.token`);
|
|
239
|
-
const canonicalWorkspace = await realpath(workspaceDir);
|
|
240
|
-
const canonicalInstall = await realpath(installDir);
|
|
241
|
-
if (containsPath(canonicalWorkspace, tokenPath) || containsPath(canonicalInstall, tokenPath)) throw new Error('Host token state directory must be outside the workspace and Nestbox installation.');
|
|
242
|
-
await writeFile(tokenPath, `${token}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
243
|
-
const composeTokenPath = tokenPath.replaceAll('\\', '/');
|
|
244
|
-
const configuredEnv = (await readFile(envPath, 'utf8')).replace(/^NESTBOX_HOST_TOKEN_FILE=.*\r?$/m, () => `NESTBOX_HOST_TOKEN_FILE=${composeTokenPath}`);
|
|
245
|
-
await writeFile(envPath, configuredEnv, 'utf8');
|
|
246
223
|
}
|
|
247
224
|
|
|
248
225
|
function commandExists(command) {
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
FROM python:3.12-alpine
|
|
2
2
|
|
|
3
3
|
COPY ./docker/control/server.py /usr/local/bin/nestbox-control
|
|
4
|
+
COPY ./docker/control/host_bridge.py /usr/local/bin/nestbox-host-bridge
|
|
4
5
|
|
|
5
|
-
RUN python -m py_compile /usr/local/bin/nestbox-control && \
|
|
6
|
-
chmod 0755 /usr/local/bin/nestbox-control
|
|
6
|
+
RUN python -m py_compile /usr/local/bin/nestbox-control /usr/local/bin/nestbox-host-bridge && \
|
|
7
|
+
chmod 0755 /usr/local/bin/nestbox-control /usr/local/bin/nestbox-host-bridge
|
|
7
8
|
|
|
8
9
|
WORKDIR /workspace
|
|
9
10
|
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
import http.client
|
|
3
|
+
import json
|
|
4
|
+
import signal
|
|
5
|
+
import sys
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
PROTOCOL_VERSION = 1
|
|
11
|
+
MAX_LINE_BYTES = 64 * 1024 * 1024
|
|
12
|
+
HEARTBEAT_SECONDS = 4
|
|
13
|
+
stopping = threading.Event()
|
|
14
|
+
input_condition = threading.Condition()
|
|
15
|
+
input_messages = []
|
|
16
|
+
input_error = None
|
|
17
|
+
input_eof = False
|
|
18
|
+
connections = set()
|
|
19
|
+
connections_lock = threading.Lock()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def diagnostic(message):
|
|
23
|
+
sys.stderr.write(f"nestbox host bridge: {message}\n")
|
|
24
|
+
sys.stderr.flush()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def stop_requests():
|
|
28
|
+
stopping.set()
|
|
29
|
+
with connections_lock:
|
|
30
|
+
active = list(connections)
|
|
31
|
+
for connection in active:
|
|
32
|
+
connection.close()
|
|
33
|
+
with input_condition:
|
|
34
|
+
input_condition.notify_all()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def send(message):
|
|
38
|
+
data = json.dumps({"protocolVersion": PROTOCOL_VERSION, **message}, separators=(",", ":"))
|
|
39
|
+
sys.stdout.write(data + "\n")
|
|
40
|
+
sys.stdout.flush()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def read_input():
|
|
44
|
+
global input_error, input_eof
|
|
45
|
+
try:
|
|
46
|
+
while True:
|
|
47
|
+
line = sys.stdin.buffer.readline(MAX_LINE_BYTES + 1)
|
|
48
|
+
if not line:
|
|
49
|
+
with input_condition:
|
|
50
|
+
input_eof = True
|
|
51
|
+
input_condition.notify_all()
|
|
52
|
+
stop_requests()
|
|
53
|
+
return
|
|
54
|
+
if len(line) > MAX_LINE_BYTES or not line.endswith(b"\n"):
|
|
55
|
+
raise ValueError("protocol line exceeds the limit")
|
|
56
|
+
value = json.loads(line)
|
|
57
|
+
if not isinstance(value, dict) or value.get("protocolVersion") != PROTOCOL_VERSION:
|
|
58
|
+
raise ValueError("unsupported protocol message")
|
|
59
|
+
with input_condition:
|
|
60
|
+
if len(input_messages) >= 2:
|
|
61
|
+
raise ValueError("too many queued protocol messages")
|
|
62
|
+
input_messages.append(value)
|
|
63
|
+
input_condition.notify_all()
|
|
64
|
+
except Exception as error:
|
|
65
|
+
with input_condition:
|
|
66
|
+
input_error = error
|
|
67
|
+
input_condition.notify_all()
|
|
68
|
+
stop_requests()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def receive():
|
|
72
|
+
with input_condition:
|
|
73
|
+
while not input_messages and not input_eof and input_error is None and not stopping.is_set():
|
|
74
|
+
input_condition.wait()
|
|
75
|
+
if input_error is not None:
|
|
76
|
+
raise input_error
|
|
77
|
+
if not input_messages:
|
|
78
|
+
raise EOFError
|
|
79
|
+
return input_messages.pop(0)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def request(method, path, runner_id, session_id, body=None, timeout=35):
|
|
83
|
+
payload = json.dumps(body, separators=(",", ":")).encode() if body is not None else None
|
|
84
|
+
headers = {
|
|
85
|
+
"X-Nestbox-Runner-Id": runner_id,
|
|
86
|
+
"X-Nestbox-Session-Id": session_id,
|
|
87
|
+
"Content-Type": "application/json",
|
|
88
|
+
}
|
|
89
|
+
connection = http.client.HTTPConnection("127.0.0.1", 4089, timeout=timeout)
|
|
90
|
+
with connections_lock:
|
|
91
|
+
connections.add(connection)
|
|
92
|
+
try:
|
|
93
|
+
connection.request(method, path, body=payload, headers=headers)
|
|
94
|
+
response = connection.getresponse()
|
|
95
|
+
data = response.read(MAX_LINE_BYTES + 1)
|
|
96
|
+
finally:
|
|
97
|
+
connection.close()
|
|
98
|
+
with connections_lock:
|
|
99
|
+
connections.discard(connection)
|
|
100
|
+
if len(data) > MAX_LINE_BYTES:
|
|
101
|
+
raise RuntimeError("control response exceeds the limit")
|
|
102
|
+
value = json.loads(data) if data else None
|
|
103
|
+
return response.status, value
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def heartbeat(runner_id, session_id, metadata):
|
|
107
|
+
while not stopping.wait(HEARTBEAT_SECONDS):
|
|
108
|
+
try:
|
|
109
|
+
status, value = request("POST", "/host/heartbeat", runner_id, session_id, metadata, timeout=5)
|
|
110
|
+
if status == 409:
|
|
111
|
+
diagnostic((value or {}).get("error", "session was superseded"))
|
|
112
|
+
stop_requests()
|
|
113
|
+
return
|
|
114
|
+
if status >= 400:
|
|
115
|
+
diagnostic(f"heartbeat returned HTTP {status}")
|
|
116
|
+
except Exception as error:
|
|
117
|
+
if not stopping.is_set():
|
|
118
|
+
diagnostic(f"heartbeat failed: {error}")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def deliver_result(runner_id, session_id, result):
|
|
122
|
+
job_id = result.get("jobId")
|
|
123
|
+
delay = 0.5
|
|
124
|
+
while not stopping.is_set():
|
|
125
|
+
try:
|
|
126
|
+
status, response = request("POST", f"/host/jobs/{quote(str(job_id), safe='')}/result", runner_id, session_id, result)
|
|
127
|
+
if status in (200, 202):
|
|
128
|
+
send({"type": "result.ack", "jobId": job_id, "duplicate": bool((response or {}).get("duplicate"))})
|
|
129
|
+
return
|
|
130
|
+
error = (response or {}).get("error", f"result returned HTTP {status}")
|
|
131
|
+
if status == 410:
|
|
132
|
+
send({"type": "gone", "jobId": job_id, "error": error})
|
|
133
|
+
return
|
|
134
|
+
if status == 409 and "session was superseded" in error.lower():
|
|
135
|
+
raise RuntimeError(error)
|
|
136
|
+
if status < 500:
|
|
137
|
+
send({"type": "result.rejected", "jobId": job_id, "error": error})
|
|
138
|
+
return
|
|
139
|
+
except (OSError, http.client.HTTPException) as error:
|
|
140
|
+
if stopping.is_set():
|
|
141
|
+
raise EOFError from error
|
|
142
|
+
diagnostic(f"result delivery failed: {error}; retrying")
|
|
143
|
+
stopping.wait(delay)
|
|
144
|
+
delay = min(10, delay * 2)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def main():
|
|
148
|
+
threading.Thread(target=read_input, daemon=True).start()
|
|
149
|
+
hello = receive()
|
|
150
|
+
if hello.get("type") != "hello":
|
|
151
|
+
raise ValueError("first protocol message must be hello")
|
|
152
|
+
runner_id, session_id = hello.get("runnerId"), hello.get("sessionId")
|
|
153
|
+
metadata = hello.get("metadata") or {}
|
|
154
|
+
if not isinstance(runner_id, str) or not isinstance(session_id, str) or not isinstance(metadata, dict):
|
|
155
|
+
raise ValueError("invalid hello message")
|
|
156
|
+
status, value = request("POST", "/host/connect", runner_id, session_id, metadata)
|
|
157
|
+
if status >= 400:
|
|
158
|
+
raise RuntimeError(f"connect returned HTTP {status}: {(value or {}).get('error', value)}")
|
|
159
|
+
thread = threading.Thread(target=heartbeat, args=(runner_id, session_id, metadata), daemon=True)
|
|
160
|
+
thread.start()
|
|
161
|
+
retained_result = hello.get("retainedResult")
|
|
162
|
+
if retained_result is not None:
|
|
163
|
+
if not isinstance(retained_result, dict):
|
|
164
|
+
raise ValueError("invalid retained result")
|
|
165
|
+
deliver_result(runner_id, session_id, retained_result)
|
|
166
|
+
if stopping.is_set():
|
|
167
|
+
return
|
|
168
|
+
send({"type": "ready"})
|
|
169
|
+
while not stopping.is_set():
|
|
170
|
+
status, value = request("GET", "/host/jobs/next?timeout=25", runner_id, session_id, timeout=30)
|
|
171
|
+
if status == 204:
|
|
172
|
+
send({"type": "idle"})
|
|
173
|
+
continue
|
|
174
|
+
if status == 409:
|
|
175
|
+
raise RuntimeError((value or {}).get("error", "session was superseded"))
|
|
176
|
+
if status == 410:
|
|
177
|
+
send({"type": "gone", "jobId": (value or {}).get("jobId"), "error": (value or {}).get("error", "job is gone")})
|
|
178
|
+
continue
|
|
179
|
+
if status >= 400 or not isinstance(value, dict):
|
|
180
|
+
raise RuntimeError(f"next job returned HTTP {status}: {value}")
|
|
181
|
+
send({"type": "job", "job": value})
|
|
182
|
+
result = receive()
|
|
183
|
+
if result.get("type") != "result" or not isinstance(result.get("result"), dict):
|
|
184
|
+
raise ValueError("expected a result message")
|
|
185
|
+
job_id = result["result"].get("jobId")
|
|
186
|
+
if job_id != value.get("jobId"):
|
|
187
|
+
raise ValueError("result does not match the assigned job")
|
|
188
|
+
deliver_result(runner_id, session_id, result["result"])
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def stop(_signum, _frame):
|
|
192
|
+
stop_requests()
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
if __name__ == "__main__":
|
|
196
|
+
signal.signal(signal.SIGTERM, stop)
|
|
197
|
+
signal.signal(signal.SIGINT, stop)
|
|
198
|
+
try:
|
|
199
|
+
main()
|
|
200
|
+
except EOFError:
|
|
201
|
+
pass
|
|
202
|
+
except Exception as error:
|
|
203
|
+
if not stopping.is_set():
|
|
204
|
+
diagnostic(str(error))
|
|
205
|
+
sys.exit(1)
|
package/docker/control/server.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
+
import hashlib
|
|
2
3
|
import http.client
|
|
3
|
-
import hmac
|
|
4
4
|
import json
|
|
5
5
|
import os
|
|
6
6
|
import re
|
|
@@ -17,12 +17,6 @@ DOCKER_SOCKET = os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock")
|
|
|
17
17
|
WORKSPACE_PATH = os.environ.get("WORKSPACE_PATH", "/workspace")
|
|
18
18
|
PROJECT_NAME = os.environ.get("COMPOSE_PROJECT_NAME", "").strip()
|
|
19
19
|
SERVICE_NAME = os.environ.get("NESTBOX_CONTROL_SERVICE", "control")
|
|
20
|
-
HOST_TOKEN_FILE = os.environ.get("NESTBOX_HOST_TOKEN_FILE", "/run/secrets/nestbox-host-token")
|
|
21
|
-
try:
|
|
22
|
-
with open(HOST_TOKEN_FILE, encoding="utf-8") as token_stream:
|
|
23
|
-
HOST_TOKEN = token_stream.read().strip()
|
|
24
|
-
except OSError:
|
|
25
|
-
HOST_TOKEN = ""
|
|
26
20
|
HOST_TIMEOUT = int(os.environ.get("NESTBOX_HOST_RUNNER_TIMEOUT", "3600"))
|
|
27
21
|
HOST_HEARTBEAT_TTL = int(os.environ.get("NESTBOX_HOST_RUNNER_HEARTBEAT_TTL", "10"))
|
|
28
22
|
HOST_LONG_POLL_TIMEOUT = int(os.environ.get("NESTBOX_HOST_LONG_POLL_TIMEOUT", "25"))
|
|
@@ -206,10 +200,16 @@ class HostBroker:
|
|
|
206
200
|
self.pending = []
|
|
207
201
|
self.jobs = {}
|
|
208
202
|
self.runner = {}
|
|
203
|
+
self.gone = {}
|
|
209
204
|
|
|
210
|
-
def
|
|
205
|
+
def _validate_ids(self, runner_id, session_id):
|
|
211
206
|
if not re.fullmatch(r"[A-Za-z0-9._:-]{8,128}", runner_id):
|
|
212
207
|
raise ValueError("invalid runner ID")
|
|
208
|
+
if not re.fullmatch(r"[A-Za-z0-9._:-]{8,128}", session_id):
|
|
209
|
+
raise ValueError("invalid session ID")
|
|
210
|
+
|
|
211
|
+
def connect(self, runner_id, session_id, metadata):
|
|
212
|
+
self._validate_ids(runner_id, session_id)
|
|
213
213
|
now = time.monotonic()
|
|
214
214
|
with self.condition:
|
|
215
215
|
active_id = self.runner.get("id")
|
|
@@ -228,13 +228,22 @@ class HostBroker:
|
|
|
228
228
|
}
|
|
229
229
|
self.runner = {
|
|
230
230
|
"id": runner_id,
|
|
231
|
+
"sessionId": session_id,
|
|
231
232
|
"seen": now,
|
|
232
|
-
"pid":
|
|
233
|
-
"platform":
|
|
234
|
-
"hostname":
|
|
233
|
+
"pid": metadata.get("pid"),
|
|
234
|
+
"platform": metadata.get("platform"),
|
|
235
|
+
"hostname": metadata.get("hostname"),
|
|
235
236
|
}
|
|
236
237
|
self.condition.notify_all()
|
|
237
238
|
|
|
239
|
+
def touch(self, runner_id, session_id):
|
|
240
|
+
self._validate_ids(runner_id, session_id)
|
|
241
|
+
with self.condition:
|
|
242
|
+
if self.runner.get("id") != runner_id or self.runner.get("sessionId") != session_id:
|
|
243
|
+
raise PermissionError("host runner session was superseded")
|
|
244
|
+
self.runner["seen"] = time.monotonic()
|
|
245
|
+
self.condition.notify_all()
|
|
246
|
+
|
|
238
247
|
def status(self):
|
|
239
248
|
with self.condition:
|
|
240
249
|
if not self.runner:
|
|
@@ -256,14 +265,16 @@ class HostBroker:
|
|
|
256
265
|
self.pending.append(job["jobId"])
|
|
257
266
|
self.condition.notify_all()
|
|
258
267
|
|
|
259
|
-
def next(self, runner_id,
|
|
268
|
+
def next(self, runner_id, session_id, timeout):
|
|
260
269
|
deadline = time.monotonic() + timeout
|
|
261
270
|
with self.condition:
|
|
262
271
|
while True:
|
|
263
|
-
if self.runner.get("id") != runner_id:
|
|
264
|
-
raise PermissionError("
|
|
265
|
-
|
|
266
|
-
|
|
272
|
+
if self.runner.get("id") != runner_id or self.runner.get("sessionId") != session_id:
|
|
273
|
+
raise PermissionError("host runner session was superseded")
|
|
274
|
+
self.runner["seen"] = time.monotonic()
|
|
275
|
+
for record in self.jobs.values():
|
|
276
|
+
if record.get("runnerId") == runner_id and record.get("result") is None:
|
|
277
|
+
return record["job"]
|
|
267
278
|
if self.pending:
|
|
268
279
|
identifier = self.pending.pop(0)
|
|
269
280
|
record = self.jobs.get(identifier)
|
|
@@ -275,13 +286,28 @@ class HostBroker:
|
|
|
275
286
|
return None
|
|
276
287
|
self.condition.wait(min(1, remaining))
|
|
277
288
|
|
|
278
|
-
def complete(self, runner_id, identifier, result):
|
|
289
|
+
def complete(self, runner_id, session_id, identifier, result):
|
|
279
290
|
with self.condition:
|
|
291
|
+
if self.runner.get("id") != runner_id or self.runner.get("sessionId") != session_id:
|
|
292
|
+
raise PermissionError("host runner session was superseded")
|
|
293
|
+
serialized = json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
|
|
294
|
+
fingerprint = hashlib.sha256(serialized).hexdigest()
|
|
280
295
|
record = self.jobs.get(identifier)
|
|
281
|
-
|
|
282
|
-
|
|
296
|
+
terminal = self.gone.get(identifier)
|
|
297
|
+
if not record and terminal:
|
|
298
|
+
if terminal.get("fingerprint") == fingerprint:
|
|
299
|
+
return True
|
|
300
|
+
if terminal.get("fingerprint"):
|
|
301
|
+
raise RuntimeError("conflicting host result replay")
|
|
302
|
+
raise FileNotFoundError(terminal["error"])
|
|
303
|
+
if not record:
|
|
304
|
+
raise FileNotFoundError("host job is no longer available")
|
|
305
|
+
if record.get("runnerId") != runner_id:
|
|
306
|
+
raise ValueError("host job belongs to another runner")
|
|
283
307
|
if record.get("result") is not None:
|
|
284
|
-
|
|
308
|
+
if record.get("fingerprint") == fingerprint:
|
|
309
|
+
return True
|
|
310
|
+
raise RuntimeError("conflicting host result replay")
|
|
285
311
|
if result.get("jobId") != identifier:
|
|
286
312
|
raise ValueError("host result contains a mismatched job ID")
|
|
287
313
|
job = record["job"]
|
|
@@ -298,6 +324,31 @@ class HostBroker:
|
|
|
298
324
|
if result.get("status") != "success" and not isinstance(result.get("exitCode"), int):
|
|
299
325
|
raise ValueError("invalid script confirmation result")
|
|
300
326
|
record["result"] = result
|
|
327
|
+
record["fingerprint"] = fingerprint
|
|
328
|
+
self.condition.notify_all()
|
|
329
|
+
return False
|
|
330
|
+
|
|
331
|
+
def _remember(self, identifier, record=None, error="host job is gone"):
|
|
332
|
+
self.gone[identifier] = {
|
|
333
|
+
"fingerprint": record.get("fingerprint") if record else None,
|
|
334
|
+
"error": error,
|
|
335
|
+
"time": time.monotonic(),
|
|
336
|
+
}
|
|
337
|
+
while len(self.gone) > MAX_HOST_JOBS:
|
|
338
|
+
del self.gone[next(iter(self.gone))]
|
|
339
|
+
|
|
340
|
+
def reject(self, runner_id, identifier, error):
|
|
341
|
+
with self.condition:
|
|
342
|
+
record = self.jobs.get(identifier)
|
|
343
|
+
if not record or record.get("runnerId") != runner_id or record.get("result") is not None:
|
|
344
|
+
return
|
|
345
|
+
record["result"] = {
|
|
346
|
+
"jobId": identifier,
|
|
347
|
+
"exitCode": 1,
|
|
348
|
+
"stdout": "",
|
|
349
|
+
"stderr": f"Host result was rejected: {error}",
|
|
350
|
+
"interrupted": True,
|
|
351
|
+
}
|
|
301
352
|
self.condition.notify_all()
|
|
302
353
|
|
|
303
354
|
def wait(self, identifier, timeout=HOST_TIMEOUT):
|
|
@@ -305,33 +356,31 @@ class HostBroker:
|
|
|
305
356
|
with self.condition:
|
|
306
357
|
while True:
|
|
307
358
|
record = self.jobs.get(identifier)
|
|
308
|
-
if record and record.get("
|
|
309
|
-
|
|
310
|
-
|
|
359
|
+
if record and record.get("result") is None:
|
|
360
|
+
age = time.monotonic() - self.runner.get("seen", 0) if self.runner else float("inf")
|
|
361
|
+
owner = record.get("runnerId")
|
|
362
|
+
lost_owner = bool(owner and self.runner.get("id") != owner)
|
|
363
|
+
if lost_owner or age > HOST_HEARTBEAT_TTL:
|
|
311
364
|
record["result"] = {
|
|
312
365
|
"jobId": identifier,
|
|
313
366
|
"exitCode": 1,
|
|
314
367
|
"stdout": "",
|
|
315
|
-
"stderr": "The assigned host runner disconnected before returning a result.",
|
|
368
|
+
"stderr": "The assigned host runner disconnected before returning a result." if owner else "The host runner disconnected before accepting the job.",
|
|
316
369
|
"interrupted": True,
|
|
317
370
|
}
|
|
318
|
-
elif record and record.get("result") is None and self.runner and time.monotonic() - self.runner.get("seen", 0) > HOST_HEARTBEAT_TTL:
|
|
319
|
-
record["result"] = {
|
|
320
|
-
"jobId": identifier,
|
|
321
|
-
"exitCode": 1,
|
|
322
|
-
"stdout": "",
|
|
323
|
-
"stderr": "The host runner disconnected before accepting the job.",
|
|
324
|
-
"interrupted": True,
|
|
325
|
-
}
|
|
326
371
|
if record and record.get("result") is not None:
|
|
327
372
|
result = record["result"]
|
|
328
373
|
del self.jobs[identifier]
|
|
374
|
+
if identifier in self.pending:
|
|
375
|
+
self.pending.remove(identifier)
|
|
376
|
+
self._remember(identifier, record)
|
|
329
377
|
return result
|
|
330
378
|
remaining = deadline - time.monotonic()
|
|
331
379
|
if remaining <= 0:
|
|
332
|
-
self.jobs.pop(identifier, None)
|
|
380
|
+
expired = self.jobs.pop(identifier, None)
|
|
333
381
|
if identifier in self.pending:
|
|
334
382
|
self.pending.remove(identifier)
|
|
383
|
+
self._remember(identifier, expired, f"host job expired after {timeout} seconds")
|
|
335
384
|
raise TimeoutError(f"host runner did not finish within {timeout} seconds")
|
|
336
385
|
self.condition.wait(min(1, remaining))
|
|
337
386
|
|
|
@@ -372,7 +421,7 @@ def valid_command(command):
|
|
|
372
421
|
|
|
373
422
|
|
|
374
423
|
class Handler(BaseHTTPRequestHandler):
|
|
375
|
-
server_version = "nestbox-control/0.
|
|
424
|
+
server_version = "nestbox-control/0.5"
|
|
376
425
|
|
|
377
426
|
def setup(self):
|
|
378
427
|
super().setup()
|
|
@@ -560,7 +609,7 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
560
609
|
|
|
561
610
|
|
|
562
611
|
class HostHandler(BaseHTTPRequestHandler):
|
|
563
|
-
server_version = "nestbox-host-control/0.
|
|
612
|
+
server_version = "nestbox-host-control/0.5"
|
|
564
613
|
|
|
565
614
|
def setup(self):
|
|
566
615
|
super().setup()
|
|
@@ -577,14 +626,12 @@ class HostHandler(BaseHTTPRequestHandler):
|
|
|
577
626
|
self.end_headers()
|
|
578
627
|
self.wfile.write(payload)
|
|
579
628
|
|
|
580
|
-
def authorized(self):
|
|
581
|
-
supplied = self.headers.get("Authorization", "")
|
|
582
|
-
expected = f"Bearer {HOST_TOKEN}"
|
|
583
|
-
return hmac.compare_digest(supplied.encode(), expected.encode())
|
|
584
|
-
|
|
585
629
|
def runner(self):
|
|
586
630
|
return self.headers.get("X-Nestbox-Runner-Id", "")
|
|
587
631
|
|
|
632
|
+
def session(self):
|
|
633
|
+
return self.headers.get("X-Nestbox-Session-Id", "")
|
|
634
|
+
|
|
588
635
|
def body(self):
|
|
589
636
|
if self.headers.get("Transfer-Encoding"):
|
|
590
637
|
raise ValueError("transfer encoding is not supported")
|
|
@@ -596,12 +643,9 @@ class HostHandler(BaseHTTPRequestHandler):
|
|
|
596
643
|
raise ValueError("request body must be a JSON object")
|
|
597
644
|
return value
|
|
598
645
|
|
|
599
|
-
def
|
|
600
|
-
if not self.authorized():
|
|
601
|
-
self.send_json(401, {"error": "invalid host runner token"})
|
|
602
|
-
return False
|
|
646
|
+
def active_session(self):
|
|
603
647
|
try:
|
|
604
|
-
HOST_BROKER.touch(self.runner(), self.
|
|
648
|
+
HOST_BROKER.touch(self.runner(), self.session())
|
|
605
649
|
return True
|
|
606
650
|
except PermissionError as error:
|
|
607
651
|
self.send_json(409, {"error": str(error)})
|
|
@@ -612,26 +656,36 @@ class HostHandler(BaseHTTPRequestHandler):
|
|
|
612
656
|
def do_GET(self):
|
|
613
657
|
parsed = urlparse(self.path)
|
|
614
658
|
if parsed.path == "/host/health":
|
|
615
|
-
if not self.
|
|
659
|
+
if not self.active_session(): return
|
|
616
660
|
self.send_json(200, {"ok": True, "hostRunner": host_status()})
|
|
617
661
|
return
|
|
618
662
|
if parsed.path == "/host/jobs/next":
|
|
619
|
-
if not self.authenticate(): return
|
|
620
663
|
try:
|
|
621
664
|
requested = int((parse_qs(parsed.query).get("timeout") or [HOST_LONG_POLL_TIMEOUT])[0])
|
|
622
665
|
timeout = max(1, min(HOST_LONG_POLL_TIMEOUT, requested))
|
|
623
|
-
job = HOST_BROKER.next(self.runner(), self.
|
|
666
|
+
job = HOST_BROKER.next(self.runner(), self.session(), timeout)
|
|
624
667
|
if job is None:
|
|
625
668
|
self.send_response(204); self.end_headers(); return
|
|
626
669
|
self.send_json(200, job)
|
|
670
|
+
except PermissionError as error:
|
|
671
|
+
self.send_json(409, {"error": str(error)})
|
|
627
672
|
except (TypeError, ValueError) as error:
|
|
628
673
|
self.send_json(400, {"error": str(error)})
|
|
629
674
|
return
|
|
630
675
|
self.send_json(404, {"error": "not found"})
|
|
631
676
|
|
|
632
677
|
def do_POST(self):
|
|
633
|
-
if not self.authenticate(): return
|
|
634
678
|
path = urlparse(self.path).path
|
|
679
|
+
if path == "/host/connect":
|
|
680
|
+
try:
|
|
681
|
+
HOST_BROKER.connect(self.runner(), self.session(), self.body())
|
|
682
|
+
self.send_json(200, {"ok": True})
|
|
683
|
+
except PermissionError as error:
|
|
684
|
+
self.send_json(409, {"error": str(error)})
|
|
685
|
+
except ValueError as error:
|
|
686
|
+
self.send_json(400, {"error": str(error)})
|
|
687
|
+
return
|
|
688
|
+
if not self.active_session(): return
|
|
635
689
|
if path == "/host/heartbeat":
|
|
636
690
|
self.send_json(200, {"ok": True})
|
|
637
691
|
return
|
|
@@ -640,18 +694,23 @@ class HostHandler(BaseHTTPRequestHandler):
|
|
|
640
694
|
self.send_json(404, {"error": "not found"}); return
|
|
641
695
|
try:
|
|
642
696
|
result = self.body()
|
|
643
|
-
HOST_BROKER.complete(self.runner(), match.group(1), result)
|
|
644
|
-
self.send_json(202, {"accepted": True})
|
|
697
|
+
duplicate = HOST_BROKER.complete(self.runner(), self.session(), match.group(1), result)
|
|
698
|
+
self.send_json(202, {"accepted": True, "duplicate": duplicate})
|
|
699
|
+
except FileNotFoundError as error:
|
|
700
|
+
self.send_json(410, {"error": str(error), "jobId": match.group(1)})
|
|
701
|
+
except PermissionError as error:
|
|
702
|
+
self.send_json(409, {"error": str(error)})
|
|
703
|
+
except RuntimeError as error:
|
|
704
|
+
self.send_json(409, {"error": str(error)})
|
|
645
705
|
except ValueError as error:
|
|
706
|
+
HOST_BROKER.reject(self.runner(), match.group(1), str(error))
|
|
646
707
|
self.send_json(400, {"error": str(error)})
|
|
647
708
|
|
|
648
709
|
|
|
649
710
|
def main():
|
|
650
711
|
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", PROJECT_NAME):
|
|
651
712
|
raise SystemExit("COMPOSE_PROJECT_NAME must be a non-empty lowercase project slug")
|
|
652
|
-
|
|
653
|
-
raise SystemExit("NESTBOX_HOST_TOKEN_FILE must contain a token of at least 32 characters")
|
|
654
|
-
host_server = ThreadingHTTPServer(("0.0.0.0", 4089), HostHandler)
|
|
713
|
+
host_server = ThreadingHTTPServer(("127.0.0.1", 4089), HostHandler)
|
|
655
714
|
threading.Thread(target=host_server.serve_forever, daemon=True).start()
|
|
656
715
|
ThreadingHTTPServer(("0.0.0.0", int(os.environ.get("NESTBOX_CONTROL_PORT", "4088"))), Handler).serve_forever()
|
|
657
716
|
|
package/docker-compose.yaml
CHANGED
|
@@ -114,8 +114,6 @@ services:
|
|
|
114
114
|
dockerfile: ./docker/control/Dockerfile
|
|
115
115
|
expose:
|
|
116
116
|
- "4088"
|
|
117
|
-
# Authenticated host-runner transport, proxied only through Nginx.
|
|
118
|
-
- "4089"
|
|
119
117
|
volumes:
|
|
120
118
|
- ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock
|
|
121
119
|
- type: bind
|
|
@@ -124,16 +122,9 @@ services:
|
|
|
124
122
|
read_only: true
|
|
125
123
|
bind:
|
|
126
124
|
create_host_path: false
|
|
127
|
-
- type: bind
|
|
128
|
-
source: ${NESTBOX_HOST_TOKEN_FILE:?NESTBOX_HOST_TOKEN_FILE must be configured}
|
|
129
|
-
target: /run/secrets/nestbox-host-token
|
|
130
|
-
read_only: true
|
|
131
|
-
bind:
|
|
132
|
-
create_host_path: false
|
|
133
125
|
environment:
|
|
134
126
|
- COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}
|
|
135
127
|
- NESTBOX_CONTROL_SERVICE=control
|
|
136
|
-
- NESTBOX_HOST_TOKEN_FILE=/run/secrets/nestbox-host-token
|
|
137
128
|
- NESTBOX_CONTROL_LOG_OUTPUT=${NESTBOX_CONTROL_LOG_OUTPUT:-text}
|
|
138
129
|
- NESTBOX_CONTROL_LOG_MAX_BYTES=${NESTBOX_CONTROL_LOG_MAX_BYTES:-262144}
|
|
139
130
|
healthcheck:
|