@molecule/api-code-sandbox-docker 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -2
- package/dist/archive.d.ts +49 -0
- package/dist/archive.d.ts.map +1 -0
- package/dist/archive.js +61 -0
- package/dist/capacity.d.ts +71 -0
- package/dist/capacity.d.ts.map +1 -0
- package/dist/capacity.js +98 -0
- package/dist/egress.d.ts +49 -0
- package/dist/egress.d.ts.map +1 -0
- package/dist/egress.js +424 -0
- package/dist/hibernate.d.ts +79 -0
- package/dist/hibernate.d.ts.map +1 -0
- package/dist/hibernate.js +142 -0
- package/dist/index.d.ts +44 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +44 -1
- package/dist/inspect.d.ts +131 -0
- package/dist/inspect.d.ts.map +1 -0
- package/dist/inspect.js +212 -0
- package/dist/provider.d.ts +14 -0
- package/dist/provider.d.ts.map +1 -1
- package/dist/provider.js +496 -19
- package/dist/request.d.ts +66 -0
- package/dist/request.d.ts.map +1 -0
- package/dist/request.js +56 -0
- package/dist/templates.d.ts +109 -0
- package/dist/templates.d.ts.map +1 -0
- package/dist/templates.js +381 -0
- package/dist/types.d.ts +29 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -3
package/dist/egress.js
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Observe whether a Docker host actually denies sandbox egress.
|
|
3
|
+
*
|
|
4
|
+
* TWO LEGS, AND BOTH ARE REQUIRED. This is the part that is easy to get wrong and
|
|
5
|
+
* was in fact got wrong: an earlier version of this check implemented only leg 1
|
|
6
|
+
* and therefore reported `filtered` on a host where sandboxes could still reach
|
|
7
|
+
* SSH, Redis, the monitoring stack, the MinIO admin console and the API itself.
|
|
8
|
+
*
|
|
9
|
+
* - **Leg 1 — DOCKER-USER (outbound to the internet).** A container on the sandbox
|
|
10
|
+
* network tries to open raw TCP to literal public IPs. Traffic leaving the
|
|
11
|
+
* bridge for the outside world traverses the `DOCKER-USER` chain.
|
|
12
|
+
* - **Leg 2 — INPUT (host-bound).** Packets a sandbox sends to the HOST GATEWAY
|
|
13
|
+
* are delivered locally and **never traverse `DOCKER-USER`** — they hit the
|
|
14
|
+
* `INPUT` chain instead. The two diverge in practice: a `ufw reload` rebuilds
|
|
15
|
+
* INPUT while leaving DOCKER-USER intact, so leg 1 alone still passes while
|
|
16
|
+
* every host-bound service is exposed.
|
|
17
|
+
*
|
|
18
|
+
* Leg 2's difficulty is that a firewall REJECT and "nothing is listening" are the
|
|
19
|
+
* same `ECONNREFUSED` to a client, so dialling an arbitrary closed port proves
|
|
20
|
+
* nothing. It therefore opens a REAL listener in the host network namespace on an
|
|
21
|
+
* ephemeral, deliberately non-allowlisted port and has a sandbox-network peer dial
|
|
22
|
+
* it. Both ends are ours, so the answer is unambiguous.
|
|
23
|
+
*
|
|
24
|
+
* Every failure mode collapses to `inconclusive` — never `filtered`. "I could not
|
|
25
|
+
* look" must never be reported as "I looked and it is safe"; that conflation is
|
|
26
|
+
* the original failure this whole mechanism exists to prevent.
|
|
27
|
+
*
|
|
28
|
+
* @module
|
|
29
|
+
*/
|
|
30
|
+
import net from 'net';
|
|
31
|
+
/** Exit codes the probe programs use to report what they observed. */
|
|
32
|
+
const EXIT_BLOCKED = 0;
|
|
33
|
+
const EXIT_OPEN = 9;
|
|
34
|
+
const EXIT_LISTENER_ACCEPTED = 10;
|
|
35
|
+
const EXIT_LISTENER_IDLE = 11;
|
|
36
|
+
const LISTENER_READY_MARKER = 'MOL_LISTENER_READY';
|
|
37
|
+
const PROBE_NAME_PREFIX = 'mol-egress-probe-';
|
|
38
|
+
/** Ephemeral range the host-port leg binds in. */
|
|
39
|
+
const HOST_PROBE_PORT_MIN = 41_000;
|
|
40
|
+
const HOST_PROBE_PORT_MAX = 45_000;
|
|
41
|
+
const REMEDIATION = 'Apply the host default-deny for the sandbox subnet (scripts/provision-egress-firewall.sh), then re-check.';
|
|
42
|
+
/**
|
|
43
|
+
* Parse an integer env var, tolerating garbage.
|
|
44
|
+
*
|
|
45
|
+
* `Math.max(min, parseInt(x))` returns NaN when x is malformed — NaN loses every
|
|
46
|
+
* comparison, so a typo silently produced a NaN timeout rather than the default.
|
|
47
|
+
*
|
|
48
|
+
* @param raw - Raw env value.
|
|
49
|
+
* @param fallback - Value to use when absent or unparseable.
|
|
50
|
+
* @param min - Lower clamp.
|
|
51
|
+
* @param max - Upper clamp.
|
|
52
|
+
* @returns A usable integer.
|
|
53
|
+
*/
|
|
54
|
+
function intEnv(raw, fallback, min, max) {
|
|
55
|
+
const parsed = Number.parseInt(raw ?? '', 10);
|
|
56
|
+
const value = Number.isFinite(parsed) ? parsed : fallback;
|
|
57
|
+
return Math.min(max, Math.max(min, value));
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Literal `ip:port` destinations for leg 1.
|
|
61
|
+
*
|
|
62
|
+
* TWO by default, not one: a blocked verdict is inferred from failure, so a single
|
|
63
|
+
* unreachable target is indistinguishable from a working firewall. Requiring every
|
|
64
|
+
* target to fail makes that coincidence need two independent outages.
|
|
65
|
+
*
|
|
66
|
+
* Hostnames are REJECTED rather than resolved — a hostname would make DNS the
|
|
67
|
+
* thing under test, and a blocked resolver would read as `filtered` on a wide-open
|
|
68
|
+
* host.
|
|
69
|
+
*
|
|
70
|
+
* @returns Validated targets.
|
|
71
|
+
*/
|
|
72
|
+
function probeTargets() {
|
|
73
|
+
const raw = process.env.SANDBOX_EGRESS_PROBE_TARGETS ?? '1.1.1.1:443,8.8.8.8:443';
|
|
74
|
+
const targets = [];
|
|
75
|
+
for (const entry of raw.split(',')) {
|
|
76
|
+
const match = /^\s*(\d{1,3}(?:\.\d{1,3}){3}):(\d{1,5})\s*$/.exec(entry);
|
|
77
|
+
if (!match)
|
|
78
|
+
continue;
|
|
79
|
+
const port = Number(match[2]);
|
|
80
|
+
if (net.isIPv4(match[1]) && port > 0 && port < 65_536)
|
|
81
|
+
targets.push({ host: match[1], port });
|
|
82
|
+
}
|
|
83
|
+
return targets;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Host-gateway ports the firewall intentionally allows, mirroring
|
|
87
|
+
* `provision-egress-firewall.sh`. Read from the same env vars that script reads,
|
|
88
|
+
* so an operator who moves a port does not silently turn leg 2 into a false pass.
|
|
89
|
+
*
|
|
90
|
+
* @returns Ports a sandbox is expected to reach on the host gateway.
|
|
91
|
+
*/
|
|
92
|
+
function allowedHostPorts() {
|
|
93
|
+
const raw = process.env.SANDBOX_ALLOWED_HOST_PORTS ??
|
|
94
|
+
[
|
|
95
|
+
process.env.SANDBOX_EGRESS_PROXY_PORT ?? '3129',
|
|
96
|
+
process.env.SANDBOX_DB_PORT ?? '5432',
|
|
97
|
+
process.env.MINIO_PORT ?? '9000',
|
|
98
|
+
].join(',');
|
|
99
|
+
const ports = new Set();
|
|
100
|
+
for (const entry of raw.split(',')) {
|
|
101
|
+
const port = Number(entry.trim());
|
|
102
|
+
if (Number.isInteger(port) && port > 0 && port < 65_536)
|
|
103
|
+
ports.add(port);
|
|
104
|
+
}
|
|
105
|
+
return ports;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* The client program: raw `net.connect` to each target.
|
|
109
|
+
*
|
|
110
|
+
* Raw sockets, never `fetch` — proxy env is blanked AND a raw connect ignores it
|
|
111
|
+
* regardless, so this exercises the DIRECT path the proxy does not cover and only
|
|
112
|
+
* the firewall stops.
|
|
113
|
+
*
|
|
114
|
+
* @param targets - Destinations to attempt.
|
|
115
|
+
* @param timeoutMs - Per-connection timeout.
|
|
116
|
+
* @returns Program source.
|
|
117
|
+
*/
|
|
118
|
+
export function buildProbeScript(targets, timeoutMs) {
|
|
119
|
+
return (`const net=require('net');const ts=${JSON.stringify(targets)};let open=0,left=ts.length;` +
|
|
120
|
+
`const done=()=>{if(--left===0)process.exit(open>0?${EXIT_OPEN}:${EXIT_BLOCKED})};` +
|
|
121
|
+
`for(const t of ts){const s=net.connect({host:t.host,port:t.port});` +
|
|
122
|
+
`s.setTimeout(${timeoutMs});` +
|
|
123
|
+
`s.on('connect',()=>{open++;s.destroy();done()});` +
|
|
124
|
+
`s.on('timeout',()=>{s.destroy();done()});` +
|
|
125
|
+
`s.on('error',()=>done())}`);
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* The host-netns listener program for leg 2.
|
|
129
|
+
*
|
|
130
|
+
* @param port - Port to bind.
|
|
131
|
+
* @param timeoutMs - How long to wait for a connection.
|
|
132
|
+
* @returns Program source.
|
|
133
|
+
*/
|
|
134
|
+
export function buildListenerScript(port, timeoutMs) {
|
|
135
|
+
return [
|
|
136
|
+
"const net=require('net');",
|
|
137
|
+
'let settled=false;',
|
|
138
|
+
'const exit=(c)=>{if(settled)return;settled=true;process.exit(c);};',
|
|
139
|
+
`const s=net.createServer((c)=>{c.destroy();exit(${EXIT_LISTENER_ACCEPTED});});`,
|
|
140
|
+
"s.on('error',()=>exit(40));",
|
|
141
|
+
`s.listen(${port},'0.0.0.0',()=>{console.log('${LISTENER_READY_MARKER}');});`,
|
|
142
|
+
`setTimeout(()=>exit(${EXIT_LISTENER_IDLE}),${timeoutMs});`,
|
|
143
|
+
].join('');
|
|
144
|
+
}
|
|
145
|
+
/** Sleep, for the readiness poll. */
|
|
146
|
+
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
147
|
+
/**
|
|
148
|
+
* Create and start a hardened throwaway probe container.
|
|
149
|
+
*
|
|
150
|
+
* The name is PRE-CHOSEN, not read back from the create response: a create that
|
|
151
|
+
* times out client-side may still have landed daemon-side, and without a known
|
|
152
|
+
* name that container is unreachable and leaks.
|
|
153
|
+
*
|
|
154
|
+
* @param ctx - Probe context.
|
|
155
|
+
* @param name - Pre-chosen container name.
|
|
156
|
+
* @param script - Program to run.
|
|
157
|
+
* @param hostConfig - Extra HostConfig (network mode).
|
|
158
|
+
* @returns The container id, or a failure reason.
|
|
159
|
+
*/
|
|
160
|
+
async function startProbeContainer(ctx, name, script, hostConfig) {
|
|
161
|
+
try {
|
|
162
|
+
const created = (await ctx.request(`/containers/create?name=${name}`, 'POST', {
|
|
163
|
+
Image: ctx.baseImage,
|
|
164
|
+
Cmd: ['node', '-e', script],
|
|
165
|
+
// Blank the proxy env the sandbox image bakes in, plus NODE_OPTIONS —
|
|
166
|
+
// its --max-old-space-size fights the tiny memory cap below.
|
|
167
|
+
Env: [
|
|
168
|
+
'HTTP_PROXY=',
|
|
169
|
+
'HTTPS_PROXY=',
|
|
170
|
+
'http_proxy=',
|
|
171
|
+
'https_proxy=',
|
|
172
|
+
'NO_PROXY=',
|
|
173
|
+
'no_proxy=',
|
|
174
|
+
'NODE_USE_ENV_PROXY=',
|
|
175
|
+
'NODE_OPTIONS=',
|
|
176
|
+
],
|
|
177
|
+
Labels: { [`${ctx.labelPrefix}.egress-probe`]: 'true' },
|
|
178
|
+
// Raw (non-multiplexed) logs, so the readiness marker reads back without
|
|
179
|
+
// decoding Docker's stream framing.
|
|
180
|
+
Tty: true,
|
|
181
|
+
HostConfig: {
|
|
182
|
+
// AutoRemove OFF deliberately: it races /wait, and the exit code matters
|
|
183
|
+
// more than the convenience. Callers remove unconditionally.
|
|
184
|
+
AutoRemove: false,
|
|
185
|
+
RestartPolicy: { Name: '' },
|
|
186
|
+
Memory: 128 * 1024 * 1024,
|
|
187
|
+
MemorySwap: 128 * 1024 * 1024,
|
|
188
|
+
NanoCPUs: 250_000_000,
|
|
189
|
+
PidsLimit: 32,
|
|
190
|
+
CapDrop: ['ALL'],
|
|
191
|
+
SecurityOpt: ['no-new-privileges'],
|
|
192
|
+
ReadonlyRootfs: true,
|
|
193
|
+
Tmpfs: { '/tmp': 'rw,noexec,nosuid,size=8m' },
|
|
194
|
+
...hostConfig,
|
|
195
|
+
},
|
|
196
|
+
}));
|
|
197
|
+
await ctx.request(`/containers/${created.Id}/start`, 'POST');
|
|
198
|
+
return { id: created.Id };
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
return { failure: error instanceof Error ? error.message : String(error) };
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
/** Remove a probe container by id or pre-chosen name; never throws. */
|
|
205
|
+
async function removeProbeContainer(ctx, ref) {
|
|
206
|
+
await ctx.request(`/containers/${ref}?force=true`, 'DELETE').catch((error) => {
|
|
207
|
+
ctx.warn?.('Failed to remove egress probe container', { ref, error });
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
/** Wait for a container to exit and return its status code. */
|
|
211
|
+
async function waitForExit(ctx, id, timeoutMs) {
|
|
212
|
+
try {
|
|
213
|
+
const waited = (await ctx.request(`/containers/${id}/wait`, 'POST', undefined, timeoutMs));
|
|
214
|
+
return typeof waited?.StatusCode === 'number' ? waited.StatusCode : null;
|
|
215
|
+
}
|
|
216
|
+
catch (_error) {
|
|
217
|
+
return null;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Resolve the host-gateway IP, preferring the daemon's own answer and falling
|
|
222
|
+
* back to the `bridge` network's gateway — `docker info` omits `HostGatewayIP`
|
|
223
|
+
* on some daemon versions.
|
|
224
|
+
*
|
|
225
|
+
* @param ctx - Probe context.
|
|
226
|
+
* @returns The gateway IP, or null when undiscoverable.
|
|
227
|
+
*/
|
|
228
|
+
async function resolveHostGatewayIp(ctx) {
|
|
229
|
+
try {
|
|
230
|
+
const info = (await ctx.request('/info'));
|
|
231
|
+
if (info?.HostGatewayIP && net.isIP(info.HostGatewayIP))
|
|
232
|
+
return info.HostGatewayIP;
|
|
233
|
+
}
|
|
234
|
+
catch (_error) {
|
|
235
|
+
// Fall through to the bridge lookup.
|
|
236
|
+
}
|
|
237
|
+
try {
|
|
238
|
+
const bridge = (await ctx.request('/networks/bridge'));
|
|
239
|
+
for (const entry of bridge?.IPAM?.Config ?? []) {
|
|
240
|
+
if (entry.Gateway && net.isIP(entry.Gateway))
|
|
241
|
+
return entry.Gateway;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
catch (_error) {
|
|
245
|
+
// Reported as inconclusive by the caller.
|
|
246
|
+
}
|
|
247
|
+
return null;
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Leg 2 — prove the INPUT chain denies sandbox→host traffic.
|
|
251
|
+
*
|
|
252
|
+
* @param ctx - Probe context.
|
|
253
|
+
* @param timeoutMs - Per-connection timeout.
|
|
254
|
+
* @returns What was observed on the locally-delivered path.
|
|
255
|
+
*/
|
|
256
|
+
async function probeHostPort(ctx, timeoutMs) {
|
|
257
|
+
const gateway = await resolveHostGatewayIp(ctx);
|
|
258
|
+
if (!gateway) {
|
|
259
|
+
return {
|
|
260
|
+
result: 'inconclusive',
|
|
261
|
+
detail: 'could not determine the host-gateway IP, so the INPUT-chain deny (host-bound ports: ' +
|
|
262
|
+
`SSH, Redis, monitoring, the MinIO console, this API) could not be verified. ${REMEDIATION}`,
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
const allowed = allowedHostPorts();
|
|
266
|
+
let port = 0;
|
|
267
|
+
for (let attempt = 0; attempt < 8 && port === 0; attempt++) {
|
|
268
|
+
const candidate = HOST_PROBE_PORT_MIN + Math.floor(Math.random() * (HOST_PROBE_PORT_MAX - HOST_PROBE_PORT_MIN));
|
|
269
|
+
if (!allowed.has(candidate))
|
|
270
|
+
port = candidate;
|
|
271
|
+
}
|
|
272
|
+
if (port === 0) {
|
|
273
|
+
return {
|
|
274
|
+
result: 'inconclusive',
|
|
275
|
+
detail: 'every candidate port collided with SANDBOX_ALLOWED_HOST_PORTS',
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
const listenerName = `${PROBE_NAME_PREFIX}l-${Math.random().toString(36).slice(2, 12)}`;
|
|
279
|
+
const clientName = `${PROBE_NAME_PREFIX}c-${Math.random().toString(36).slice(2, 12)}`;
|
|
280
|
+
try {
|
|
281
|
+
// The listener must live in the HOST network namespace — the only way to bind
|
|
282
|
+
// a port a sandbox reaches via the gateway, container or not.
|
|
283
|
+
const listener = await startProbeContainer(ctx, listenerName, buildListenerScript(port, timeoutMs + 5_000), { NetworkMode: 'host' });
|
|
284
|
+
if ('failure' in listener) {
|
|
285
|
+
return {
|
|
286
|
+
result: 'inconclusive',
|
|
287
|
+
detail: `host listener could not start: ${listener.failure} (INPUT-chain deny unverified)`,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
// Wait for the bind. Without this a fast client races the listener and
|
|
291
|
+
// reports a false `blocked`.
|
|
292
|
+
const readyBudget = intEnv(process.env.SANDBOX_EGRESS_LISTENER_READY_MS, 5_000, 100, 30_000);
|
|
293
|
+
let ready = false;
|
|
294
|
+
for (let waited = 0; waited < readyBudget && !ready; waited += 100) {
|
|
295
|
+
const logs = await ctx
|
|
296
|
+
.request(`/containers/${listener.id}/logs?stdout=1&stderr=1`)
|
|
297
|
+
.catch(() => '');
|
|
298
|
+
if (String(logs).includes(LISTENER_READY_MARKER)) {
|
|
299
|
+
ready = true;
|
|
300
|
+
break;
|
|
301
|
+
}
|
|
302
|
+
await delay(100);
|
|
303
|
+
}
|
|
304
|
+
if (!ready) {
|
|
305
|
+
return {
|
|
306
|
+
result: 'inconclusive',
|
|
307
|
+
detail: `listener on ${gateway}:${port} never reported ready (INPUT-chain deny unverified)`,
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
const client = await startProbeContainer(ctx, clientName, buildProbeScript([{ host: gateway, port }], timeoutMs), { NetworkMode: ctx.network });
|
|
311
|
+
if ('failure' in client) {
|
|
312
|
+
return {
|
|
313
|
+
result: 'inconclusive',
|
|
314
|
+
detail: `host-port client could not start: ${client.failure} (INPUT-chain deny unverified)`,
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
const code = await waitForExit(ctx, client.id, timeoutMs + 10_000);
|
|
318
|
+
if (code === EXIT_OPEN) {
|
|
319
|
+
return {
|
|
320
|
+
result: 'reachable',
|
|
321
|
+
detail: `a container on "${ctx.network}" opened TCP to the host at ${gateway}:${port}, which is ` +
|
|
322
|
+
'NOT an allowed host port. The INPUT-chain deny is not in effect, so sandboxes can ' +
|
|
323
|
+
`reach host-bound services (SSH, Redis, monitoring, the MinIO console, this API). ${REMEDIATION}`,
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
if (code === EXIT_BLOCKED) {
|
|
327
|
+
return { result: 'blocked', detail: `host ${gateway}:${port} refused (INPUT deny verified)` };
|
|
328
|
+
}
|
|
329
|
+
return {
|
|
330
|
+
result: 'inconclusive',
|
|
331
|
+
detail: `host-port probe exited ${String(code)} (INPUT-chain deny unverified)`,
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
catch (error) {
|
|
335
|
+
return {
|
|
336
|
+
result: 'inconclusive',
|
|
337
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
finally {
|
|
341
|
+
await removeProbeContainer(ctx, listenerName);
|
|
342
|
+
await removeProbeContainer(ctx, clientName);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* Run BOTH legs and combine them into one verdict.
|
|
347
|
+
*
|
|
348
|
+
* A `filtered` verdict requires both legs to be denied. Either leg observing a
|
|
349
|
+
* successful connection is conclusive `open`; anything that could not be observed
|
|
350
|
+
* is `inconclusive`.
|
|
351
|
+
*
|
|
352
|
+
* @param ctx - Probe context.
|
|
353
|
+
* @returns The combined verdict.
|
|
354
|
+
*/
|
|
355
|
+
export async function verifyDockerEgress(ctx) {
|
|
356
|
+
const targets = probeTargets();
|
|
357
|
+
if (targets.length === 0) {
|
|
358
|
+
return {
|
|
359
|
+
state: 'inconclusive',
|
|
360
|
+
detail: 'No valid literal ip:port probe targets configured.',
|
|
361
|
+
remediation: 'Set SANDBOX_EGRESS_PROBE_TARGETS to comma-separated ip:port values.',
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
const timeoutMs = intEnv(process.env.SANDBOX_EGRESS_PROBE_TIMEOUT_MS, 3_000, 500, 15_000);
|
|
365
|
+
// ── Leg 1: DOCKER-USER / outbound ──────────────────────────────────────────
|
|
366
|
+
const outboundName = `${PROBE_NAME_PREFIX}o-${Math.random().toString(36).slice(2, 12)}`;
|
|
367
|
+
let outbound;
|
|
368
|
+
try {
|
|
369
|
+
const started = await startProbeContainer(ctx, outboundName, buildProbeScript(targets, timeoutMs), { NetworkMode: ctx.network });
|
|
370
|
+
if ('failure' in started) {
|
|
371
|
+
outbound = {
|
|
372
|
+
result: 'inconclusive',
|
|
373
|
+
detail: `outbound probe could not run: ${started.failure}`,
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
else {
|
|
377
|
+
const code = await waitForExit(ctx, started.id, timeoutMs * targets.length + 10_000);
|
|
378
|
+
outbound =
|
|
379
|
+
code === EXIT_OPEN
|
|
380
|
+
? {
|
|
381
|
+
result: 'reachable',
|
|
382
|
+
detail: `a container on "${ctx.network}" reached ${targets
|
|
383
|
+
.map((t) => `${t.host}:${t.port}`)
|
|
384
|
+
.join(' or ')} directly — outbound egress is NOT filtered. ${REMEDIATION}`,
|
|
385
|
+
}
|
|
386
|
+
: code === EXIT_BLOCKED
|
|
387
|
+
? {
|
|
388
|
+
result: 'blocked',
|
|
389
|
+
detail: `raw connects from "${ctx.network}" to ${targets.length} public target(s) were refused or timed out`,
|
|
390
|
+
}
|
|
391
|
+
: {
|
|
392
|
+
result: 'inconclusive',
|
|
393
|
+
detail: `outbound probe exited ${String(code)} — it did not complete its attempts`,
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
finally {
|
|
398
|
+
await removeProbeContainer(ctx, outboundName);
|
|
399
|
+
}
|
|
400
|
+
if (outbound.result === 'reachable') {
|
|
401
|
+
return { state: 'open', detail: outbound.detail, remediation: REMEDIATION };
|
|
402
|
+
}
|
|
403
|
+
// ── Leg 2: INPUT / host-bound ──────────────────────────────────────────────
|
|
404
|
+
const hostBound = await probeHostPort(ctx, timeoutMs);
|
|
405
|
+
if (hostBound.result === 'reachable') {
|
|
406
|
+
return { state: 'open', detail: hostBound.detail, remediation: REMEDIATION };
|
|
407
|
+
}
|
|
408
|
+
// A pass needs BOTH legs denied. One inconclusive leg makes the whole verdict
|
|
409
|
+
// inconclusive — half a proof is not a proof.
|
|
410
|
+
if (outbound.result === 'inconclusive' || hostBound.result === 'inconclusive') {
|
|
411
|
+
const parts = [outbound, hostBound]
|
|
412
|
+
.filter((leg) => leg.result === 'inconclusive')
|
|
413
|
+
.map((leg) => leg.detail);
|
|
414
|
+
return {
|
|
415
|
+
state: 'inconclusive',
|
|
416
|
+
detail: `egress could not be fully verified: ${parts.join('; ')}`,
|
|
417
|
+
remediation: `Ensure the Docker daemon is reachable and the base image ("${ctx.baseImage}") is present. ${REMEDIATION}`,
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
return {
|
|
421
|
+
state: 'filtered',
|
|
422
|
+
detail: `both legs denied — ${outbound.detail}; ${hostBound.detail}`,
|
|
423
|
+
};
|
|
424
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Suspend and resume a Docker sandbox, and report which mechanism actually ran.
|
|
3
|
+
*
|
|
4
|
+
* Docker has two, and they are not interchangeable. A CRIU **checkpoint** freezes
|
|
5
|
+
* the whole process tree to disk and restores it; a **stop** does not, and a
|
|
6
|
+
* sandbox that comes back from one is running with nothing in it. Which one is
|
|
7
|
+
* available is a property of the HOST — CRIU needs the daemon's experimental
|
|
8
|
+
* flag and the `criu` package — so the same code path produces different
|
|
9
|
+
* fidelity on different machines, and the caller cannot know which it got.
|
|
10
|
+
*
|
|
11
|
+
* That is why both functions return a {@link HibernationOutcome} instead of
|
|
12
|
+
* `void`. molecule.dev's sandboxes run their dev servers as detached exec
|
|
13
|
+
* processes; after a stop-style wake the container is "running", the preview
|
|
14
|
+
* answers nothing, and `status` says everything is fine. The relaunch that fixes
|
|
15
|
+
* it is only expressible if the resume reports what it did.
|
|
16
|
+
*
|
|
17
|
+
* ## The parameter that is a query parameter
|
|
18
|
+
*
|
|
19
|
+
* Restoring from a checkpoint is `POST /containers/{id}/start?checkpoint=<name>`.
|
|
20
|
+
* The checkpoint name is a QUERY parameter. Passing it in a JSON body — which is
|
|
21
|
+
* an easy and silent mistake, since the daemon accepts the request and starts the
|
|
22
|
+
* container — performs an ordinary cold start while looking exactly like a
|
|
23
|
+
* successful restore.
|
|
24
|
+
*
|
|
25
|
+
* @module
|
|
26
|
+
*/
|
|
27
|
+
import type { HibernationOutcome } from '@molecule/api-code-sandbox';
|
|
28
|
+
import type { DockerRequest } from './request.js';
|
|
29
|
+
/** What the hibernation capability needs from the provider. */
|
|
30
|
+
export interface HibernateContext {
|
|
31
|
+
request: DockerRequest;
|
|
32
|
+
/**
|
|
33
|
+
* Whether this host is believed to support CRIU checkpoints. Owned by the
|
|
34
|
+
* provider and flipped to `false` the first time the daemon says otherwise, so
|
|
35
|
+
* a host without CRIU pays one failed request per process rather than one per
|
|
36
|
+
* hibernation. Mutable state on purpose: the alternative is module-level state
|
|
37
|
+
* that tests cannot reset and two providers cannot hold separately.
|
|
38
|
+
*/
|
|
39
|
+
checkpoints: {
|
|
40
|
+
supported: boolean;
|
|
41
|
+
};
|
|
42
|
+
warn?: (message: string, meta?: Record<string, unknown>) => void;
|
|
43
|
+
info?: (message: string, meta?: Record<string, unknown>) => void;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The checkpoint name for a container. Derived, not passed in: it is an
|
|
47
|
+
* implementation detail of this bond's own suspend/resume pair, and a caller that
|
|
48
|
+
* supplied it could only get it wrong.
|
|
49
|
+
*
|
|
50
|
+
* @param containerId - The container id.
|
|
51
|
+
* @returns A stable checkpoint name.
|
|
52
|
+
*/
|
|
53
|
+
export declare function checkpointName(containerId: string): string;
|
|
54
|
+
/**
|
|
55
|
+
* Suspend a sandbox, preferring a checkpoint and falling back to a stop.
|
|
56
|
+
*
|
|
57
|
+
* Never fails just because the high-fidelity path is unavailable: a sandbox that
|
|
58
|
+
* cannot be checkpointed must still be suspendable. The outcome reports which
|
|
59
|
+
* one ran.
|
|
60
|
+
*
|
|
61
|
+
* @param ctx - Hibernation context.
|
|
62
|
+
* @param containerId - The sandbox to suspend.
|
|
63
|
+
* @returns What the suspend actually did.
|
|
64
|
+
*/
|
|
65
|
+
export declare function hibernateContainer(ctx: HibernateContext, containerId: string): Promise<HibernationOutcome>;
|
|
66
|
+
/**
|
|
67
|
+
* Resume a suspended sandbox, preferring a checkpoint restore.
|
|
68
|
+
*
|
|
69
|
+
* A missing checkpoint is the ROUTINE case — it is what a stop-suspended sandbox
|
|
70
|
+
* looks like — so it falls through quietly. Any other restore failure is logged,
|
|
71
|
+
* because it means every wake on this host is silently taking the slow, empty
|
|
72
|
+
* path.
|
|
73
|
+
*
|
|
74
|
+
* @param ctx - Hibernation context.
|
|
75
|
+
* @param containerId - The sandbox to resume.
|
|
76
|
+
* @returns What the resume actually did.
|
|
77
|
+
*/
|
|
78
|
+
export declare function resumeContainer(ctx: HibernateContext, containerId: string): Promise<HibernationOutcome>;
|
|
79
|
+
//# sourceMappingURL=hibernate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hibernate.d.ts","sourceRoot":"","sources":["../src/hibernate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAEpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAEjD,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,aAAa,CAAA;IACtB;;;;;;OAMG;IACH,WAAW,EAAE;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAA;IACnC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAChE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;CACjE;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAE1D;AAcD;;;;;;;;;;GAUG;AACH,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,gBAAgB,EACrB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,kBAAkB,CAAC,CAiC7B;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,gBAAgB,EACrB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,kBAAkB,CAAC,CAgD7B"}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Suspend and resume a Docker sandbox, and report which mechanism actually ran.
|
|
3
|
+
*
|
|
4
|
+
* Docker has two, and they are not interchangeable. A CRIU **checkpoint** freezes
|
|
5
|
+
* the whole process tree to disk and restores it; a **stop** does not, and a
|
|
6
|
+
* sandbox that comes back from one is running with nothing in it. Which one is
|
|
7
|
+
* available is a property of the HOST — CRIU needs the daemon's experimental
|
|
8
|
+
* flag and the `criu` package — so the same code path produces different
|
|
9
|
+
* fidelity on different machines, and the caller cannot know which it got.
|
|
10
|
+
*
|
|
11
|
+
* That is why both functions return a {@link HibernationOutcome} instead of
|
|
12
|
+
* `void`. molecule.dev's sandboxes run their dev servers as detached exec
|
|
13
|
+
* processes; after a stop-style wake the container is "running", the preview
|
|
14
|
+
* answers nothing, and `status` says everything is fine. The relaunch that fixes
|
|
15
|
+
* it is only expressible if the resume reports what it did.
|
|
16
|
+
*
|
|
17
|
+
* ## The parameter that is a query parameter
|
|
18
|
+
*
|
|
19
|
+
* Restoring from a checkpoint is `POST /containers/{id}/start?checkpoint=<name>`.
|
|
20
|
+
* The checkpoint name is a QUERY parameter. Passing it in a JSON body — which is
|
|
21
|
+
* an easy and silent mistake, since the daemon accepts the request and starts the
|
|
22
|
+
* container — performs an ordinary cold start while looking exactly like a
|
|
23
|
+
* successful restore.
|
|
24
|
+
*
|
|
25
|
+
* @module
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* The checkpoint name for a container. Derived, not passed in: it is an
|
|
29
|
+
* implementation detail of this bond's own suspend/resume pair, and a caller that
|
|
30
|
+
* supplied it could only get it wrong.
|
|
31
|
+
*
|
|
32
|
+
* @param containerId - The container id.
|
|
33
|
+
* @returns A stable checkpoint name.
|
|
34
|
+
*/
|
|
35
|
+
export function checkpointName(containerId) {
|
|
36
|
+
return `mol-hibernate-${containerId.slice(0, 12)}`;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Whether an error means "this host has no CRIU", as opposed to "this checkpoint
|
|
40
|
+
* failed". The former is permanent for the process; the latter is not, and
|
|
41
|
+
* treating it as permanent would silently downgrade every later hibernation.
|
|
42
|
+
*
|
|
43
|
+
* @param message - The error message.
|
|
44
|
+
* @returns `true` when the host does not support checkpointing at all.
|
|
45
|
+
*/
|
|
46
|
+
function isCheckpointUnsupported(message) {
|
|
47
|
+
return /\b501\b|not implemented|not supported|checkpoint.*disabled|experimental/i.test(message);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Suspend a sandbox, preferring a checkpoint and falling back to a stop.
|
|
51
|
+
*
|
|
52
|
+
* Never fails just because the high-fidelity path is unavailable: a sandbox that
|
|
53
|
+
* cannot be checkpointed must still be suspendable. The outcome reports which
|
|
54
|
+
* one ran.
|
|
55
|
+
*
|
|
56
|
+
* @param ctx - Hibernation context.
|
|
57
|
+
* @param containerId - The sandbox to suspend.
|
|
58
|
+
* @returns What the suspend actually did.
|
|
59
|
+
*/
|
|
60
|
+
export async function hibernateContainer(ctx, containerId) {
|
|
61
|
+
let fallbackReason = 'this host does not support CRIU checkpoints';
|
|
62
|
+
if (ctx.checkpoints.supported) {
|
|
63
|
+
try {
|
|
64
|
+
await ctx.request(`/containers/${containerId}/checkpoints`, 'POST', {
|
|
65
|
+
CheckpointID: checkpointName(containerId),
|
|
66
|
+
Exit: true,
|
|
67
|
+
});
|
|
68
|
+
return { processesPreserved: true, mechanism: 'checkpoint' };
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
72
|
+
if (isCheckpointUnsupported(message)) {
|
|
73
|
+
ctx.checkpoints.supported = false;
|
|
74
|
+
ctx.info?.('CRIU checkpoints are not available on this host — hibernating by stopping from now on', { containerId, error: message });
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
// A real checkpoint failure, on a host that CAN checkpoint. Do not
|
|
78
|
+
// disable the mechanism for the whole process over one bad attempt, but
|
|
79
|
+
// do not hide it either: every wake after this one costs a full relaunch.
|
|
80
|
+
ctx.warn?.('Checkpoint failed — falling back to stopping the sandbox', {
|
|
81
|
+
containerId,
|
|
82
|
+
error: message,
|
|
83
|
+
});
|
|
84
|
+
fallbackReason = `checkpoint failed: ${message}`;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
await ctx.request(`/containers/${containerId}/stop`, 'POST');
|
|
89
|
+
return { processesPreserved: false, mechanism: 'stop', detail: fallbackReason };
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Resume a suspended sandbox, preferring a checkpoint restore.
|
|
93
|
+
*
|
|
94
|
+
* A missing checkpoint is the ROUTINE case — it is what a stop-suspended sandbox
|
|
95
|
+
* looks like — so it falls through quietly. Any other restore failure is logged,
|
|
96
|
+
* because it means every wake on this host is silently taking the slow, empty
|
|
97
|
+
* path.
|
|
98
|
+
*
|
|
99
|
+
* @param ctx - Hibernation context.
|
|
100
|
+
* @param containerId - The sandbox to resume.
|
|
101
|
+
* @returns What the resume actually did.
|
|
102
|
+
*/
|
|
103
|
+
export async function resumeContainer(ctx, containerId) {
|
|
104
|
+
const name = checkpointName(containerId);
|
|
105
|
+
if (ctx.checkpoints.supported) {
|
|
106
|
+
try {
|
|
107
|
+
// Query parameter, not a body field — see this module's header.
|
|
108
|
+
await ctx.request(`/containers/${containerId}/start?checkpoint=${encodeURIComponent(name)}`, 'POST', undefined, 120_000);
|
|
109
|
+
// The checkpoint has been consumed; leaving it behind would make the next
|
|
110
|
+
// resume restore a stale process tree.
|
|
111
|
+
await ctx
|
|
112
|
+
.request(`/containers/${containerId}/checkpoints/${name}`, 'DELETE')
|
|
113
|
+
.catch((error) => {
|
|
114
|
+
ctx.warn?.('Could not delete a consumed checkpoint — the next resume may restore stale state', {
|
|
115
|
+
containerId,
|
|
116
|
+
error,
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
return { processesPreserved: true, mechanism: 'restore' };
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
123
|
+
if (isCheckpointUnsupported(message)) {
|
|
124
|
+
ctx.checkpoints.supported = false;
|
|
125
|
+
}
|
|
126
|
+
else if (!/no such (checkpoint|file)|does not exist|not found|\b404\b/i.test(message)) {
|
|
127
|
+
ctx.warn?.('Checkpoint restore failed — starting the sandbox cold instead', {
|
|
128
|
+
containerId,
|
|
129
|
+
error: message,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
// 120 s: a first-ever start populates a fresh named volume from the image,
|
|
135
|
+
// which is a multi-gigabyte copy on this fleet's base image.
|
|
136
|
+
await ctx.request(`/containers/${containerId}/start`, 'POST', undefined, 120_000);
|
|
137
|
+
return {
|
|
138
|
+
processesPreserved: false,
|
|
139
|
+
mechanism: 'start',
|
|
140
|
+
detail: 'started from a stopped state — no process tree was restored, so anything launched inside the sandbox must be relaunched',
|
|
141
|
+
};
|
|
142
|
+
}
|