@nylorun/runtime 0.34.0-beta → 0.36.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/adapters/sandbox/local.d.ts +10 -3
- package/dist/adapters/sandbox/local.js +21 -4
- package/dist/gates/client-hello.d.ts +26 -0
- package/dist/gates/client-hello.js +103 -0
- package/dist/gates/credential-provider.d.ts +15 -3
- package/dist/gates/credential-provider.js +54 -13
- package/dist/gates/egress.d.ts +2 -0
- package/dist/gates/egress.js +66 -5
- package/dist/harness/pod.js +3 -0
- package/dist/harness-api/start.js +9 -7
- package/dist/host/harness-entry.d.ts +1 -0
- package/dist/host/harness-entry.js +26 -0
- package/dist/management-openapi.json +1 -1
- package/dist/openapi.json +1 -1
- package/dist/sandbox/policy.d.ts +1 -1
- package/dist/sandbox/policy.js +15 -5
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.36.0-beta
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 27253ea: A pod starts faster and its commands can install what they need (ADR-92).
|
|
8
|
+
|
|
9
|
+
- **The pod copies a bundled engine.** The Runtime image ships the pod's engine (`--service harness`) bundled into one script, with the one native addon it loads, as `/engine`: 12 files instead of `/app`'s ~30,000. The sandboxes service's `nylo-copy` copies `/engine` when the image has it, and `/app` otherwise, and the workload runs `/nylo/entry.mjs`. On beta's GKE cluster the copy went from 6–29 s to 0.2 s, and the engine's start from 7.2 s to 3.7 s.
|
|
10
|
+
- **A pod's disk is owned once:** `fsGroupChangePolicy: OnRootMismatch`. A mount of a disk holding 91,000 files went from 15–21 s to 3–8 s.
|
|
11
|
+
- **Commands run with bash** where the image has it (`/bin/bash -c`), as the `bash` tool says; `/bin/sh -c` only in an image without bash.
|
|
12
|
+
- **Node uses the proxy.** `NODE_USE_ENV_PROXY=1` joins the proxy variables a pod's commands get, so Node's `fetch`, and corepack's pnpm and yarn, reach the egress gate.
|
|
13
|
+
- **The dev preset allows toolchain downloads:** `nodejs.org`, `releases.astral.sh`, `static.rust-lang.org`, `sh.rustup.rs`, `dl.google.com`, `mise-versions.jdx.dev`, `tuf-repo-cdn.sigstore.dev`, and `release-assets.githubusercontent.com`, where GitHub's release downloads now redirect.
|
|
14
|
+
- **The egress gate checks the server name.** On a 443 tunnel, the gate reads the client's TLS ClientHello and closes the tunnel when its server name (SNI) isn't the CONNECT host, is missing, or the bytes aren't TLS, recording the refusal (`sni_mismatch`, `sni_missing`, `sni_invalid`, `sni_timeout`). A tunnel to an allowed host can no longer ask a shared front end for another one.
|
|
15
|
+
|
|
16
|
+
## 0.35.0-beta
|
|
17
|
+
|
|
18
|
+
### Minor Changes
|
|
19
|
+
|
|
20
|
+
- 1376d3e: The outside credential provider can name host patterns, stand-ins and plain values (no protocol change). `POST /v1/hosts` may now answer `*.<suffix>` entries, such as `*.googleapis.com`, beside exact host names: egress-gate matches them as it matches `network.allow`, so `run.googleapis.com` and `a.b.googleapis.com` pass and `googleapis.com` itself does not. They still pass on 443 only, inside the Tenant's egress ceiling, a secret of the vault bound to the same host still wins, and `nylorun-provider-host` is still the host the sandbox addressed; an IP literal or a malformed entry is dropped. `POST /v1/environment` may now answer, for each variable, `{ name, standIn }`, which sets it to that stand-in (one shaped like the vendor's keys, for a CLI that checks a key's format), or `{ name, value }`, which sets it to that plain value (a setting beside a key, never a secret), beside a bare name, which is still `nylorun_managed`. A `standIn` or `value` is 1 to 1,024 printable characters with no control character or line break; any other entry is dropped, and the first entry for a name wins. The vault's own variables still win, and an older provider's names-only answer works as before.
|
|
21
|
+
|
|
3
22
|
## 0.34.0-beta
|
|
4
23
|
|
|
5
24
|
### Minor Changes
|
|
@@ -8,11 +8,18 @@ export interface LocalBackendOptions {
|
|
|
8
8
|
readonly env: Readonly<Record<string, string | undefined>>;
|
|
9
9
|
/** The engine's `/harness` on the same volume, a disk root of its versions (R2d step 3). */
|
|
10
10
|
readonly harnessRoot?: string;
|
|
11
|
+
/** The shell commands run under; default `/bin/bash` where the image has it, else `/bin/sh`. */
|
|
12
|
+
readonly shell?: string;
|
|
11
13
|
}
|
|
12
14
|
/**
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
|
|
15
|
+
* The shell the `bash` tool's commands run under: bash, as the tool says, where the image has it;
|
|
16
|
+
* `/bin/sh` (dash on Debian and Ubuntu) only in an image without it (ADR-92 §10).
|
|
17
|
+
*/
|
|
18
|
+
export declare function commandShell(exists?: (path: string) => boolean): string;
|
|
19
|
+
/**
|
|
20
|
+
* How the pod runs a command: `bash -c` (`sh -c` in an image without bash) in its own process
|
|
21
|
+
* group, with the base environment, the request's variables and the proxy of the moment. The
|
|
22
|
+
* sandbox tools and the disk's setup (`harness/setup.ts`) both run commands this way.
|
|
16
23
|
*/
|
|
17
24
|
export declare function localExec(options: LocalBackendOptions): (request: ExecRequest, signal: AbortSignal) => Promise<ExecResult>;
|
|
18
25
|
export declare function localBackend(options: LocalBackendOptions): SandboxBackend;
|
|
@@ -17,6 +17,22 @@ import { posix } from "node:path";
|
|
|
17
17
|
import { SANDBOX_WORKSPACE } from "@nylorun/core/define";
|
|
18
18
|
import { SandboxFileTooLargeError } from "../../sandbox/types.js";
|
|
19
19
|
import { hostTuning } from "../../tuning.js";
|
|
20
|
+
/**
|
|
21
|
+
* The shell the `bash` tool's commands run under: bash, as the tool says, where the image has it;
|
|
22
|
+
* `/bin/sh` (dash on Debian and Ubuntu) only in an image without it (ADR-92 §10).
|
|
23
|
+
*/
|
|
24
|
+
export function commandShell(exists = executable) {
|
|
25
|
+
return exists("/bin/bash") ? "/bin/bash" : "/bin/sh";
|
|
26
|
+
}
|
|
27
|
+
function executable(path) {
|
|
28
|
+
try {
|
|
29
|
+
accessSync(path, constants.X_OK);
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
20
36
|
/** This process's environment without the engine's own variables. */
|
|
21
37
|
function baseEnv(env, workspace) {
|
|
22
38
|
const out = {};
|
|
@@ -53,16 +69,17 @@ function collector() {
|
|
|
53
69
|
};
|
|
54
70
|
}
|
|
55
71
|
/**
|
|
56
|
-
* How the pod runs a command:
|
|
57
|
-
* the request's variables and the proxy of the moment. The
|
|
58
|
-
* (`harness/setup.ts`) both run commands this way.
|
|
72
|
+
* How the pod runs a command: `bash -c` (`sh -c` in an image without bash) in its own process
|
|
73
|
+
* group, with the base environment, the request's variables and the proxy of the moment. The
|
|
74
|
+
* sandbox tools and the disk's setup (`harness/setup.ts`) both run commands this way.
|
|
59
75
|
*/
|
|
60
76
|
export function localExec(options) {
|
|
61
77
|
const env = baseEnv(options.env, options.workspace ?? SANDBOX_WORKSPACE);
|
|
78
|
+
const shell = options.shell ?? commandShell();
|
|
62
79
|
return (request, signal) => new Promise((resolve, reject) => {
|
|
63
80
|
if (signal.aborted)
|
|
64
81
|
return reject(signal.reason ?? new Error("Turn cancelled"));
|
|
65
|
-
const child = spawn(
|
|
82
|
+
const child = spawn(shell, ["-c", request.command], {
|
|
66
83
|
cwd: request.cwd,
|
|
67
84
|
env: { ...env, ...(request.env ?? {}), ...(options.proxyEnv?.() ?? {}) },
|
|
68
85
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The server name (SNI) in a TLS ClientHello, read from the first bytes a client sends through a
|
|
3
|
+
* tunnel (ADR-92 §11). egress-gate compares it with the CONNECT host, so a tunnel opened to an
|
|
4
|
+
* allowed host can't ask a shared front end for another one (domain fronting by SNI).
|
|
5
|
+
*
|
|
6
|
+
* The ClientHello may span several TLS records; they are reassembled up to `MAX_HELLO_BYTES`.
|
|
7
|
+
*/
|
|
8
|
+
/** A ClientHello is far smaller; one that isn't complete by here is refused. */
|
|
9
|
+
export declare const MAX_HELLO_BYTES: number;
|
|
10
|
+
export type ClientHelloRead =
|
|
11
|
+
/** More bytes are needed. */
|
|
12
|
+
{
|
|
13
|
+
readonly status: "incomplete";
|
|
14
|
+
}
|
|
15
|
+
/** Not a TLS ClientHello. */
|
|
16
|
+
| {
|
|
17
|
+
readonly status: "invalid";
|
|
18
|
+
readonly reason: string;
|
|
19
|
+
}
|
|
20
|
+
/** A whole ClientHello; `serverName` is lowercase, or undefined when it names none. */
|
|
21
|
+
| {
|
|
22
|
+
readonly status: "complete";
|
|
23
|
+
readonly serverName?: string;
|
|
24
|
+
};
|
|
25
|
+
/** Reads the ClientHello at the start of `data`, the bytes a client has sent so far. */
|
|
26
|
+
export declare function readClientHello(data: Buffer): ClientHelloRead;
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The server name (SNI) in a TLS ClientHello, read from the first bytes a client sends through a
|
|
3
|
+
* tunnel (ADR-92 §11). egress-gate compares it with the CONNECT host, so a tunnel opened to an
|
|
4
|
+
* allowed host can't ask a shared front end for another one (domain fronting by SNI).
|
|
5
|
+
*
|
|
6
|
+
* The ClientHello may span several TLS records; they are reassembled up to `MAX_HELLO_BYTES`.
|
|
7
|
+
*/
|
|
8
|
+
/** A ClientHello is far smaller; one that isn't complete by here is refused. */
|
|
9
|
+
export const MAX_HELLO_BYTES = 64 * 1024;
|
|
10
|
+
const RECORD_HANDSHAKE = 0x16;
|
|
11
|
+
const HANDSHAKE_CLIENT_HELLO = 0x01;
|
|
12
|
+
const EXTENSION_SERVER_NAME = 0x0000;
|
|
13
|
+
const NAME_TYPE_HOST = 0x00;
|
|
14
|
+
/** Reads the ClientHello at the start of `data`, the bytes a client has sent so far. */
|
|
15
|
+
export function readClientHello(data) {
|
|
16
|
+
// The handshake message, reassembled from consecutive handshake records.
|
|
17
|
+
const chunks = [];
|
|
18
|
+
let have = 0;
|
|
19
|
+
let need;
|
|
20
|
+
let at = 0;
|
|
21
|
+
while (need === undefined || have < need) {
|
|
22
|
+
if (data.length < at + 5)
|
|
23
|
+
return data.length > MAX_HELLO_BYTES ? invalid("too long") : incomplete;
|
|
24
|
+
if (data[at] !== RECORD_HANDSHAKE)
|
|
25
|
+
return invalid("not a TLS handshake");
|
|
26
|
+
if (data[at + 1] !== 0x03)
|
|
27
|
+
return invalid("not a TLS record");
|
|
28
|
+
const length = data.readUInt16BE(at + 3);
|
|
29
|
+
if (length === 0)
|
|
30
|
+
return invalid("an empty record");
|
|
31
|
+
if (data.length < at + 5 + length)
|
|
32
|
+
return at + 5 + length > MAX_HELLO_BYTES ? invalid("too long") : incomplete;
|
|
33
|
+
const fragment = data.subarray(at + 5, at + 5 + length);
|
|
34
|
+
chunks.push(fragment);
|
|
35
|
+
have += fragment.length;
|
|
36
|
+
at += 5 + length;
|
|
37
|
+
if (need === undefined && have >= 4) {
|
|
38
|
+
const head = Buffer.concat(chunks);
|
|
39
|
+
if (head[0] !== HANDSHAKE_CLIENT_HELLO)
|
|
40
|
+
return invalid("not a ClientHello");
|
|
41
|
+
need = 4 + head.readUIntBE(1, 3);
|
|
42
|
+
if (need > MAX_HELLO_BYTES)
|
|
43
|
+
return invalid("too long");
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return serverNameIn(Buffer.concat(chunks).subarray(4, need));
|
|
47
|
+
}
|
|
48
|
+
/** The server name in a ClientHello's body (after its 4-byte handshake header). */
|
|
49
|
+
function serverNameIn(body) {
|
|
50
|
+
let at = 2 + 32; // client_version, random
|
|
51
|
+
if (body.length < at + 1)
|
|
52
|
+
return invalid("truncated");
|
|
53
|
+
at += 1 + body[at]; // session_id
|
|
54
|
+
if (body.length < at + 2)
|
|
55
|
+
return invalid("truncated");
|
|
56
|
+
at += 2 + body.readUInt16BE(at); // cipher_suites
|
|
57
|
+
if (body.length < at + 1)
|
|
58
|
+
return invalid("truncated");
|
|
59
|
+
at += 1 + body[at]; // compression_methods
|
|
60
|
+
if (body.length === at)
|
|
61
|
+
return { status: "complete" }; // no extensions
|
|
62
|
+
if (body.length < at + 2)
|
|
63
|
+
return invalid("truncated");
|
|
64
|
+
const end = at + 2 + body.readUInt16BE(at);
|
|
65
|
+
if (end > body.length)
|
|
66
|
+
return invalid("truncated");
|
|
67
|
+
at += 2;
|
|
68
|
+
while (at + 4 <= end) {
|
|
69
|
+
const type = body.readUInt16BE(at);
|
|
70
|
+
const length = body.readUInt16BE(at + 2);
|
|
71
|
+
const start = at + 4;
|
|
72
|
+
if (start + length > end)
|
|
73
|
+
return invalid("truncated");
|
|
74
|
+
if (type === EXTENSION_SERVER_NAME)
|
|
75
|
+
return hostIn(body.subarray(start, start + length));
|
|
76
|
+
at = start + length;
|
|
77
|
+
}
|
|
78
|
+
return { status: "complete" };
|
|
79
|
+
}
|
|
80
|
+
/** The host name in a server_name extension's data. */
|
|
81
|
+
function hostIn(extension) {
|
|
82
|
+
if (extension.length < 2)
|
|
83
|
+
return invalid("truncated");
|
|
84
|
+
const end = Math.min(extension.length, 2 + extension.readUInt16BE(0));
|
|
85
|
+
let at = 2;
|
|
86
|
+
while (at + 3 <= end) {
|
|
87
|
+
const type = extension[at];
|
|
88
|
+
const length = extension.readUInt16BE(at + 1);
|
|
89
|
+
if (at + 3 + length > end)
|
|
90
|
+
return invalid("truncated");
|
|
91
|
+
if (type === NAME_TYPE_HOST) {
|
|
92
|
+
const name = extension.subarray(at + 3, at + 3 + length).toString("latin1");
|
|
93
|
+
// A host name is letters, digits, hyphens and dots; anything else (a NUL, a space) is refused.
|
|
94
|
+
if (!/^[A-Za-z0-9.-]+$/.test(name))
|
|
95
|
+
return invalid("a server name that isn't a host name");
|
|
96
|
+
return { status: "complete", serverName: name.toLowerCase().replace(/\.$/, "") };
|
|
97
|
+
}
|
|
98
|
+
at += 3 + length;
|
|
99
|
+
}
|
|
100
|
+
return { status: "complete" };
|
|
101
|
+
}
|
|
102
|
+
const incomplete = { status: "incomplete" };
|
|
103
|
+
const invalid = (reason) => ({ status: "invalid", reason });
|
|
@@ -12,6 +12,12 @@ export declare const PROVIDER_HEADERS: {
|
|
|
12
12
|
};
|
|
13
13
|
/** Every header of this prefix a sandbox sends is dropped before the engine sets its own. */
|
|
14
14
|
export declare const PROVIDER_HEADER_PREFIX = "nylorun-provider-";
|
|
15
|
+
/** One variable a pod Agent's segments set: its name and the value the sandbox sees. */
|
|
16
|
+
export interface ProviderVariable {
|
|
17
|
+
readonly name: string;
|
|
18
|
+
/** The stand-in (`nylorun_managed` unless the provider names another) or a plain value. */
|
|
19
|
+
readonly value: string;
|
|
20
|
+
}
|
|
15
21
|
/** Where the provider is and its token (`StackConfig.credentialProvider`). */
|
|
16
22
|
export interface CredentialProviderSettings {
|
|
17
23
|
/** `NYLORUN_CREDENTIAL_PROVIDER_URL`, without a trailing slash. */
|
|
@@ -22,9 +28,15 @@ export interface CredentialProviderSettings {
|
|
|
22
28
|
export interface CredentialProvider {
|
|
23
29
|
/** The provider's base URL, without a trailing slash. */
|
|
24
30
|
readonly url: string;
|
|
25
|
-
/**
|
|
26
|
-
|
|
27
|
-
|
|
31
|
+
/**
|
|
32
|
+
* The variables `agentId`'s sessions set, each with its stand-in or plain value; none without
|
|
33
|
+
* an owner.
|
|
34
|
+
*/
|
|
35
|
+
environment(agentId: string, owner: string | undefined): Promise<readonly ProviderVariable[]>;
|
|
36
|
+
/**
|
|
37
|
+
* The hosts `owner`'s keys are bound to (lowercase): exact names and `*.<suffix>` patterns, to
|
|
38
|
+
* match as `network.allow`'s do (`egressAllows`); none without an owner.
|
|
39
|
+
*/
|
|
28
40
|
hosts(agentId: string, owner: string | undefined): Promise<readonly string[]>;
|
|
29
41
|
/** Where one request to `path` (origin form, `/v2/projects?x=1`) is handed. */
|
|
30
42
|
forwardUrl(path: string): URL;
|
|
@@ -6,11 +6,19 @@
|
|
|
6
6
|
* here runs.
|
|
7
7
|
*
|
|
8
8
|
* Three calls, each with `Authorization: Bearer <token>`:
|
|
9
|
-
* - `POST <url>/v1/hosts` `{ agentId, owner }` → `{ hosts: string[] }`: the
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
9
|
+
* - `POST <url>/v1/hosts` `{ agentId, owner }` → `{ hosts: string[] }`: the hosts the owner's keys
|
|
10
|
+
* are bound to, each an exact host name or a `*.<suffix>` pattern (`*.googleapis.com`, v0.26
|
|
11
|
+
* TM-E1). A pattern matches as `network.allow`'s do: `run.googleapis.com` and
|
|
12
|
+
* `a.b.googleapis.com`, never `googleapis.com` itself. An IP literal or a malformed entry is
|
|
13
|
+
* dropped. egress-gate lets a pod Agent's sandbox reach them (inside the Tenant's egress
|
|
14
|
+
* ceiling), terminates TLS for them on 443 and hands each request on;
|
|
15
|
+
* - `POST <url>/v1/environment` `{ agentId, owner }` → `{ variables: Entry[] }`: the variables each
|
|
16
|
+
* of a pod Agent's segments sets, under the vault's own (v0.26 TM-E2). An entry is a name (`"X"`,
|
|
17
|
+
* set to the stand-in `nylorun_managed`, as before v0.26), `{ name, standIn }` (set to that
|
|
18
|
+
* stand-in, shaped like the vendor's keys) or `{ name, value }` (set to that plain value, a
|
|
19
|
+
* setting beside a key and never a secret). A `standIn` or `value` is 1 to 1,024 printable
|
|
20
|
+
* characters, with no control character or line break; an entry with anything else, or a name
|
|
21
|
+
* a shell can't set, is dropped, and the first entry for a name wins;
|
|
14
22
|
* - `ANY <url>/v1/forward/<path and query>`: one request of the sandbox's, with its headers as it
|
|
15
23
|
* sent them (its own `Authorization` or inject header included, which the provider replaces),
|
|
16
24
|
* plus `nylorun-provider-host`, `-agent`, `-owner` and, in place of a bearer the sandbox's own
|
|
@@ -23,6 +31,7 @@
|
|
|
23
31
|
* answers nothing, kept for `retryMs`, so the gate refuses what it refused before; it is logged
|
|
24
32
|
* once until a call succeeds again. The token is never logged.
|
|
25
33
|
*/
|
|
34
|
+
import { ENVIRONMENT_SECRET_SENTINEL } from "@nylorun/core/contracts";
|
|
26
35
|
import { isSandboxHostPattern } from "@nylorun/core/define";
|
|
27
36
|
import { hostTuning } from "../tuning.js";
|
|
28
37
|
/** The headers the engine adds to a request it hands to the provider's forward route. */
|
|
@@ -42,6 +51,8 @@ export const PROVIDER_HEADER_PREFIX = "nylorun-provider-";
|
|
|
42
51
|
const ANSWER_MAX = 256;
|
|
43
52
|
/** A variable name a shell can set. */
|
|
44
53
|
const VARIABLE = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/;
|
|
54
|
+
/** A stand-in or plain value: 1 to 1,024 printable characters, no control character or line break. */
|
|
55
|
+
const VALUE = /^[^\p{Cc}\p{Cs}\p{Zl}\p{Zp}]{1,1024}$/u;
|
|
45
56
|
export function createCredentialProvider(settings, options) {
|
|
46
57
|
const url = settings.url.replace(/\/+$/, "");
|
|
47
58
|
const { logger } = options;
|
|
@@ -89,11 +100,7 @@ export function createCredentialProvider(settings, options) {
|
|
|
89
100
|
return fail(call, agentId, "shape");
|
|
90
101
|
if (failing.delete(call))
|
|
91
102
|
logger.info("credential_provider_recovered", { call });
|
|
92
|
-
|
|
93
|
-
const value = call === "hosts"
|
|
94
|
-
? [...new Set(items.map((host) => host.toLowerCase().replace(/\.$/, "")).filter(isExactHost))]
|
|
95
|
-
: [...new Set(items.filter((name) => VARIABLE.test(name)))];
|
|
96
|
-
return { ok: true, value };
|
|
103
|
+
return { ok: true, value: call === "hosts" ? hostsOf(list) : variablesOf(list) };
|
|
97
104
|
}
|
|
98
105
|
function answer(call, agentId, owner) {
|
|
99
106
|
if (owner === undefined || owner === "")
|
|
@@ -134,7 +141,41 @@ export function createCredentialProvider(settings, options) {
|
|
|
134
141
|
}),
|
|
135
142
|
};
|
|
136
143
|
}
|
|
137
|
-
/**
|
|
138
|
-
|
|
139
|
-
|
|
144
|
+
/**
|
|
145
|
+
* The hosts of a `/v1/hosts` answer: exact host names and `*.<suffix>` patterns, lowercase, without
|
|
146
|
+
* a trailing root dot, once each; an IP literal or a malformed entry is dropped.
|
|
147
|
+
*/
|
|
148
|
+
function hostsOf(list) {
|
|
149
|
+
const items = list.filter((item) => typeof item === "string").slice(0, ANSWER_MAX);
|
|
150
|
+
return [...new Set(items.map((host) => host.toLowerCase().replace(/\.$/, "")).filter(isSandboxHostPattern))];
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* The variables of a `/v1/environment` answer, each with the value the sandbox sees: a bare name is
|
|
154
|
+
* the stand-in `nylorun_managed`, `{ name, standIn }` that stand-in, `{ name, value }` that value.
|
|
155
|
+
* Any other entry is dropped, and the first entry for a name wins.
|
|
156
|
+
*/
|
|
157
|
+
function variablesOf(list) {
|
|
158
|
+
const variables = new Map();
|
|
159
|
+
for (const item of list) {
|
|
160
|
+
if (variables.size >= ANSWER_MAX)
|
|
161
|
+
break;
|
|
162
|
+
const variable = variableOf(item);
|
|
163
|
+
if (variable && !variables.has(variable.name))
|
|
164
|
+
variables.set(variable.name, variable);
|
|
165
|
+
}
|
|
166
|
+
return [...variables.values()];
|
|
167
|
+
}
|
|
168
|
+
function variableOf(item) {
|
|
169
|
+
if (typeof item === "string")
|
|
170
|
+
return VARIABLE.test(item) ? { name: item, value: ENVIRONMENT_SECRET_SENTINEL } : undefined;
|
|
171
|
+
if (typeof item !== "object" || item === null || Array.isArray(item))
|
|
172
|
+
return undefined;
|
|
173
|
+
const { name, standIn, value } = item;
|
|
174
|
+
if (typeof name !== "string" || !VARIABLE.test(name))
|
|
175
|
+
return undefined;
|
|
176
|
+
// Exactly one of the two: an entry naming both, or neither, is not one the engine can read.
|
|
177
|
+
if ((standIn === undefined) === (value === undefined))
|
|
178
|
+
return undefined;
|
|
179
|
+
const set = standIn ?? value;
|
|
180
|
+
return typeof set === "string" && VALUE.test(set) ? { name, value: set } : undefined;
|
|
140
181
|
}
|
package/dist/gates/egress.d.ts
CHANGED
|
@@ -56,6 +56,8 @@ export interface StartEgressGateOptions {
|
|
|
56
56
|
readonly cacheMs?: number;
|
|
57
57
|
/** How long the gate waits for the upstream connection. Default `egress.connectTimeoutMs`. */
|
|
58
58
|
readonly connectTimeoutMs?: number;
|
|
59
|
+
/** How long a 443 tunnel waits for the client's ClientHello. Default 15 s. */
|
|
60
|
+
readonly helloMs?: number;
|
|
59
61
|
/** Tests replace name resolution. */
|
|
60
62
|
readonly lookup?: (host: string) => Promise<readonly {
|
|
61
63
|
address: string;
|
package/dist/gates/egress.js
CHANGED
|
@@ -12,12 +12,17 @@
|
|
|
12
12
|
* else 407;
|
|
13
13
|
* 3. the port is 443 or 80, the target is a host name and no IP literal, and it matches the spec's
|
|
14
14
|
* `network.allow` exactly or by `*.suffix`, or (443 only) the outside credential provider lists
|
|
15
|
-
* it for the Agent's owner and the Tenant's egress ceiling
|
|
15
|
+
* it for the Agent's owner, exactly or by `*.suffix` likewise, and the Tenant's egress ceiling
|
|
16
|
+
* holds it, else 403;
|
|
16
17
|
* 4. the sandbox has fewer than `maxPerSandbox` tunnels open, else 429;
|
|
17
18
|
* 5. the name resolves, and to at least one address that is not private, loopback, link-local
|
|
18
19
|
* (cloud metadata included), multicast or reserved, else 502 or 403;
|
|
19
20
|
* 6. the gate connects to that checked address (never to the name again) and pipes.
|
|
20
21
|
*
|
|
22
|
+
* On port 443 the tunnel waits for the client's TLS ClientHello and refuses it, closing the tunnel,
|
|
23
|
+
* when its server name (SNI) isn't the CONNECT host, is missing, or the bytes aren't TLS (ADR-92
|
|
24
|
+
* §11): a tunnel to an allowed host can't ask a shared front end for another one.
|
|
25
|
+
*
|
|
21
26
|
* A tunnel is opaque bytes, with two exceptions. First (R2c, D50): port 443 of a host that an
|
|
22
27
|
* `environment_secret` of a live session on the sandbox is bound to. There the gate terminates
|
|
23
28
|
* TLS and sets the credential's header on every request (`egress-credentials.ts`), still sending
|
|
@@ -47,6 +52,7 @@ import { isPrivateAddress } from "../tenant/outbound.js";
|
|
|
47
52
|
import { readSandboxConfig } from "../sandbox/tenant-config.js";
|
|
48
53
|
import { withinCeiling } from "../sandbox/resolve.js";
|
|
49
54
|
import { bindListener } from "../host/http.js";
|
|
55
|
+
import { MAX_HELLO_BYTES, readClientHello } from "./client-hello.js";
|
|
50
56
|
import { hostTuning } from "../tuning.js";
|
|
51
57
|
import { CONNECT_ESTABLISHED, credentialTerminator, } from "./egress-credentials.js";
|
|
52
58
|
/**
|
|
@@ -227,15 +233,18 @@ export async function startEgressGate(options) {
|
|
|
227
233
|
: undefined;
|
|
228
234
|
const provider = terminator ? options.provider : undefined;
|
|
229
235
|
/**
|
|
230
|
-
* Whether the credential provider lists `host` for the sandbox's owner,
|
|
231
|
-
*
|
|
236
|
+
* Whether the credential provider lists `host` for the sandbox's owner, exactly or by a
|
|
237
|
+
* `*.suffix` pattern (`*.x.com` holds `a.x.com` and `b.a.x.com`, not `x.com`), inside the
|
|
238
|
+
* Tenant's ceiling: only on 443, where the gate terminates TLS, and never for an Agent without an
|
|
239
|
+
* owner. The request still names the host the sandbox addressed (`nylorun-provider-host`).
|
|
232
240
|
*/
|
|
233
241
|
async function providerLists(sandboxId, sandbox, host, port) {
|
|
234
242
|
if (!provider || port !== 443 || sandbox.owner === undefined)
|
|
235
243
|
return false;
|
|
236
244
|
if (sandbox.ceiling && !withinCeiling(host, sandbox.ceiling))
|
|
237
245
|
return false;
|
|
238
|
-
|
|
246
|
+
// An exact name or a `*.suffix` pattern, matched as `network.allow`'s are (v0.26 TM-E1).
|
|
247
|
+
return egressAllows(host, await provider.hosts(sandboxId, sandbox.owner));
|
|
239
248
|
}
|
|
240
249
|
const open = new Map();
|
|
241
250
|
const tunnels = new Set();
|
|
@@ -357,11 +366,63 @@ export async function startEgressGate(options) {
|
|
|
357
366
|
record.closed();
|
|
358
367
|
});
|
|
359
368
|
client.write(CONNECT_ESTABLISHED);
|
|
360
|
-
if (
|
|
369
|
+
if (port === 443) {
|
|
370
|
+
// The server name the client asks for must be the host it was allowed (ADR-92 §11).
|
|
371
|
+
const hello = await clientHello(client, head, options.helloMs ?? 15_000);
|
|
372
|
+
if (hello.serverName !== host.toLowerCase()) {
|
|
373
|
+
const reason = hello.serverName === undefined ? `sni_${hello.problem ?? "missing"}` : "sni_mismatch";
|
|
374
|
+
logger.warn("egress_refused", { status: 403, reason, sandboxId, host, port, ...(hello.serverName ? { serverName: hello.serverName } : {}) });
|
|
375
|
+
record.refused(403, reason);
|
|
376
|
+
close();
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
upstream.write(hello.bytes);
|
|
380
|
+
}
|
|
381
|
+
else if (head.length > 0)
|
|
361
382
|
upstream.write(head);
|
|
362
383
|
upstream.pipe(client);
|
|
363
384
|
client.pipe(upstream);
|
|
364
385
|
}
|
|
386
|
+
/**
|
|
387
|
+
* The client's first bytes on a tunnel, up to the end of its ClientHello, and the server name in
|
|
388
|
+
* it. The client is paused afterwards, so the bytes stay in order for the pipe.
|
|
389
|
+
*/
|
|
390
|
+
function clientHello(client, head, timeoutMs) {
|
|
391
|
+
return new Promise((resolve) => {
|
|
392
|
+
let bytes = head;
|
|
393
|
+
let done = false;
|
|
394
|
+
const finish = (result) => {
|
|
395
|
+
if (done)
|
|
396
|
+
return;
|
|
397
|
+
done = true;
|
|
398
|
+
clearTimeout(timer);
|
|
399
|
+
client.off("data", onData);
|
|
400
|
+
client.off("close", onClose);
|
|
401
|
+
client.pause();
|
|
402
|
+
resolve({ bytes, ...result });
|
|
403
|
+
};
|
|
404
|
+
const check = () => {
|
|
405
|
+
const read = readClientHello(bytes);
|
|
406
|
+
if (read.status === "complete")
|
|
407
|
+
finish(read.serverName ? { serverName: read.serverName } : { problem: "missing" });
|
|
408
|
+
else if (read.status === "invalid")
|
|
409
|
+
finish({ problem: "invalid" });
|
|
410
|
+
else if (bytes.length > MAX_HELLO_BYTES)
|
|
411
|
+
finish({ problem: "invalid" });
|
|
412
|
+
};
|
|
413
|
+
const onData = (chunk) => {
|
|
414
|
+
bytes = Buffer.concat([bytes, chunk]);
|
|
415
|
+
check();
|
|
416
|
+
};
|
|
417
|
+
const onClose = () => finish({ problem: "missing" });
|
|
418
|
+
const timer = setTimeout(() => finish({ problem: "timeout" }), timeoutMs);
|
|
419
|
+
timer.unref?.();
|
|
420
|
+
client.on("data", onData);
|
|
421
|
+
client.once("close", onClose);
|
|
422
|
+
if (bytes.length > 0)
|
|
423
|
+
check();
|
|
424
|
+
});
|
|
425
|
+
}
|
|
365
426
|
/** Connects to the first of `addresses` that answers within the connect timeout. */
|
|
366
427
|
async function dialFirst(addresses, port) {
|
|
367
428
|
for (const address of addresses) {
|
package/dist/harness/pod.js
CHANGED
|
@@ -160,6 +160,9 @@ export function podHost(config, logger, options = {}) {
|
|
|
160
160
|
http_proxy: proxy,
|
|
161
161
|
NO_PROXY: "localhost,127.0.0.1",
|
|
162
162
|
no_proxy: "localhost,127.0.0.1",
|
|
163
|
+
// Node's fetch (and corepack's pnpm and yarn) use the proxy only with this set; the engine
|
|
164
|
+
// drops `*_PROXY` names from the image's own environment, so it comes from here (ADR-92 §10).
|
|
165
|
+
NODE_USE_ENV_PROXY: "1",
|
|
163
166
|
};
|
|
164
167
|
},
|
|
165
168
|
get epoch() {
|
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
*
|
|
8
8
|
* A pod Agent's sandbox variables are its vault's (R2c), then, with an outside credential
|
|
9
9
|
* provider (`gates/credential-provider.ts`) and an owner on the Agent, each variable the provider
|
|
10
|
-
* names for that owner, set to the stand-in `nylorun_managed
|
|
10
|
+
* names for that owner, set to the value it gives (the stand-in `nylorun_managed`, a stand-in of
|
|
11
|
+
* its own or a plain value) and never over a vault variable.
|
|
11
12
|
* The provider is asked after the transaction, and one that fails adds nothing.
|
|
12
13
|
*/
|
|
13
|
-
import {
|
|
14
|
+
import { harnessLimitsOf } from "@nylorun/core/contracts";
|
|
14
15
|
import { SANDBOX_WORKSPACE } from "@nylorun/core/define";
|
|
15
16
|
import { isWorkflowManifest } from "../core/flow-host.js";
|
|
16
17
|
import { sandboxWorkspaceOf } from "../sandbox/share.js";
|
|
@@ -114,18 +115,19 @@ options = {}) {
|
|
|
114
115
|
return { ...started, transcript, cursor };
|
|
115
116
|
}
|
|
116
117
|
/**
|
|
117
|
-
* `opened` with the credential provider's variables for `ask` added to its sandbox's environment
|
|
118
|
-
*
|
|
118
|
+
* `opened` with the credential provider's variables for `ask` added to its sandbox's environment,
|
|
119
|
+
* each with the value the provider gives (a stand-in or a plain value), under the vault's own;
|
|
120
|
+
* `opened` itself when there are none.
|
|
119
121
|
*/
|
|
120
122
|
async function withProviderVariables(ctx, opened, ask) {
|
|
121
123
|
if (!ask || !ctx.credentialProvider)
|
|
122
124
|
return opened;
|
|
123
|
-
const
|
|
125
|
+
const variables = await ctx.credentialProvider.environment(ask.agentId, ask.owner);
|
|
124
126
|
const vault = opened.sandbox.environment ?? {};
|
|
125
|
-
const added =
|
|
127
|
+
const added = variables.filter(({ name }) => !Object.hasOwn(vault, name));
|
|
126
128
|
if (added.length === 0)
|
|
127
129
|
return opened;
|
|
128
|
-
const environment = { ...Object.fromEntries(added.map((name) => [name,
|
|
130
|
+
const environment = { ...Object.fromEntries(added.map(({ name, value }) => [name, value])), ...vault };
|
|
129
131
|
return { ...opened, sandbox: { ...opened.sandbox, environment } };
|
|
130
132
|
}
|
|
131
133
|
/** The run's `turn.start`. */
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pod's engine (`--service harness` in a sandbox pod), on its own: what `main()` does for that
|
|
3
|
+
* service, without importing the rest of the Host. `scripts/bundle-harness.mjs` bundles this file
|
|
4
|
+
* into one script, which the Runtime image ships as `/engine/harness.mjs` and the sandboxes service
|
|
5
|
+
* copies into each pod instead of `/app` (ADR-92 §5: 12 files instead of about 30,000).
|
|
6
|
+
*/
|
|
7
|
+
import { baselineEnvironment } from "./environment.js";
|
|
8
|
+
import { parseStackConfig } from "./stack-config.js";
|
|
9
|
+
import { runHarness } from "../harness/main.js";
|
|
10
|
+
import { useHostTuning } from "../tuning.js";
|
|
11
|
+
import { useHostIdentity } from "../vault/host-identity.js";
|
|
12
|
+
import { useWorkloadToken } from "../vault/workload-token.js";
|
|
13
|
+
async function start() {
|
|
14
|
+
const stack = parseStackConfig(process.env, process.argv.slice(2));
|
|
15
|
+
if (!stack.services.has("harness"))
|
|
16
|
+
throw new Error("This entry runs only --service harness");
|
|
17
|
+
if (stack.tuning)
|
|
18
|
+
useHostTuning(stack.tuning.values, stack.tuning.overrides);
|
|
19
|
+
useWorkloadToken(stack.workloadToken ?? {});
|
|
20
|
+
useHostIdentity(stack.modelHostIdentity ?? []);
|
|
21
|
+
await runHarness(stack, baselineEnvironment(process.env));
|
|
22
|
+
}
|
|
23
|
+
void start().catch((error) => {
|
|
24
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
25
|
+
process.exit(1);
|
|
26
|
+
});
|
package/dist/openapi.json
CHANGED
package/dist/sandbox/policy.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ export declare const DEFAULT_SANDBOX_CPUS = 2;
|
|
|
6
6
|
export declare const DEFAULT_SANDBOX_MEMORY_MIB = 1024;
|
|
7
7
|
/** The default of the Tenant's Environment setting `limits.idle` (`environments.config`, D36). */
|
|
8
8
|
export declare const DEFAULT_SANDBOX_IDLE = "15m";
|
|
9
|
-
/** Package registries and code hosts. Concrete names so every backend can express them. */
|
|
9
|
+
/** Package registries, toolchain downloads and code hosts. Concrete names so every backend can express them. */
|
|
10
10
|
export declare const DEV_PRESET_HOSTS: readonly string[];
|
|
11
11
|
export declare function resolveNetwork(manifest: SandboxManifest | undefined): ResolvedNetwork;
|
|
12
12
|
/**
|
package/dist/sandbox/policy.js
CHANGED
|
@@ -6,21 +6,29 @@ export const DEFAULT_SANDBOX_CPUS = 2;
|
|
|
6
6
|
export const DEFAULT_SANDBOX_MEMORY_MIB = 1024;
|
|
7
7
|
/** The default of the Tenant's Environment setting `limits.idle` (`environments.config`, D36). */
|
|
8
8
|
export const DEFAULT_SANDBOX_IDLE = "15m";
|
|
9
|
-
/** Package registries and code hosts. Concrete names so every backend can express them. */
|
|
9
|
+
/** Package registries, toolchain downloads and code hosts. Concrete names so every backend can express them. */
|
|
10
10
|
export const DEV_PRESET_HOSTS = Object.freeze([
|
|
11
|
-
// npm and yarn
|
|
11
|
+
// npm and yarn; Node's own downloads, including node-gyp's headers for native modules
|
|
12
12
|
"registry.npmjs.org",
|
|
13
13
|
"registry.yarnpkg.com",
|
|
14
|
-
|
|
14
|
+
"nodejs.org",
|
|
15
|
+
// Python; uv's Python builds
|
|
15
16
|
"pypi.org",
|
|
16
17
|
"files.pythonhosted.org",
|
|
17
|
-
|
|
18
|
+
"releases.astral.sh",
|
|
19
|
+
// Rust and rustup
|
|
18
20
|
"crates.io",
|
|
19
21
|
"index.crates.io",
|
|
20
22
|
"static.crates.io",
|
|
21
|
-
|
|
23
|
+
"static.rust-lang.org",
|
|
24
|
+
"sh.rustup.rs",
|
|
25
|
+
// Go, and Go's toolchain downloads
|
|
22
26
|
"proxy.golang.org",
|
|
23
27
|
"sum.golang.org",
|
|
28
|
+
"dl.google.com",
|
|
29
|
+
// mise: its version lists, and the signatures it checks on GitHub releases
|
|
30
|
+
"mise-versions.jdx.dev",
|
|
31
|
+
"tuf-repo-cdn.sigstore.dev",
|
|
24
32
|
// Debian and Ubuntu package mirrors
|
|
25
33
|
"deb.debian.org",
|
|
26
34
|
"security.debian.org",
|
|
@@ -32,6 +40,8 @@ export const DEV_PRESET_HOSTS = Object.freeze([
|
|
|
32
40
|
"api.github.com",
|
|
33
41
|
"codeload.github.com",
|
|
34
42
|
"objects.githubusercontent.com",
|
|
43
|
+
// GitHub's release downloads redirect here
|
|
44
|
+
"release-assets.githubusercontent.com",
|
|
35
45
|
"raw.githubusercontent.com",
|
|
36
46
|
"gitlab.com",
|
|
37
47
|
]);
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const RUNTIME_VERSION = "0.
|
|
1
|
+
export declare const RUNTIME_VERSION = "0.36.0-beta";
|
package/dist/version.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
// Kept in step with package.json by runtime/server/scripts/check-package.mjs, which runs in npm run check.
|
|
2
2
|
// release:prepare also rewrites this when it bumps @nylorun/runtime.
|
|
3
|
-
export const RUNTIME_VERSION = "0.
|
|
3
|
+
export const RUNTIME_VERSION = "0.36.0-beta";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nylorun/runtime",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.36.0-beta",
|
|
4
4
|
"description": "Nylorun Runtime: the agent execution Host on Postgres, Restate and S2, and model providers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -92,6 +92,7 @@
|
|
|
92
92
|
"@types/ws": "^8.18.2",
|
|
93
93
|
"@typescript/native": "npm:typescript@^7.0.2",
|
|
94
94
|
"drizzle-kit": "^0.31.11",
|
|
95
|
+
"esbuild": "^0.28.2",
|
|
95
96
|
"typescript": "npm:@typescript/typescript6@^6.0.2",
|
|
96
97
|
"vitest": "^5.0.4"
|
|
97
98
|
}
|