nylorun 0.21.0-beta → 0.22.0-beta

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # nylorun
 
+## 0.22.0-beta
+
+### Minor Changes
+
+- 77902da: `nylorun local sandbox enable --probe-ports <allowed>,<denied>` publishes the NetworkPolicy probe's listener on those two host ports instead of two free ones. A firewall in front of the Docker host that opens only named ports to the pods, such as a cloud VPC's, can then open them; without it, the probe's random ports never reach the host and `enable` refuses the cluster. The ports must differ from each other and from the sandbox ports.
+
+### Patch Changes
+
+- 41b5741: `nylorun local sandbox enable` waits for a newly installed agent-sandbox CRD by polling its `Established` condition. It used `kubectl wait --for=condition=Established`, which fails at once, with ".status.conditions accessor error", when the API server has not yet given the CRD any conditions; GKE answers that way right after the apply.
+
 ## 0.21.0-beta
 
 ### Minor Changes

package/README.md

@@ -117,7 +117,7 @@ nylorun local ls [--json]          # the Tenants on this machine, with their sta
 nylorun local delete <tenant> --yes   # remove a Tenant: containers, volumes, Host root and vault key
 nylorun local logs [service] [--tenant <name>] [-f] [--tail <n>]   # postgres, restate, s2-lite, rustfs, gateway, runtime, harness, studio, sandboxes
 nylorun local reset [--tenant <name>] [--yes]   # delete the Tenant's volumes, Tenant directory and vault key; the next start creates it anew
-nylorun local sandbox enable --context <name> [--tenant <name>] [--host-address <ip>] [--bind-address <ip>] [--no-pull]
+nylorun local sandbox enable --context <name> [--tenant <name>] [--host-address <ip>] [--bind-address <ip>] [--probe-ports <allowed>,<denied>] [--no-pull]
 nylorun local sandbox disable [--tenant <name>] [--delete-namespace]
 nylorun local sandbox status [--tenant <name>] [--json]   # pods on a Kubernetes context (see Sandboxes)
 nylorun local telemetry [status|enable|disable]   # Studio's anonymous usage analytics
@@ -428,6 +428,10 @@ with a probe that the cluster enforces NetworkPolicy (and refuses it otherwise),
 lets pods reach only three ports on the Docker host, and adds the `sandboxes`
 service to the Tenant: the one container holding the cluster credentials
 (`<Host root>/sandboxes/`). On kind under Linux, pass `--host-address 172.17.0.1`.
+The probe publishes two free ports on the Docker host. Behind a firewall that
+opens only named ports to the pods, such as a cloud VPC's, pass two of them with
+`--probe-ports <allowed>,<denied>`. Open both, so that only NetworkPolicy can
+block the denied one.
 `nylorun local sandbox disable` removes the service (`--delete-namespace` also deletes
 the namespace and every sandbox in it). Sessions run in a pod when their
 Agent's manifest has `runtime.kind` `pod`.

package/dist/sandbox/commands.d.ts

@@ -1,8 +1,9 @@
 import { type StackDeps } from "../stack/commands.js";
 import type { FetchLike } from "../stack/studio-login.js";
 import { type ClusterFile } from "./cluster-file.js";
-import { type KubectlRunner } from "./kubectl.js";
-export declare const sandboxUsage = "  sandbox enable --context <name> [--tenant <name>] [--host-address <ip>] [--bind-address <ip>] [--no-pull]\n                                      run the Tenant's sandboxes as pods on that kubeconfig context (agent-sandbox v1.0.5)\n  sandbox disable [--tenant <name>] [--delete-namespace]\n                                      stop the sandboxes service and forget the cluster\n  sandbox status [--tenant <name>] [--json]\n                                      the cluster, the sandboxes service's readiness and what it reports";
+import { type Kube, type KubectlRunner } from "./kubectl.js";
+import { type ProbePorts } from "./probe.js";
+export declare const sandboxUsage = "  sandbox enable --context <name> [--tenant <name>] [--host-address <ip>] [--bind-address <ip>] [--probe-ports <allowed>,<denied>] [--no-pull]\n                                      run the Tenant's sandboxes as pods on that kubeconfig context (agent-sandbox v1.0.5)\n  sandbox disable [--tenant <name>] [--delete-namespace]\n                                      stop the sandboxes service and forget the cluster\n  sandbox status [--tenant <name>] [--json]\n                                      the cluster, the sandboxes service's readiness and what it reports";
 /** The default workload image (D36), pre-pulled with the Runtime image. */
 export declare const DEFAULT_SANDBOX_IMAGE = "python:3.13-slim";
 export interface SandboxDeps {
@@ -14,6 +15,12 @@ export interface SandboxDeps {
     pollMs?: number;
     policyTimeoutMs?: number;
 }
+/**
+ * Waits until the Sandbox CRD is Established. Polled, not `kubectl wait`: a CRD the API server has
+ * not given any conditions yet makes `kubectl wait --for=condition` fail at once instead of waiting
+ * (seen on GKE right after the apply).
+ */
+export declare function waitForCrd(k: Kube, pollMs: number): Promise<void>;
 export interface InstallResult {
     cluster: ClusterFile;
     /** The ServiceAccount token. */
@@ -30,6 +37,7 @@ export declare function installCluster(deps: SandboxDeps, input: {
     tenant: string;
     hostAddress?: string;
     bindAddress?: string;
+    probePorts?: ProbePorts;
     prePull: readonly string[];
     ports: {
         harness: number;

package/dist/sandbox/commands.js

@@ -9,15 +9,15 @@ import { readStackEnv } from "../stack/prepare.js";
 import { apiRoute, readClusterFile, removeClusterFiles, writeClusterFiles, } from "./cluster-file.js";
 import { contexts, kube } from "./kubectl.js";
 import { CONTROLLER_DEPLOYMENT, CONTROLLER_NAMESPACE, CONTROLLER_SHA256, CONTROLLER_URL, CONTROLLER_VERSION, networkPolicy, SANDBOX_API_VERSION, SANDBOX_CRD, tenantLabel, tenantManifests, tenantNamespace, TOKEN_SECRET, } from "./manifests.js";
-import { assertIPv4, runProbe } from "./probe.js";
+import { assertIPv4, parseProbePorts, runProbe } from "./probe.js";
 import { cliTuning } from "../tuning.js";
-export const sandboxUsage = `  sandbox enable --context <name> [--tenant <name>] [--host-address <ip>] [--bind-address <ip>] [--no-pull]
+export const sandboxUsage = `  sandbox enable --context <name> [--tenant <name>] [--host-address <ip>] [--bind-address <ip>] [--probe-ports <allowed>,<denied>] [--no-pull]
                                       run the Tenant's sandboxes as pods on that kubeconfig context (agent-sandbox ${CONTROLLER_VERSION})
   sandbox disable [--tenant <name>] [--delete-namespace]
                                       stop the sandboxes service and forget the cluster
   sandbox status [--tenant <name>] [--json]
                                       the cluster, the sandboxes service's readiness and what it reports`;
-const ENABLE_USAGE = "nylorun local sandbox enable --context <name> [--tenant <name>] [--host-address <ip>] [--bind-address <ip>] [--no-pull]";
+const ENABLE_USAGE = "nylorun local sandbox enable --context <name> [--tenant <name>] [--host-address <ip>] [--bind-address <ip>] [--probe-ports <allowed>,<denied>] [--no-pull]";
 /** The default workload image (D36), pre-pulled with the Runtime image. */
 export const DEFAULT_SANDBOX_IMAGE = "python:3.13-slim";
 /** Preferred host ports for the Harness API, gates and egress (pods reach them). */
@@ -40,7 +40,7 @@ async function readKubeconfig(k) {
  * agent-sandbox: install the pinned release when absent; accept it when the pinned version
  * serves v1beta1; refuse anything else. Never upgrades or uninstalls.
  */
-async function ensureController(k, fetch, err) {
+async function ensureController(k, fetch, err, pollMs) {
     const crd = await k.getJson(["crd", SANDBOX_CRD]);
     if (crd) {
         if (!crd.spec?.versions?.some((v) => v.name === SANDBOX_API_VERSION && v.served))
@@ -61,10 +61,26 @@ async function ensureController(k, fetch, err) {
     if (digest !== CONTROLLER_SHA256)
         throw new CliError(`${CONTROLLER_URL} has sha256 ${digest}, not the pinned ${CONTROLLER_SHA256}; not applying it.`, 1);
     await k.check("Installing agent-sandbox", ["apply", "--server-side", "-f", "-"], { input: manifest });
-    await k.check("Waiting for the Sandbox CRD", ["wait", "--for=condition=Established", `crd/${SANDBOX_CRD}`, `--timeout=${cliTuning().sandbox.crdWaitSeconds}s`]);
+    await waitForCrd(k, pollMs);
     const tuning = cliTuning().sandbox;
     await k.check("Waiting for the agent-sandbox controller", ["rollout", "status", `deployment/${CONTROLLER_DEPLOYMENT}`, "-n", CONTROLLER_NAMESPACE, `--timeout=${tuning.rolloutWaitSeconds}s`], { timeoutMs: tuning.rolloutWaitSeconds * 1000 + tuning.kubectlMarginMs });
 }
+/**
+ * Waits until the Sandbox CRD is Established. Polled, not `kubectl wait`: a CRD the API server has
+ * not given any conditions yet makes `kubectl wait --for=condition` fail at once instead of waiting
+ * (seen on GKE right after the apply).
+ */
+export async function waitForCrd(k, pollMs) {
+    const deadline = Date.now() + cliTuning().sandbox.crdWaitSeconds * 1000;
+    for (;;) {
+        const crd = await k.getJson(["crd", SANDBOX_CRD]);
+        if (crd?.status?.conditions?.some((c) => c.type === "Established" && c.status === "True"))
+            return;
+        if (Date.now() > deadline)
+            throw new CliError(`The ${SANDBOX_CRD} CRD in ${k.context} was not Established within ${cliTuning().sandbox.crdWaitSeconds}s.`, 6);
+        await new Promise((resolve) => setTimeout(resolve, pollMs));
+    }
+}
 /** The ServiceAccount token the token controller writes into the Secret. */
 async function serviceAccountToken(k, namespace, pollMs) {
     const deadline = Date.now() + cliTuning().sandbox.tokenWaitMs;
@@ -93,7 +109,7 @@ export async function installCluster(deps, input) {
     catch (error) {
         throw new CliError(`Context ${input.context}: ${error instanceof Error ? error.message : String(error)}`, 6);
     }
-    await ensureController(k, deps.fetch ?? stack.fetch, stack.err);
+    await ensureController(k, deps.fetch ?? stack.fetch, stack.err, pollMs);
     const namespace = tenantNamespace(input.tenant);
     const existing = await k.getJson(["namespace", namespace]);
     const owner = existing?.metadata?.labels?.["dev.nylorun.tenant"];
@@ -112,6 +128,7 @@ export async function installCluster(deps, input) {
     }, {
         ...(input.hostAddress ? { hostAddress: input.hostAddress } : {}),
         ...(input.bindAddress ? { bindAddress: input.bindAddress } : {}),
+        ...(input.probePorts ? { probePorts: input.probePorts } : {}),
         prePull: input.prePull,
     });
     await k.apply("Applying the namespace's NetworkPolicy", [
@@ -138,7 +155,7 @@ export async function installCluster(deps, input) {
 }
 async function enable(deps, args) {
     const { stack } = deps;
-    const flags = parseStackFlags(args, { booleans: ["--no-pull"], values: ["--context", "--tenant", "--host-address", "--bind-address"] }, ENABLE_USAGE);
+    const flags = parseStackFlags(args, { booleans: ["--no-pull"], values: ["--context", "--tenant", "--host-address", "--bind-address", "--probe-ports"] }, ENABLE_USAGE);
     if (flags.rest.length)
         throw usageError(`Usage: ${ENABLE_USAGE}`);
     const context = flags.values.get("--context");
@@ -150,6 +167,8 @@ async function enable(deps, args) {
         assertIPv4(hostAddress, "--host-address");
     if (bindAddress)
         assertIPv4(bindAddress, "--bind-address");
+    const probePortsFlag = flags.values.get("--probe-ports");
+    const probePorts = probePortsFlag === undefined ? undefined : parseProbePorts(probePortsFlag);
     const name = flags.values.get("--tenant");
     const tenant = await selectTenant(stack, name === undefined ? {} : { name });
     await dockerPreflight(stack.docker);
@@ -169,12 +188,15 @@ async function enable(deps, args) {
         gates: await pick(PREFERRED_PORTS.gates, persisted?.gatesPort),
         egress: await pick(PREFERRED_PORTS.egress, persisted?.egressPort),
     };
+    if (probePorts && [probePorts.allowed, probePorts.denied].some((port) => taken.has(port)))
+        throw usageError(`--probe-ports ${probePortsFlag} overlaps the sandbox ports ${ports.harness}, ${ports.gates} and ${ports.egress}; pick two others.`);
     const images = stackImages(stack.env, { runtime: stack.runtimeVersion, studio: stack.studioVersion });
     const installed = await installCluster(deps, {
         context,
         tenant: tenant.name,
         ...(hostAddress ? { hostAddress } : {}),
         ...(bindAddress ? { bindAddress } : {}),
+        ...(probePorts ? { probePorts } : {}),
         prePull: flags.booleans.has("--no-pull") ? [] : [DEFAULT_SANDBOX_IMAGE, images.runtime],
         ports,
     });

package/dist/sandbox/probe.d.ts

@@ -11,8 +11,9 @@ import type { Kube } from "./kubectl.js";
  *   to `<host>/32` on one port), must reach the allowed host port and nothing else: not the
  *   peer pod, not another port on the host.
  *
- * The host ports are served by a temporary busybox container published on the bind address.
- * The namespace and the container are always removed.
+ * The host ports are served by a temporary busybox container published on the bind address: two
+ * free ports, or `--probe-ports` where a firewall in front of the host (a cloud VPC's) opens only
+ * named ports to the pods. The namespace and the container are always removed.
  */
 export interface ProbeDeps {
     kube: Kube;
@@ -35,10 +36,20 @@ export interface ProbeResult {
 /** Docker Desktop and OrbStack forward the host address to this machine's loopback. */
 export declare function defaultBind(context: string, hostAddress: string): string;
 export declare function assertIPv4(value: string, option: string): string;
+/** The probe's two host ports. */
+export interface ProbePorts {
+    /** The port the probe's policy allows. */
+    allowed: number;
+    /** The port it denies. */
+    denied: number;
+}
+/** `--probe-ports <allowed>,<denied>`: two different TCP ports. */
+export declare function parseProbePorts(value: string): ProbePorts;
 /** The last IPv4 address `nslookup` prints for the name (the first is the DNS server). */
 export declare function parseNslookup(output: string): string | undefined;
 export declare function runProbe(deps: ProbeDeps, options: {
     hostAddress?: string;
     bindAddress?: string;
+    probePorts?: ProbePorts;
     prePull?: readonly string[];
 }): Promise<ProbeResult>;

package/dist/sandbox/probe.js

@@ -15,6 +15,13 @@ export function assertIPv4(value, option) {
         throw new CliError(`${option} must be an IPv4 address: ${value}`, 2);
     return value;
 }
+/** `--probe-ports <allowed>,<denied>`: two different TCP ports. */
+export function parseProbePorts(value) {
+    const ports = value.split(",").map((part) => Number(part.trim()));
+    if (ports.length !== 2 || !ports.every((port) => Number.isInteger(port) && port > 0 && port < 65536) || ports[0] === ports[1])
+        throw new CliError(`--probe-ports must be two different TCP ports, <allowed>,<denied>: ${value}`, 2);
+    return { allowed: ports[0], denied: ports[1] };
+}
 /** The last IPv4 address `nslookup` prints for the name (the first is the DNS server). */
 export function parseNslookup(output) {
     const answer = output.split(/Name:\s*\S+/).slice(1).join("\n");
@@ -59,10 +66,17 @@ export async function runProbe(deps, options) {
                 throw fail("pods cannot resolve host.docker.internal. Pass --host-address with the Docker host's address as pods reach it (172.17.0.1 for kind on Linux).");
         }
         const bindAddress = options.bindAddress ?? defaultBind(kube.context, hostAddress);
-        const allowed = await deps.ports.pickFree();
-        let denied = await deps.ports.pickFree();
-        if (denied === allowed)
+        let allowed;
+        let denied;
+        if (options.probePorts) {
+            ({ allowed, denied } = options.probePorts);
+        }
+        else {
+            allowed = await deps.ports.pickFree();
             denied = await deps.ports.pickFree();
+            if (denied === allowed)
+                denied = await deps.ports.pickFree();
+        }
         const started = await docker.run([
             "run", "--detach", "--rm", "--name", listener, "--label", "dev.nylorun.probe=true",
             "--publish", `${bindAddress}:${allowed}:8080`, "--publish", `${bindAddress}:${denied}:8081`,

package/dist/tuning.d.ts

@@ -55,7 +55,7 @@ export declare const DEFAULT_CLI_TUNING: {
         kubectlMarginMs: number;
         /** Downloading the agent-sandbox manifest. */
         manifestDownloadMs: number;
-        /** `kubectl wait` for the CRD, and `rollout status` of the controller, in seconds. */
+        /** How long the CRD may take to be Established, and `rollout status` of the controller, in seconds. */
         crdWaitSeconds: number;
         rolloutWaitSeconds: number;
         /** How long to wait for the service account's token Secret. */

package/dist/tuning.js

@@ -67,7 +67,7 @@ export const DEFAULT_CLI_TUNING = deepFreeze({
         kubectlMarginMs: 20_000,
         /** Downloading the agent-sandbox manifest. */
         manifestDownloadMs: 60_000,
-        /** `kubectl wait` for the CRD, and `rollout status` of the controller, in seconds. */
+        /** How long the CRD may take to be Established, and `rollout status` of the controller, in seconds. */
         crdWaitSeconds: 60,
         rolloutWaitSeconds: 180,
         /** How long to wait for the service account's token Secret. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nylorun",
-  "version": "0.21.0-beta",
+  "version": "0.22.0-beta",
   "type": "module",
   "license": "Apache-2.0",
   "description": "Set up and run local Nylorun Tenants (Runtime and Studio) with Docker Compose, and act on a Tenant through its Runtime and Management APIs",