hammerkit 1.25.0 → 1.27.0-next.1

package/dist/kubernetes/network-gate.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getNetworkGateContainer = exports.HELPER_SECURITY_CONTEXT = exports.NETWORK_GATE_SCRIPT = exports.NETWORK_GATE_CONTAINER = exports.NETWORK_GATE_TIMEOUT_MS = void 0;
+const egress_proxy_1 = require("../network/egress-proxy");
+// Some network plugins (kind's kindnet among them) enforce a NetworkPolicy a
+// moment after a pod started, so a limited pod's first connections could
+// still get out. The gate is the pod's first init container: it waits until
+// the pod no longer reaches the cluster DNS, which the policy of every limited
+// item denies, so the item's own containers start only once the policy holds.
+// A policy that never takes hold fails the pod instead of running it open.
+// A script for `node -e` in the proxy's image, like the proxy.
+exports.NETWORK_GATE_TIMEOUT_MS = 60000;
+// the name of the gate's init container
+exports.NETWORK_GATE_CONTAINER = 'hammerkit-network';
+// Kept free of anything but Node's own modules.
+exports.NETWORK_GATE_SCRIPT = `
+const fs = require('fs')
+const net = require('net')
+const resolvConf = process.env.HAMMERKIT_GATE_RESOLV_CONF || '/etc/resolv.conf'
+const port = parseInt(process.env.HAMMERKIT_GATE_PORT || '53', 10)
+const deadline = Date.now() + parseInt(process.env.HAMMERKIT_GATE_TIMEOUT_MS || '${exports.NETWORK_GATE_TIMEOUT_MS}', 10)
+function fail(message) {
+  process.stderr.write('hammerkit: ' + message + '\\n')
+  try {
+    fs.writeFileSync('/dev/termination-log', 'hammerkit: ' + message)
+  } catch (e) {}
+  process.exit(1)
+}
+let conf = ''
+try {
+  conf = fs.readFileSync(resolvConf, 'utf8')
+} catch (e) {}
+const match = /^nameserver\\s+(\\S+)/m.exec(conf)
+if (!match) {
+  fail('the pod has no cluster DNS, so its network policy can not be checked')
+}
+const nameserver = match[1]
+function attempt() {
+  const socket = net.connect({ host: nameserver, port, timeout: 2000 })
+  socket.once('connect', () => {
+    socket.destroy()
+    if (Date.now() > deadline) {
+      fail('the network policy of this pod is not enforced, it still reaches ' + nameserver)
+    }
+    setTimeout(attempt, 500)
+  })
+  // dropped or refused: the policy holds
+  socket.once('timeout', () => process.exit(0))
+  socket.once('error', () => process.exit(0))
+}
+attempt()
+`;
+exports.HELPER_SECURITY_CONTEXT = {
+    runAsNonRoot: true,
+    // the image's `node` user
+    runAsUser: 1000,
+    allowPrivilegeEscalation: false,
+    readOnlyRootFilesystem: true,
+    capabilities: { drop: ['ALL'] },
+};
+function getNetworkGateContainer() {
+    return {
+        name: exports.NETWORK_GATE_CONTAINER,
+        image: egress_proxy_1.EGRESS_PROXY_IMAGE,
+        command: ['node', '-e', exports.NETWORK_GATE_SCRIPT],
+        securityContext: exports.HELPER_SECURITY_CONTEXT,
+        resources: { limits: { memory: '64Mi' } },
+    };
+}
+exports.getNetworkGateContainer = getNetworkGateContainer;
+//# sourceMappingURL=network-gate.js.map

package/dist/kubernetes/network-policy.js

@@ -0,0 +1,526 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeSandboxProxies = exports.getProxyReadinessScript = exports.removeAllKubernetesItemNetworks = exports.releaseKubernetesItemNetwork = exports.awaitPodsEnded = exports.removeKubernetesItemNetwork = exports.ensureKubernetesItemNetwork = exports.buildProxyPolicy = exports.buildNetworkPolicy = exports.getNetworkPolicyName = exports.getSandboxProxy = exports.getProxyName = exports.getNetworkLabels = exports.NETWORK_PROXY = exports.NETWORK_LIMITED = exports.NETWORK_LABEL = exports.RUN_LABEL = void 0;
+const crypto_1 = require("crypto");
+const stream_1 = require("stream");
+const readline_1 = require("readline");
+const work_item_1 = require("../planner/work-item");
+const work_service_1 = require("../planner/work-service");
+const work_network_1 = require("../planner/work-network");
+const apply_1 = require("./apply");
+const get_service_ip_1 = require("./get-service-ip");
+const version_1 = require("../version");
+const await_running_state_1 = require("./await-running-state");
+const egress_proxy_1 = require("../network/egress-proxy");
+const log_1 = require("../log");
+const sleep_1 = require("../utils/sleep");
+const network_gate_1 = require("./network-gate");
+const job_placement_1 = require("./job-placement");
+Object.defineProperty(exports, "RUN_LABEL", { enumerable: true, get: function () { return job_placement_1.RUN_LABEL; } });
+const ID_LABEL = 'hammerkit.dev/id';
+const PROXY_LABEL = 'hammerkit.dev/proxy-for';
+const NETWORK_OF_LABEL = 'hammerkit.dev/network-of';
+// on the proxy of a task on a sandbox: the sandbox, whose pod of files it stays with
+const PROXY_OF_SANDBOX_LABEL = 'hammerkit.dev/proxy-of-sandbox';
+const PROXY_READY_TIMEOUT_MS = 60000;
+// how long the pods of an item may take to end before their policy goes
+const POD_END_TIMEOUT_MS = 120000;
+const POD_END_POLL_MS = 500;
+// on the pods of a limited item on a server: its own policy governs their egress, the
+// one of the caller's namespace (see getNamespacePolicies) leaves them out
+// (`limited`), and on its egress proxy (`proxy`), whose ingress only its own policy lets in
+exports.NETWORK_LABEL = 'hammerkit.dev/network';
+exports.NETWORK_LIMITED = 'limited';
+exports.NETWORK_PROXY = 'proxy';
+function getNetworkLabels(env, item) {
+    return env.server && (0, work_network_1.getEffectiveNetwork)(item.data).type !== 'open' ? { [exports.NETWORK_LABEL]: exports.NETWORK_LIMITED } : {};
+}
+exports.getNetworkLabels = getNetworkLabels;
+// A key short enough for a Service name (63 characters) with its prefix. A server's run
+// label is longer than a run id of the CLI, and its first 8 digits tell runs apart.
+function getNetworkKey(item, scope) {
+    return scope.run ? `${item.id().substring(0, 32)}-${scope.run.substring(0, 8)}` : item.id();
+}
+function getProxyName(item, scope) {
+    return `hammerkit-proxy-${getNetworkKey(item, scope)}`;
+}
+exports.getProxyName = getProxyName;
+function getRunProxy(item, scope) {
+    return { key: getNetworkKey(item, scope), name: getProxyName(item, scope) };
+}
+// A proxy that outlives its run is told apart by all it is made of: a run whose task,
+// network, needed services, server or pod of files changed gets one of its own, and the
+// one before goes (removeSandboxProxies).
+function getSandboxProxy(env, item, network, hostAliases, sandbox) {
+    const key = (0, crypto_1.createHash)('sha256')
+        .update(JSON.stringify({
+        sandbox: sandbox.label,
+        pod: sandbox.pod?.uid ?? null,
+        item: item.id(),
+        network,
+        env: (0, egress_proxy_1.getItemProxyEnvironment)(item),
+        hostAliases,
+        placement: env.server ? (0, job_placement_1.getPodPlacement)(env.server, undefined) : null,
+        image: egress_proxy_1.EGRESS_PROXY_IMAGE,
+        version: (0, version_1.getVersion)(),
+    }))
+        .digest('hex')
+        .slice(0, 20);
+    return { key, name: `hammerkit-proxy-${key}`, sandbox };
+}
+exports.getSandboxProxy = getSandboxProxy;
+// What the proxy's objects are labeled with: removed with the run, or, on a sandbox,
+// with the pod of its files.
+function getProxyLabels(env, item, proxy) {
+    return {
+        [NETWORK_OF_LABEL]: item.id(),
+        ...(proxy.sandbox ? { [PROXY_OF_SANDBOX_LABEL]: proxy.sandbox.label } : (0, job_placement_1.getRunLabels)(env)),
+    };
+}
+function getNetworkPolicyName(item, scope) {
+    return `hammerkit-network-${getNetworkKey(item, scope)}`;
+}
+exports.getNetworkPolicyName = getNetworkPolicyName;
+// the pods of the item, of the run when the scope has one
+function getItemPodLabels(item, scope) {
+    return { [ID_LABEL]: item.id(), ...(scope.run ? { [job_placement_1.RUN_LABEL]: scope.run } : {}) };
+}
+function toSelector(labels) {
+    return Object.entries(labels)
+        .map(([key, value]) => `${key}=${value}`)
+        .join(',');
+}
+// Who may connect to a limited item: the items that need it, and for a
+// service a Gateway API route of the environment, the cluster's equivalent
+// of a published port. A task takes no connections.
+function getIngress(env, item) {
+    if (!(0, work_service_1.isContainerWorkService)(item.data)) {
+        return [];
+    }
+    if (env.httpRoutes.some((route) => route.service === item.name)) {
+        return [{}];
+    }
+    const ids = [...new Set(item.requiredBy.map((requirer) => requirer.id()))];
+    if (ids.length === 0) {
+        return [];
+    }
+    const from = { matchExpressions: [{ key: ID_LABEL, operator: 'In', values: ids }] };
+    // the client's name for `from`, a reserved word
+    return [{ _from: [{ podSelector: from }] }];
+}
+// The NetworkPolicy of a limited item: egress to the pods of the services it
+// needs and, under `allow`, to its proxy; nothing else, DNS included (needs
+// and the proxy are host aliases). Ingress only from what needs it. An open
+// item has none.
+function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item, scope)) {
+    const needs = item.needs.filter((need) => (0, work_item_1.isContainerWorkServiceItem)(need.service));
+    return {
+        apiVersion: 'networking.k8s.io/v1',
+        kind: 'NetworkPolicy',
+        metadata: {
+            namespace: env.namespace,
+            name: getNetworkPolicyName(item, scope),
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            labels: { [NETWORK_OF_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) },
+        },
+        spec: {
+            podSelector: { matchLabels: getItemPodLabels(item, scope) },
+            policyTypes: ['Ingress', 'Egress'],
+            ingress: getIngress(env, item),
+            egress: [
+                ...(needs.length > 0
+                    ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: { [ID_LABEL]: need.service.id() } } })) }]
+                    : []),
+                ...(network.type === 'allow'
+                    ? [
+                        {
+                            to: [{ podSelector: { matchLabels: { [PROXY_LABEL]: proxy.key } } }],
+                            ports: (0, egress_proxy_1.getEgressProxyPorts)(network).map((port) => ({ protocol: 'TCP', port })),
+                        },
+                    ]
+                    : []),
+            ],
+        },
+    };
+}
+exports.buildNetworkPolicy = buildNetworkPolicy;
+// The proxy takes connections from its item's pods only, so no other pod
+// uses its allow list or reaches the item's services through it; a proxy that
+// outlives its run, from the pods of the current run only.
+function buildProxyPolicy(env, item, network, scope, proxy = getRunProxy(item, scope)) {
+    return {
+        apiVersion: 'networking.k8s.io/v1',
+        kind: 'NetworkPolicy',
+        metadata: {
+            namespace: env.namespace,
+            name: proxy.name,
+            annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+            labels: getProxyLabels(env, item, proxy),
+            ...(proxy.owner ? { ownerReferences: [proxy.owner] } : {}),
+        },
+        spec: {
+            podSelector: { matchLabels: { [PROXY_LABEL]: proxy.key } },
+            policyTypes: ['Ingress'],
+            ingress: [
+                {
+                    _from: [{ podSelector: { matchLabels: getItemPodLabels(item, scope) } }],
+                    ports: (0, egress_proxy_1.getEgressProxyPorts)(network).map((port) => ({ protocol: 'TCP', port })),
+                },
+            ],
+        },
+    };
+}
+exports.buildProxyPolicy = buildProxyPolicy;
+// What the item's pods are created with, after its policy (and proxy) are in
+// place, so a pod never runs unlimited. A limited item needs an environment
+// that states its cluster enforces NetworkPolicy: hammerkit can't tell from
+// the API, and a policy nobody enforces would run the item open.
+// An open service gets what it had before: whatever an earlier limited run
+// left goes.
+// A task follows the proxy's log while its jobs run. A service outlives the
+// run on a cluster, so its refused hosts are in the proxy's own log.
+async function ensureKubernetesItemNetwork(instance, env, item, options) {
+    const network = (0, work_network_1.getEffectiveNetwork)(item.data);
+    if (network.type !== 'open' && !env.networkPolicyEnforced) {
+        throw new Error(`${item.name}: network ${network.type} needs a cluster that enforces NetworkPolicy, ` +
+            'state it with networkPolicy: enforced on the kubernetes environment');
+    }
+    const hostAliases = await getNeedHostAliases(instance, env, item);
+    if (network.type === 'open') {
+        if (!options.run) {
+            // without the rights to manage NetworkPolicies, hammerkit never made one
+            await removeKubernetesItemNetwork(instance, env, item, { run: null }, { ignoreForbidden: true });
+        }
+        return { env: [], hostAliases, initContainers: [], close() { } };
+    }
+    const proxy = network.type === 'allow' && options.sandbox
+        ? getSandboxProxy(env, item, network, hostAliases, options.sandbox)
+        : getRunProxy(item, options);
+    await (0, apply_1.apply)(instance, buildNetworkPolicy(env, item, network, options, proxy));
+    const initContainers = [(0, network_gate_1.getNetworkGateContainer)()];
+    if (network.type === 'none') {
+        if (options.sandbox) {
+            await removeSandboxProxies(instance, env.namespace, { sandbox: options.sandbox.label, item });
+        }
+        else {
+            await removeProxy(instance, env.namespace, proxy.name, [404]);
+        }
+        return { env: [], hostAliases, initContainers, close() { } };
+    }
+    let started;
+    try {
+        if (proxy.sandbox) {
+            // a proxy of the sandbox from before, made of something else, goes
+            await removeSandboxProxies(instance, env.namespace, { sandbox: proxy.sandbox.label, item }, (deployment) => deployment.metadata?.labels?.[PROXY_LABEL] === proxy.key);
+            proxy.owner = await getFilesPodOwner(instance, env, proxy.sandbox);
+        }
+        started = await ensureProxy(instance, env, item, network, hostAliases, options, proxy);
+    }
+    catch (e) {
+        await removeKubernetesItemNetwork(instance, env, item, options).catch(() => undefined);
+        throw e;
+    }
+    const follow = options.followLog
+        ? await followProxyLog(instance, env, item, proxy, { tailLines: started.reused ? 0 : undefined })
+        : null;
+    return {
+        env: Object.entries((0, egress_proxy_1.getProxyClientEnvironment)(item.needs.map((need) => need.name))).map(([name, value]) => ({
+            name,
+            value,
+        })),
+        // the item reaches each TCP forward under its host's name, at the proxy
+        hostAliases: [
+            ...hostAliases,
+            { ip: started.ip, hostnames: [egress_proxy_1.EGRESS_PROXY_HOST, ...(0, work_network_1.getTcpForwards)(network).map((forward) => forward.host)] },
+        ],
+        initContainers,
+        close: () => follow?.abort(),
+    };
+}
+exports.ensureKubernetesItemNetwork = ensureKubernetesItemNetwork;
+// The policy and proxy of the scope go only once no pod of it runs anymore:
+// a pod that is still terminating (cancel, timeout, `down`) would otherwise
+// run without limits. When pods outlast the wait, the policy stays.
+// The proxy of a task on a sandbox stays while the pod of the sandbox's files
+// runs, which owns it; one it does not own goes with the run.
+async function removeKubernetesItemNetwork(instance, env, item, scope, options = { ignoreForbidden: false }) {
+    const ignored = options.ignoreForbidden ? [403, 404] : [404];
+    await ignoreStatus(instance.objectApi.delete({
+        apiVersion: 'networking.k8s.io/v1',
+        kind: 'NetworkPolicy',
+        metadata: { name: getNetworkPolicyName(item, scope), namespace: env.namespace },
+    }), ignored);
+    if (scope.sandbox) {
+        const owner = await getFilesPodOwner(instance, env, scope.sandbox);
+        await removeSandboxProxies(instance, env.namespace, { sandbox: scope.sandbox.label, item }, (deployment) => (deployment.metadata?.ownerReferences ?? []).some((reference) => !!owner && reference.uid === owner.uid));
+    }
+    else {
+        await removeProxy(instance, env.namespace, getProxyName(item, scope), ignored);
+    }
+}
+exports.removeKubernetesItemNetwork = removeKubernetesItemNetwork;
+// Waits until no pod with the labels runs or waits to run.
+async function awaitPodsEnded(instance, env, labels, timeout = POD_END_TIMEOUT_MS) {
+    const deadline = Date.now() + timeout;
+    for (;;) {
+        const pods = await instance.coreApi.listNamespacedPod({
+            namespace: env.namespace,
+            labelSelector: toSelector(labels),
+        });
+        if (pods.items.every((pod) => pod.status?.phase === 'Succeeded' || pod.status?.phase === 'Failed')) {
+            return true;
+        }
+        if (Date.now() >= deadline) {
+            return false;
+        }
+        await (0, sleep_1.sleep)(POD_END_POLL_MS);
+    }
+}
+exports.awaitPodsEnded = awaitPodsEnded;
+// Removes the scope's policy and proxy once its pods ended.
+async function releaseKubernetesItemNetwork(instance, env, item, scope) {
+    if (!(await awaitPodsEnded(instance, env, getItemPodLabels(item, scope)))) {
+        throw new Error(`pods of ${item.name} are still running, so its network policy stays`);
+    }
+    await removeKubernetesItemNetwork(instance, env, item, scope);
+}
+exports.releaseKubernetesItemNetwork = releaseKubernetesItemNetwork;
+// Every policy and proxy of the item, of any run, once none of its pods runs:
+// what `down` and `clean` remove.
+async function removeAllKubernetesItemNetworks(instance, env, item) {
+    if (!(await awaitPodsEnded(instance, env, { [ID_LABEL]: item.id() }))) {
+        throw new Error(`pods of ${item.name} are still running, so its network policy stays`);
+    }
+    const labelSelector = `${NETWORK_OF_LABEL}=${item.id()}`;
+    let policies;
+    try {
+        const list = await instance.objectApi.list('networking.k8s.io/v1', 'NetworkPolicy', env.namespace, undefined, undefined, undefined, undefined, labelSelector);
+        policies = list.items;
+    }
+    catch (e) {
+        // without the rights to manage NetworkPolicies, hammerkit never made one
+        if ((0, apply_1.statusCodeOf)(e) === 403) {
+            return;
+        }
+        throw e;
+    }
+    // the items of a list carry no kind of their own
+    for (const name of policies.map((policy) => policy.metadata?.name).filter((name) => !!name)) {
+        await ignoreStatus(instance.objectApi.delete({
+            apiVersion: 'networking.k8s.io/v1',
+            kind: 'NetworkPolicy',
+            metadata: { name, namespace: env.namespace },
+        }), [404]);
+    }
+    const deployments = await instance.appsApi.listNamespacedDeployment({ namespace: env.namespace, labelSelector });
+    for (const deployment of deployments.items) {
+        const name = deployment.metadata?.name;
+        if (name) {
+            await ignoreStatus(instance.appsApi.deleteNamespacedDeployment({ name, namespace: env.namespace }), [404]);
+            await ignoreStatus(instance.coreApi.deleteNamespacedService({ name, namespace: env.namespace }), [404]);
+        }
+    }
+}
+exports.removeAllKubernetesItemNetworks = removeAllKubernetesItemNetworks;
+// the services the item needs, by the names it lists them under
+async function getNeedHostAliases(instance, env, item) {
+    const hostAliases = [];
+    for (const need of item.needs) {
+        if (!(0, work_item_1.isContainerWorkServiceItem)(need.service)) {
+            continue;
+        }
+        const ip = await (0, get_service_ip_1.getServiceIp)(instance, env, need.service);
+        if (!ip) {
+            throw new Error(`unable to get service ip for ${need.name}`);
+        }
+        hostAliases.push({ ip, hostnames: [need.name] });
+    }
+    return hostAliases;
+}
+// The proxy as a Deployment and a Service; its pods reach out without an
+// egress policy of their own, and reach the needed services by name for tools
+// that ignore NO_PROXY. Unprivileged, from a pinned image. Its Service carries
+// the port of each TCP forward too.
+// Exits 0 once every port answers on the pod's loopback.
+function getProxyReadinessScript(ports) {
+    return (`const net = require('net'); let open = ${ports.length}; ` +
+        `for (const port of ${JSON.stringify(ports)}) net.connect(port, '127.0.0.1')` +
+        `.on('connect', function () { this.destroy(); if (--open === 0) process.exit(0) })` +
+        `.on('error', () => process.exit(1))`);
+}
+exports.getProxyReadinessScript = getProxyReadinessScript;
+async function ensureProxy(instance, env, item, network, hostAliases, scope, proxy) {
+    const name = proxy.name;
+    const labels = { [PROXY_LABEL]: proxy.key };
+    const metadata = {
+        namespace: env.namespace,
+        name,
+        annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
+        labels: { ...labels, ...getProxyLabels(env, item, proxy) },
+        ...(proxy.owner ? { ownerReferences: [proxy.owner] } : {}),
+    };
+    // on a server the proxy runs on the nodes of the first pool
+    const placement = env.server ? (0, job_placement_1.getPodPlacement)(env.server, undefined) : null;
+    const deployment = {
+        apiVersion: 'apps/v1',
+        kind: 'Deployment',
+        metadata,
+        spec: {
+            replicas: 1,
+            selector: { matchLabels: labels },
+            template: {
+                metadata: {
+                    labels: {
+                        ...labels,
+                        ...(proxy.sandbox ? {} : (0, job_placement_1.getRunLabels)(env)),
+                        ...(env.server ? { [exports.NETWORK_LABEL]: exports.NETWORK_PROXY } : {}),
+                    },
+                },
+                spec: {
+                    hostAliases,
+                    automountServiceAccountToken: false,
+                    ...(placement ? { nodeSelector: placement.nodeSelector, tolerations: placement.tolerations } : {}),
+                    // unprivileged, it still listens on the port of a TCP forward below
+                    // 1024; a sysctl every Pod Security level allows
+                    securityContext: { sysctls: [{ name: 'net.ipv4.ip_unprivileged_port_start', value: '0' }] },
+                    containers: [
+                        {
+                            name: 'proxy',
+                            image: egress_proxy_1.EGRESS_PROXY_IMAGE,
+                            command: (0, egress_proxy_1.getEgressProxyCommand)(),
+                            env: Object.entries((0, egress_proxy_1.getItemProxyEnvironment)(item)).map(([key, value]) => ({ name: key, value })),
+                            ports: (0, egress_proxy_1.getEgressProxyPorts)(network).map((port) => ({ containerPort: port })),
+                            // in the pod, since its ingress policy may keep the kubelet out;
+                            // ready once it listens on every port, its TCP forwards' too
+                            readinessProbe: {
+                                exec: {
+                                    command: ['node', '-e', getProxyReadinessScript((0, egress_proxy_1.getEgressProxyPorts)(network))],
+                                },
+                                periodSeconds: 2,
+                            },
+                            securityContext: network_gate_1.HELPER_SECURITY_CONTEXT,
+                            resources: { limits: { memory: '256Mi' } },
+                        },
+                    ],
+                },
+            },
+        },
+    };
+    const service = {
+        apiVersion: 'v1',
+        kind: 'Service',
+        metadata,
+        spec: {
+            type: 'ClusterIP',
+            selector: labels,
+            ports: (0, egress_proxy_1.getEgressProxyPorts)(network).map((port) => ({ name: `tcp-${port}`, port, targetPort: port })),
+        },
+    };
+    // on a sandbox, the proxy of an earlier run may still be there: the item's policy
+    // of this run lets its pods reach it, its own policy lets them in
+    const reused = proxy.sandbox
+        ? await instance.objectApi.read(deployment).then((current) => !current.metadata?.deletionTimestamp, (e) => ((0, apply_1.statusCodeOf)(e) === 404 ? false : Promise.reject(e)))
+        : false;
+    await (0, apply_1.apply)(instance, buildProxyPolicy(env, item, network, scope, proxy));
+    await (0, apply_1.apply)(instance, deployment);
+    const created = await (0, apply_1.apply)(instance, service);
+    await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
+        podSelector: toSelector(labels),
+        timeout: PROXY_READY_TIMEOUT_MS,
+        timeoutMessage: `the egress proxy of ${item.name} did not start`,
+    });
+    const ip = created.spec?.clusterIP;
+    if (!ip) {
+        throw new Error(`the egress proxy of ${item.name} has no cluster ip`);
+    }
+    item.status.write('debug', `${reused ? 'egress proxy of the sandbox' : 'egress proxy'} allows ${network.hosts.join(', ') || 'no host'}`);
+    return { ip, reused };
+}
+// Refused hosts show up in the item's log, as on Docker; of a proxy from an
+// earlier run (`tailLines: 0`), only what this run asks of it.
+async function followProxyLog(instance, env, item, proxy, options) {
+    try {
+        const pods = await instance.coreApi.listNamespacedPod({
+            namespace: env.namespace,
+            labelSelector: `${PROXY_LABEL}=${proxy.key}`,
+        });
+        const pod = pods.items.find((p) => p.status?.phase === 'Running' && !p.metadata?.deletionTimestamp);
+        if (!pod?.metadata?.name) {
+            return null;
+        }
+        const output = new stream_1.PassThrough();
+        // close() aborts the request, which errors the response stream the
+        // client pipes in here; nothing listens to it there
+        output.on('pipe', (source) => source.on('error', () => undefined));
+        (0, readline_1.createInterface)({ input: output }).on('line', (line) => {
+            const event = (0, egress_proxy_1.parseEgressProxyEvent)(line);
+            if (event && event.type !== 'ready') {
+                item.status.write('warn', (0, egress_proxy_1.formatEgressProxyEvent)(event));
+            }
+        });
+        return await instance.log.log(env.namespace, pod.metadata.name, 'proxy', output, {
+            follow: true,
+            ...(options.tailLines !== undefined ? { tailLines: options.tailLines } : {}),
+        });
+    }
+    catch (e) {
+        item.status.write('debug', `unable to follow the egress proxy log: ${(0, log_1.getErrorMessage)(e)}`);
+        return null;
+    }
+}
+async function removeProxy(instance, namespace, name, ignored) {
+    await ignoreStatus(instance.appsApi.deleteNamespacedDeployment({ name, namespace }), ignored);
+    await ignoreStatus(instance.coreApi.deleteNamespacedService({ name, namespace }), ignored);
+    await ignoreStatus(instance.objectApi.delete({
+        apiVersion: 'networking.k8s.io/v1',
+        kind: 'NetworkPolicy',
+        metadata: { name, namespace },
+    }), ignored);
+}
+// The proxies on the sandbox, of the item when one is given, but those `keep` takes.
+// The cluster removes those the pod of the sandbox's files owns when the pod goes; the
+// server removes all when it took the pod away while a run worked (see
+// KubernetesSandboxStorage.makeWayForRun).
+async function removeSandboxProxies(instance, namespace, of, keep = () => false) {
+    const deployments = await instance.appsApi.listNamespacedDeployment({
+        namespace,
+        labelSelector: [
+            `${PROXY_OF_SANDBOX_LABEL}=${of.sandbox}`,
+            ...(of.item ? [`${NETWORK_OF_LABEL}=${of.item.id()}`] : []),
+        ].join(','),
+    });
+    for (const deployment of deployments.items) {
+        const name = deployment.metadata?.name;
+        if (name && !keep(deployment)) {
+            await removeProxy(instance, namespace, name, [404]);
+        }
+    }
+}
+exports.removeSandboxProxies = removeSandboxProxies;
+// The pod of the sandbox's files the run holds, as the owner of the proxy: the cluster
+// removes the proxy when that pod goes. None when the pod is no longer the one the run
+// holds (it made way for a pod of the run): the proxy then goes with the run.
+async function getFilesPodOwner(instance, env, sandbox) {
+    if (!sandbox.pod) {
+        return undefined;
+    }
+    const { name, uid } = sandbox.pod;
+    const current = await instance.coreApi
+        .readNamespacedPod({ name, namespace: env.namespace })
+        .catch((e) => ((0, apply_1.statusCodeOf)(e) === 404 ? undefined : Promise.reject(e)));
+    if (!current || current.metadata?.uid !== uid || current.metadata?.deletionTimestamp) {
+        return undefined;
+    }
+    return { apiVersion: 'v1', kind: 'Pod', name, uid };
+}
+async function ignoreStatus(promise, statusCodes) {
+    try {
+        await promise;
+    }
+    catch (e) {
+        if (!statusCodes.includes((0, apply_1.statusCodeOf)(e) ?? 0)) {
+            throw e;
+        }
+    }
+}
+//# sourceMappingURL=network-policy.js.map

package/dist/kubernetes/resources.js

@@ -1,12 +1,23 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.removeInvalidCharacters = exports.getResourceName = void 0;
+// An object name is a DNS label: lowercase letters, digits and `-`, at most 63
+// characters, starting and ending with a letter or digit. The id makes a name
+// unique; the item's name is there for people and is cut to fit.
+const MAX_NAME = 63;
 function getResourceName(item, suffix) {
-    return removeInvalidCharacters(item.name) + '-' + item.id() + (suffix ?? '');
+    const tail = `-${item.id()}${suffix ?? ''}`;
+    const name = removeInvalidCharacters(item.name)
+        .slice(0, Math.max(0, MAX_NAME - tail.length))
+        .replace(/-+$/, '');
+    return name ? name + tail : tail.slice(1);
 }
 exports.getResourceName = getResourceName;
 function removeInvalidCharacters(val) {
-    return val.replace(/:/g, '-').replace(/\./g, '-');
+    return val
+        .toLowerCase()
+        .replace(/[^a-z0-9-]+/g, '-')
+        .replace(/^-+/, '');
 }
 exports.removeInvalidCharacters = removeInvalidCharacters;
 //# sourceMappingURL=resources.js.map

package/dist/kubernetes/restore-kubernetes-data.js

@@ -6,16 +6,27 @@ const log_1 = require("../log");
 const path_1 = require("path");
 const volumes_1 = require("./volumes");
 const ensure_persistent_data_1 = require("./ensure-persistent-data");
-async function restoreKubernetesData(work, kubernetes, instance, environment, path) {
-    const persistence = await (0, volumes_1.getKubernetesPersistence)(work);
+async function restoreKubernetesData(work, kubernetes, instance, environment, path, 
+// the kept claim of the work holds what an earlier run left, which goes first
+options = {}) {
+    const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes);
     await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'write', async (name) => {
+        if (options.reset) {
+            await (0, ensure_persistent_data_1.clearClaim)(instance, kubernetes, work, name);
+        }
         for (const generatedArchive of (0, event_cache_1.getArchivePaths)(work.data, path)) {
             if (!(await environment.file.exists(generatedArchive.filename))) {
                 continue;
             }
             const targetPath = `/dev/hammerkit/${work.id()}/${(0, path_1.basename)(generatedArchive.path)}`;
-            const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', ['tar', 'xzf', '-', '-C', targetPath], process.stdout, process.stderr, environment.file.readStream(generatedArchive.filename), false, (status) => {
-                work.status.write('debug', `upload ${generatedArchive.filename} to ${name}: ${status}`);
+            // a claim made for this run (a server removes the claims of a run when it
+            // ends) does not have the directory of the output yet
+            let failure = null;
+            const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', ['sh', '-c', 'mkdir -p "$1" && tar xzf - -C "$1"', 'sh', targetPath], process.stdout, process.stderr, environment.file.readStream(generatedArchive.filename), false, (status) => {
+                work.status.write('debug', `upload ${generatedArchive.filename} to ${name}: ${status.status}`);
+                if (status.status !== 'Success') {
+                    failure = status.message ?? status.reason ?? 'unknown';
+                }
             });
             await new Promise((resolve, reject) => {
                 try {
@@ -27,6 +38,10 @@ async function restoreKubernetesData(work, kubernetes, instance, environment, pa
                         reject(err);
                     });
                     res.on('close', () => {
+                        if (failure) {
+                            reject(new Error(`unable to restore ${generatedArchive.path} into ${name}: ${failure}`));
+                            return;
+                        }
                         work.status.write('debug', `uploaded ${generatedArchive.filename} to ${name}:${targetPath}`);
                         resolve();
                     });