hammerkit 1.26.0 → 1.27.0-next.1

package/dist/planner/service-definition.js

@@ -4,6 +4,7 @@ exports.getServiceDefinitionHash = void 0;
 const crypto_1 = require("crypto");
 const work_secret_1 = require("./work-secret");
 const work_service_1 = require("./work-service");
+const work_network_1 = require("./work-network");
 const work_cache_id_1 = require("./work-cache-id");
 // Everything a running container of the service was created from, plus the
 // definitions of the services it needs: a dependent links to the container it
@@ -12,6 +13,7 @@ const work_cache_id_1 = require("./work-cache-id");
 // service whose definition changed instead of silently reusing it.
 function getServiceDefinitionHash(item) {
     const service = item.data;
+    const network = (0, work_network_1.getEffectiveNetwork)(service);
     const definition = {
         image: service.image,
         cmd: service.cmd?.cmd ?? null,
@@ -38,6 +40,9 @@ function getServiceDefinitionHash(item) {
             .map((need) => `${need.name}=${getServiceDefinitionHash(need.service)}`),
         // only when declared, so a service without keeps its hash
         ...(service.resources ? { resources: service.resources } : {}),
+        // in this run, under its `--network` ceiling: a service started open is
+        // recreated for a run that limits it
+        ...((0, work_network_1.isLimitedNetwork)(network) ? { network } : {}),
     };
     return (0, crypto_1.createHash)('sha1').update(JSON.stringify(definition)).digest('hex');
 }

package/dist/planner/utils/append-work-environment.js

@@ -15,6 +15,8 @@ function appendWorkEnvironment(referenced) {
             kubeConfig: referenced.schema.kubernetes.kubeconfig,
             httpRoutes: referenced.schema.kubernetes.httpRoutes || [],
             namespace: referenced.schema.kubernetes.namespace ?? 'default',
+            networkPolicyEnforced: referenced.schema.kubernetes.networkPolicy === 'enforced',
+            ...(referenced.schema.kubernetes.volumeSize ? { volumeSize: referenced.schema.kubernetes.volumeSize } : {}),
         };
     }
 }

package/dist/planner/utils/append-work-service.js

@@ -23,6 +23,7 @@ const resolve_cache_1 = require("../../cache/resolve-cache");
 const units_1 = require("../../utils/units");
 const log_1 = require("../../log");
 const work_resources_1 = require("../work-resources");
+const work_network_1 = require("../work-network");
 function appendWorkService(workTree, service, environment, context) {
     const workService = parseService(workTree, service, environment, context);
     if (!workTree.services[workService.name]) {
@@ -116,6 +117,9 @@ function parseService(workTree, service, environment, context) {
             init: null,
             secrets: (0, work_secret_1.parseWorkSecrets)(service.cwd, context.projectRoot, service.schema.secrets, envs, environment, context.secrets, service.schema.account),
             resources: (0, work_resources_1.parseWorkResources)(service.schema.resources),
+            volumeSize: service.schema.volumeSize ?? null,
+            network: (0, work_network_1.parseWorkNetwork)(service.schema.network),
+            networkCeiling: null,
         };
     }
 }

package/dist/planner/utils/append-work-task.js

@@ -19,8 +19,17 @@ const get_work_runtime_1 = require("./get-work-runtime");
 const resolve_cache_1 = require("../../cache/resolve-cache");
 const units_1 = require("../../utils/units");
 const work_resources_1 = require("../work-resources");
+const job_placement_1 = require("../../kubernetes/job-placement");
+const work_network_1 = require("../work-network");
 function appendWorkTask(workTree, cwd, referenceTask, environment, context) {
-    const task = parseTask(cwd, referenceTask, environment, context);
+    const parsed = parseTask(cwd, referenceTask, environment, context);
+    // on a server's cluster a local task runs in a tool image like a container task
+    const server = workTree.environment.type === 'kubernetes' ? workTree.environment.server : undefined;
+    const task = !server
+        ? parsed
+        : parsed.type === 'local-task'
+            ? (0, job_placement_1.runInToolImage)(parsed, server, referenceTask.schema.platform)
+            : (0, job_placement_1.placeOnPool)(parsed, server);
     if (!workTree.tasks[task.name]) {
         const workItem = {
             // A dependency cycle is rejected before anything runs (checkForLoop), but
@@ -72,6 +81,8 @@ function parseTask(cwd, task, environment, context) {
         timeout: task.schema.timeout ? (0, units_1.parseDuration)(task.schema.timeout) : null,
         secrets: (0, work_secret_1.parseWorkSecrets)(cwd, context.projectRoot, task.schema.secrets, envs, environment, context.secrets, task.schema.account),
         resources: (0, work_resources_1.parseWorkResources)(task.schema.resources),
+        network: (0, work_network_1.parseWorkNetwork)(task.schema.network),
+        networkCeiling: null,
         runner: task.schema.runner ?? task.scope.schema.runner ?? null,
     };
     if ((0, build_file_task_schema_1.isBuildFileContainerTaskSchema)(task.schema)) {
@@ -81,6 +92,7 @@ function parseTask(cwd, task, environment, context) {
             user: (0, get_container_user_1.getContainerUser)(),
             image: (0, template_value_1.templateValue)(task.schema.image, envs),
             mounts: (0, parse_work_mounts_1.parseWorkMounts)(cwd, task.schema, envs),
+            volumeSize: task.schema.volumeSize ?? null,
         };
     }
     else {
@@ -88,6 +100,10 @@ function parseTask(cwd, task, environment, context) {
         if (fileSecret) {
             throw new Error(`task ${task.relativeName}: a local task takes secrets as env only (${(0, work_secret_1.getSecretName)(fileSecret)})`);
         }
+        if (baseWorkNode.network.type !== 'open') {
+            throw new Error(`task ${task.relativeName}: network ${(0, work_network_1.formatNetwork)(baseWorkNode.network)} needs a container, ` +
+                `hammerkit can't limit the network of a local task`);
+        }
         return {
             ...baseWorkNode,
             type: 'local-task',

package/dist/planner/utils/assert-item-paths.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assertWorkTreePathsWithinRoot = exports.assertItemPathsWithinRoot = void 0;
+const within_root_1 = require("../../file/within-root");
+const work_service_1 = require("../work-service");
+const work_task_1 = require("../work-task");
+const hosted_volumes_1 = require("./hosted-volumes");
+/**
+ * Refuses an item whose sources, outputs or mounts leave the root of a run, or go
+ * through a symbolic link in it. The server runs build files it did not write, in a
+ * checkout it did not write either: a `src: ../..` or a link in the checkout would
+ * have the server mount or upload its own files (its credentials, other callers'
+ * caches) into a task. Checked when the run is planned and again before each item
+ * runs, since an earlier task may have left a link behind.
+ */
+async function assertItemPathsWithinRoot(item, root) {
+    const data = item.data;
+    for (const source of 'src' in data ? data.src : []) {
+        if (!source.inherited) {
+            await (0, within_root_1.assertWithinRoot)(root, source.absolutePath, `${item.name}: src ${source.source}`);
+        }
+    }
+    if ((0, work_task_1.isContainerWorkTask)(data) || (0, work_service_1.isContainerWorkService)(data)) {
+        for (const mount of data.mounts) {
+            await (0, within_root_1.assertWithinRoot)(root, mount.localPath, `${item.name}: mount ${mount.localPath}`);
+        }
+    }
+    if ((0, work_service_1.isContainerWorkService)(data)) {
+        for (const volume of data.volumes) {
+            if (!(0, hosted_volumes_1.isVolumeName)(volume.name)) {
+                throw new Error(`${item.name}: volume ${volume.name} is not a volume name, a path of the server can't be mounted`);
+            }
+        }
+    }
+    if ((0, work_task_1.isWorkTask)(data)) {
+        for (const generate of data.generates) {
+            if (!generate.inherited) {
+                await (0, within_root_1.assertWithinRoot)(root, generate.path, `${item.name}: generates ${generate.path}`);
+            }
+        }
+    }
+}
+exports.assertItemPathsWithinRoot = assertItemPathsWithinRoot;
+async function assertWorkTreePathsWithinRoot(workTree, root) {
+    for (const item of [...Object.values(workTree.tasks), ...Object.values(workTree.services)]) {
+        await assertItemPathsWithinRoot(item, root);
+    }
+}
+exports.assertWorkTreePathsWithinRoot = assertWorkTreePathsWithinRoot;
+//# sourceMappingURL=assert-item-paths.js.map

package/dist/planner/utils/hosted-volumes.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scopeServiceVolumes = exports.isVolumeName = void 0;
+const work_service_1 = require("../work-service");
+// What Docker takes as the name of a volume. Anything else in a service's `volumes` is a
+// path of the host to Docker (`/var/run/docker.sock:/run/docker.sock` binds the socket).
+const VOLUME_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]+$/;
+function isVolumeName(name) {
+    return VOLUME_NAME.test(name);
+}
+exports.isVolumeName = isVolumeName;
+/**
+ * On a server, the docker volumes of a run's services are its caller's: each name gets the
+ * run's scope (its caller and source trust) in front, so a build file can neither open nor,
+ * with `resetOnChange`, remove a volume of another caller, of a sandbox or of the host.
+ * The outputs a service inherits from its deps keep their names: those are made from paths
+ * in the run's own files, and the task that writes them uses the same name. A Kubernetes
+ * backend keeps claims in the caller's namespace and is left as it is.
+ */
+function scopeServiceVolumes(workTree, scope) {
+    const scoped = new Set();
+    for (const service of Object.values(workTree.services)) {
+        if (!(0, work_service_1.isContainerWorkService)(service.data)) {
+            continue;
+        }
+        for (const volume of service.data.volumes) {
+            if (!volume.inherited && !scoped.has(volume)) {
+                scoped.add(volume);
+                volume.name = `hammerkit-${scope}-${volume.name}`;
+            }
+        }
+    }
+}
+exports.scopeServiceVolumes = scopeServiceVolumes;
+//# sourceMappingURL=hosted-volumes.js.map

package/dist/planner/work-network.js

@@ -0,0 +1,193 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.applyNetworkCeiling = exports.formatNetwork = exports.getTcpForwards = exports.matchesHost = exports.isLimitedNetwork = exports.toNetworkSchema = exports.exceedsNetwork = exports.getEffectiveNetwork = exports.limitNetwork = exports.parseNetworkOption = exports.parseWorkNetwork = exports.OPEN_NETWORK = void 0;
+const network_schema_1 = require("../schema/network-schema");
+exports.OPEN_NETWORK = { type: 'open' };
+function parseWorkNetwork(schema) {
+    if (!schema || schema === 'open') {
+        return exports.OPEN_NETWORK;
+    }
+    if (schema === 'none') {
+        return { type: 'none' };
+    }
+    return { type: 'allow', hosts: normalizeHosts(schema.allow) };
+}
+exports.parseWorkNetwork = parseWorkNetwork;
+// `--network none`, `--network open` or `--network allow:<host>,<host:port>`
+function parseNetworkOption(value) {
+    if (value === 'open' || value === 'none') {
+        return { type: value };
+    }
+    if (value.startsWith('allow:')) {
+        const hosts = value
+            .substring('allow:'.length)
+            .split(',')
+            .map((host) => host.trim())
+            .filter((host) => host.length > 0);
+        const invalid = hosts.find((host) => !(0, network_schema_1.isAllowEntry)(host));
+        if (invalid) {
+            throw new Error(`invalid host "${invalid}", expected a name such as registry.npmjs.org or *.example.com, ` +
+                'or a name and port such as db.example.com:5432');
+        }
+        const error = (0, network_schema_1.getAllowListError)(hosts);
+        if (error) {
+            throw new Error(error);
+        }
+        return { type: 'allow', hosts: normalizeHosts(hosts) };
+    }
+    throw new Error(`invalid network "${value}", expected none, open or allow:<host>,<host>`);
+}
+exports.parseNetworkOption = parseNetworkOption;
+// The setting under a ceiling: a more open setting is clamped down to it, and
+// an allow list under an allow ceiling keeps the hosts both lists allow. A
+// name one list allows only through a pattern reaches no private address.
+function limitNetwork(network, ceiling) {
+    if (!ceiling || ceiling.type === 'open' || network.type === 'none') {
+        return network;
+    }
+    if (ceiling.type === 'none') {
+        return ceiling;
+    }
+    if (network.type === 'open') {
+        return ceiling;
+    }
+    const hosts = [
+        ...network.hosts.filter((host) => ceiling.hosts.some((limit) => coversHost(limit, host))),
+        ...ceiling.hosts.filter((limit) => network.hosts.some((host) => coversHost(host, limit))),
+    ];
+    const normalized = normalizeHosts(hosts);
+    const publicOnly = normalized.filter((host) => !host.startsWith('*.') && !(coversByName(network, host) && coversByName(ceiling, host)));
+    return publicOnly.length > 0 ? { type: 'allow', hosts: normalized, publicOnly } : { type: 'allow', hosts: normalized };
+}
+exports.limitNetwork = limitNetwork;
+// Whether a name (not a pattern) of the list covers `host` and may reach
+// private addresses.
+function coversByName(network, host) {
+    return network.hosts.some((entry) => !entry.startsWith('*.') && !network.publicOnly?.includes(entry) && coversHost(entry, host));
+}
+// The network of a task or service in this run: its own setting under the
+// run's ceiling (`--network`).
+function getEffectiveNetwork(item) {
+    return limitNetwork(item.network, item.networkCeiling);
+}
+exports.getEffectiveNetwork = getEffectiveNetwork;
+// Whether `network` reaches more than `ceiling` allows: a host it allows that no
+// host of the ceiling covers. Redundant hosts of the ceiling don't count.
+function exceedsNetwork(network, ceiling) {
+    if (ceiling.type === 'open' || network.type === 'none') {
+        return false;
+    }
+    if (network.type === 'open') {
+        return true;
+    }
+    if (ceiling.type === 'none') {
+        return network.hosts.length > 0;
+    }
+    return network.hosts.some((host) => !ceiling.hosts.some((limit) => coversHost(limit, host)));
+}
+exports.exceedsNetwork = exceedsNetwork;
+// The setting as a build file writes it.
+function toNetworkSchema(network) {
+    return network.type === 'allow' ? { allow: [...network.hosts] } : network.type;
+}
+exports.toNetworkSchema = toNetworkSchema;
+function isLimitedNetwork(network) {
+    return network.type !== 'open';
+}
+exports.isLimitedNetwork = isLimitedNetwork;
+// Whether a request to `host` passes `pattern`: the same name, or a name under
+// the domain of a `*.` pattern (the domain itself is not included).
+function matchesHost(pattern, host) {
+    const name = host.toLowerCase().replace(/\.$/, '');
+    const rule = pattern.toLowerCase();
+    return rule.startsWith('*.') ? name.endsWith(rule.substring(1)) : name === rule;
+}
+exports.matchesHost = matchesHost;
+// Whether everything `host` (a name, a `*.` pattern or a name and port)
+// allows, `limit` allows too. A host without a port reaches every port of it
+// through the proxy, so it covers a forward to one of them; a forward covers
+// only itself.
+function coversHost(limit, host) {
+    const limitEntry = (0, network_schema_1.parseAllowEntry)(limit);
+    const hostEntry = (0, network_schema_1.parseAllowEntry)(host);
+    if (!limitEntry || !hostEntry) {
+        return false;
+    }
+    if (limitEntry.port !== null) {
+        return hostEntry.port === limitEntry.port && matchesHost(limitEntry.host, hostEntry.host);
+    }
+    if (hostEntry.host.startsWith('*.')) {
+        // `*.b.example.com` is within `*.example.com`
+        return (limitEntry.host.startsWith('*.') &&
+            hostEntry.host.toLowerCase().endsWith(limitEntry.host.toLowerCase().substring(1)));
+    }
+    return matchesHost(limitEntry.host, hostEntry.host);
+}
+// The raw TCP forwards of a network: the entries of `allow` with a port.
+function getTcpForwards(network) {
+    if (network.type !== 'allow') {
+        return [];
+    }
+    return network.hosts
+        .map(network_schema_1.parseAllowEntry)
+        .filter((entry) => entry !== null && entry.port !== null);
+}
+exports.getTcpForwards = getTcpForwards;
+function normalizeHosts(hosts) {
+    return [...new Set(hosts.map((host) => host.toLowerCase()))].sort();
+}
+function formatNetwork(network) {
+    if (network.type === 'allow') {
+        return network.hosts.length > 0 ? `allow ${network.hosts.join(', ')}` : 'allow (no hosts)';
+    }
+    return network.type;
+}
+exports.formatNetwork = formatNetwork;
+// Sets the run's ceiling on every task and service of the work tree. A local
+// task can't be limited, so a ceiling below open with one in the run is an
+// error before anything runs. So is a limited item that needs a service under
+// a name that is no host name: the proxy passes the names of the needed
+// services, and `*.com` would pass every .com host. A TCP forward under the
+// name of a needed service would hide that service, so it is an error too, as
+// are two forwards on one port, which the ceiling can leave from two lists.
+function applyNetworkCeiling(workTree, ceiling) {
+    const limit = ceiling && isLimitedNetwork(ceiling) ? ceiling : null;
+    if (limit) {
+        const local = Object.values(workTree.tasks).find((task) => task.data.type === 'local-task');
+        if (local) {
+            throw new Error(`--network ${formatNetwork(limit)} needs every task in a container, ` +
+                `hammerkit can't limit the network of the local task ${local.name}`);
+        }
+    }
+    for (const task of Object.values(workTree.tasks)) {
+        task.data.networkCeiling = limit;
+    }
+    for (const service of Object.values(workTree.services)) {
+        if (service.data.type === 'container-service') {
+            service.data.networkCeiling = limit;
+        }
+    }
+    for (const item of [...Object.values(workTree.tasks), ...Object.values(workTree.services)]) {
+        if (item.data.type === 'local-task' || item.data.type === 'kubernetes-service') {
+            continue;
+        }
+        const network = getEffectiveNetwork(item.data);
+        const invalid = item.needs.find((need) => !(0, network_schema_1.isNeedName)(need.name));
+        if (isLimitedNetwork(network) && invalid) {
+            throw new Error(`${item.name}: network ${formatNetwork(network)} reaches a needed service by a host name, ` +
+                `"${invalid.name}" is none (name it like db or api-v2)`);
+        }
+        const shared = network.type === 'allow' ? (0, network_schema_1.getAllowListError)(network.hosts) : null;
+        if (shared) {
+            const under = limit ? ` under --network ${formatNetwork(limit)}` : '';
+            throw new Error(`${item.name}: network ${formatNetwork(network)}${under}, ${shared}`);
+        }
+        const hidden = getTcpForwards(network).find((forward) => item.needs.some((need) => need.name.toLowerCase() === forward.host.toLowerCase()));
+        if (hidden) {
+            throw new Error(`${item.name}: network ${formatNetwork(network)} forwards ${hidden.host}:${hidden.port}, ` +
+                `which is the name of a service it needs`);
+        }
+    }
+}
+exports.applyNetworkCeiling = applyNetworkCeiling;
+//# sourceMappingURL=work-network.js.map

package/dist/planner/work-runtime-docker.js

@@ -17,6 +17,10 @@ const work_instance_id_1 = require("./work-instance-id");
 const stream_1 = require("stream");
 const service_definition_1 = require("./service-definition");
 const container_task_state_1 = require("../executer/container-task-state");
+const item_network_1 = require("../docker/item-network");
+const execution_steps_1 = require("../executer/execution-steps");
+const assert_item_paths_1 = require("./utils/assert-item-paths");
+const log_1 = require("../log");
 function dockerTaskRuntime(task, workEnvironment) {
     const docker = (0, execute_docker_1.getContainerCli)(workEnvironment);
     return {
@@ -56,8 +60,18 @@ function dockerTaskRuntime(task, workEnvironment) {
             for (const container of containers) {
                 await (0, remove_container_1.removeContainer)(docker.getContainer(container.Id));
             }
+            await (0, item_network_1.removeItemNetworks)(docker, task);
         },
         async execute(environment, options) {
+            if (environment.hosted) {
+                try {
+                    await (0, assert_item_paths_1.assertItemPathsWithinRoot)(task, environment.cwd);
+                }
+                catch (e) {
+                    options.state.set({ stateKey: options.stateKey, type: 'error', errorMessage: (0, log_1.getErrorMessage)(e) });
+                    return;
+                }
+            }
             await (0, docker_task_1.dockerTask)(docker, task, environment, options);
         },
         async remove(environment) {
@@ -103,6 +117,18 @@ function isOnHost(generate) {
     return generate.isFile || generate.export;
 }
 async function restoreContainer(docker, environment, item, path) {
+    await (0, execution_steps_1.refuseHostLinks)(item, environment);
+    // on a server the volumes the outputs are restored into carry the run's files, so they
+    // are removed with a sandbox
+    const root = (0, execution_steps_1.hostedRoot)(environment);
+    if (root) {
+        const names = item.data.type === 'container-service'
+            ? item.data.volumes.filter((v) => !v.inherited).map((v) => v.name)
+            : item.data.generates.filter((v) => !v.inherited && !v.isFile).map((v) => v.volumeName);
+        for (const name of names) {
+            await (0, get_docker_executor_1.ensureVolumeExists)(docker, item.status, name, (0, get_docker_executor_1.volumeLabels)(root));
+        }
+    }
     await (0, using_container_1.usingContainer)(docker, item, {
         abortSignal: environment.abortCtrl.signal,
         Image: item.data.image,
@@ -142,6 +168,7 @@ async function restoreContainer(docker, environment, item, path) {
         }
     });
     if (item.data.type === 'container-task') {
+        await (0, execution_steps_1.refuseHostLinks)(item, environment);
         const onHost = new Set(item.data.generates.filter((g) => !g.inherited && isOnHost(g)).map((g) => g.path));
         for (const generate of (0, event_cache_1.getArchivePaths)(item.data, path)) {
             if (onHost.has(generate.path) && (await environment.file.exists(generate.filename))) {
@@ -167,6 +194,8 @@ function dockerServiceRuntime(service, workEnvironment) {
             if (currentService.Labels['hammerkit-definition'] !== (0, service_definition_1.getServiceDefinitionHash)(service)) {
                 service.status.write('info', `${service.name} changed since it was started, recreating it`);
                 await (0, remove_container_1.removeContainer)(docker.getContainer(currentService.Id));
+                // its network, proxy and relay go with it, whatever its new setting
+                await (0, item_network_1.removeItemNetworks)(docker, service);
                 return;
             }
             const servicePid = 'hammerkit-pid' in currentService.Labels ? parseInt(currentService.Labels['hammerkit-pid']) : undefined;
@@ -190,6 +219,7 @@ function dockerServiceRuntime(service, workEnvironment) {
             for (const container of containers) {
                 await (0, remove_container_1.removeContainer)(docker.getContainer(container.Id));
             }
+            await (0, item_network_1.removeItemNetworks)(docker, service);
             await (0, container_secrets_1.removeContainerSecrets)((0, work_instance_id_1.getWorkInstanceId)(service));
         },
         async remove() {
@@ -203,6 +233,16 @@ function dockerServiceRuntime(service, workEnvironment) {
             }
         },
         async execute(environment, options) {
+            if (environment.hosted) {
+                try {
+                    await (0, assert_item_paths_1.assertItemPathsWithinRoot)(service, environment.cwd);
+                }
+                catch (e) {
+                    service.status.write('error', (0, log_1.getErrorMessage)(e));
+                    options.state.set({ type: 'end', reason: 'crash', stateKey: options.stateKey });
+                    return;
+                }
+            }
             await (0, docker_service_1.dockerService)(docker, service, options, environment);
         },
         async restore(environment, path) {
@@ -231,6 +271,7 @@ function dockerServiceRuntime(service, workEnvironment) {
 }
 exports.dockerServiceRuntime = dockerServiceRuntime;
 async function archiveContainer(docker, environment, item, path) {
+    await (0, execution_steps_1.refuseHostLinks)(item, environment);
     await (0, using_container_1.usingContainer)(docker, item, {
         abortSignal: environment.abortCtrl.signal,
         Image: item.data.image,