hammerkit 1.25.0 → 1.27.0-next.1

package/build.schema.json

@@ -261,6 +261,34 @@
                     "additionalProperties": false,
                     "description": "cpu and memory requests and limits"
                   },
+                  "network": {
+                    "anyOf": [
+                      {
+                        "type": "string",
+                        "const": "open"
+                      },
+                      {
+                        "type": "string",
+                        "const": "none"
+                      },
+                      {
+                        "type": "object",
+                        "properties": {
+                          "allow": {
+                            "type": "array",
+                            "items": {
+                              "type": "string"
+                            }
+                          }
+                        },
+                        "required": [
+                          "allow"
+                        ],
+                        "additionalProperties": false
+                      }
+                    ],
+                    "description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
+                  },
                   "runner": {
                     "type": "string",
                     "minLength": 1
@@ -273,6 +301,10 @@
                     "items": {
                       "type": "string"
                     }
+                  },
+                  "volumeSize": {
+                    "type": "string",
+                    "description": "size of its claim on Kubernetes, e.g. 10Gi"
                   }
                 },
                 "required": [
@@ -335,6 +367,9 @@
                   "resources": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources"
                   },
+                  "network": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network"
+                  },
                   "runner": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/runner"
                   },
@@ -468,6 +503,14 @@
                   "resources": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources",
                     "description": "cpu and memory requests and limits"
+                  },
+                  "volumeSize": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize",
+                    "description": "size of its claim on Kubernetes, e.g. 10Gi"
+                  },
+                  "network": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/network",
+                    "description": "network the task or service reaches besides its needs: open (default), none or { allow: [hosts or host:port] }"
                   }
                 },
                 "required": [
@@ -630,6 +673,15 @@
                           "additionalProperties": false
                         }
                       },
+                      "networkPolicy": {
+                        "type": "string",
+                        "const": "enforced",
+                        "description": "the cluster enforces NetworkPolicy, which limited networks need"
+                      },
+                      "volumeSize": {
+                        "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize",
+                        "description": "size of the claim of each task and service, 1Gi by default"
+                      },
                       "ingresses": {
                         "not": {}
                       }
@@ -750,7 +802,7 @@
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
                   },
                   "maxSize": {
-                    "type": "string"
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/volumeSize"
                   },
                   "keepPerTask": {
                     "type": "integer",

package/dist/cache/backends/local-cache-backend.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createLocalCacheBackend = void 0;
+exports.createLocalCacheBackend = exports.getDefaultLocalCachePath = void 0;
 const path_1 = require("path");
 const os_1 = require("os");
 const promises_1 = require("fs/promises");
@@ -16,8 +16,16 @@ async function directories(path) {
         return [];
     }
 }
+// Below the home directory, or below `HAMMERKIT_DIR` when that is set: a server sets
+// it per source trust, so the runs of one repository share their cache and nobody
+// else's run finds it (a run's home directory is its own and goes with it).
+function getDefaultLocalCachePath() {
+    const directory = process.env.HAMMERKIT_DIR;
+    return directory ? (0, path_1.join)(directory, 'remote-cache') : (0, path_1.join)((0, os_1.homedir)(), '.hammerkit', 'remote-cache');
+}
+exports.getDefaultLocalCachePath = getDefaultLocalCachePath;
 function createLocalCacheBackend(spec) {
-    const root = spec.path ?? (0, path_1.join)((0, os_1.homedir)(), '.hammerkit', 'remote-cache');
+    const root = spec.path ?? getDefaultLocalCachePath();
     async function readEntry(taskId, stateKey) {
         const dir = (0, path_1.join)(root, taskId, stateKey);
         let created;

package/dist/cache/explain.js

@@ -121,6 +121,19 @@ async function explainTask(item, defaultCacheMethod, environment, memo = new Map
         return { taskId: item.id(), taskName: item.name, status: 'miss', causes: [{ kind: 'never-cached' }] };
     }
     visiting.add(item.id());
+    // Like the scheduler, a task whose cache is off is never looked up, so its
+    // sources aren't hashed for it.
+    if ((0, state_key_1.resolveEffective)(item, defaultCacheMethod).method === 'none') {
+        const explanation = {
+            taskId: item.id(),
+            taskName: item.name,
+            status: 'uncacheable',
+            causes: [{ kind: 'caching-disabled' }],
+        };
+        memo.set(item.id(), explanation);
+        visiting.delete(item.id());
+        return explanation;
+    }
     const { stateKey, stats, resolved, provable, unmatched } = await (0, state_key_1.computeStateKey)(item, defaultCacheMethod, environment);
     // Mirrors the scheduler: a task that cannot be proven up to date always runs
     // (see computeStateKey), whatever a cache holds for it.
@@ -144,17 +157,6 @@ async function explainTask(item, defaultCacheMethod, environment, memo = new Map
         visiting.delete(item.id());
         return explanation;
     }
-    if (resolved.method === 'none') {
-        const explanation = {
-            taskId: item.id(),
-            taskName: item.name,
-            status: 'uncacheable',
-            causes: [{ kind: 'caching-disabled' }],
-        };
-        memo.set(item.id(), explanation);
-        visiting.delete(item.id());
-        return explanation;
-    }
     const runtimeStateKey = await safeCurrentStateKey(item, environment);
     const hasEntry = await safeHas(item, stateKey, environment);
     const predictedHit = runtimeStateKey === stateKey || hasEntry;

package/dist/cli.js

@@ -23,6 +23,7 @@ const dry_run_1 = require("./executer/dry-run");
 const cache_sync_1 = require("./cache/cache-sync");
 const cache_inventory_1 = require("./cache/cache-inventory");
 const prefetch_secrets_1 = require("./secrets/prefetch-secrets");
+const work_network_1 = require("./planner/work-network");
 const isCliTask = (val) => val.type === 'task';
 exports.isCliTask = isCliTask;
 const isCliService = (val) => val.type === 'service';
@@ -50,6 +51,7 @@ class Cli {
         return {
             state: processWorkTree,
             start: async () => {
+                (0, work_network_1.applyNetworkCeiling)(workTree, type === 'down' ? null : options?.network ?? null);
                 if (type !== 'down') {
                     await this.prepareSecrets(true);
                 }
@@ -80,6 +82,7 @@ class Cli {
                         skipDeps: options?.skipDeps ?? true,
                         wait: options?.wait ?? 'ready',
                         resourceCheck: options?.resourceCheck ?? true,
+                        network: options?.network ?? null,
                     });
                 }
                 const result = (0, get_scheduler_execute_result_1.getSchedulerExecuteResult)(workTree);

package/dist/docker/item-network.js

@@ -0,0 +1,471 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeOrphanedNetworks = exports.removeItemNetworks = exports.getNetworkHostConfig = exports.removeItemNetwork = exports.prepareItemNetwork = exports.checkNetworkLimitSupport = exports.getProxyNetworkName = exports.getRunNetworkName = exports.getItemNetworkName = void 0;
+const readline_1 = require("readline");
+const stream_1 = require("stream");
+const work_service_1 = require("../planner/work-service");
+const work_network_1 = require("../planner/work-network");
+const work_instance_id_1 = require("../planner/work-instance-id");
+const get_service_containers_1 = require("../executer/get-service-containers");
+const service_dns_1 = require("../executer/service-dns");
+const run_labels_1 = require("./run-labels");
+const run_id_1 = require("../utils/run-id");
+const pull_1 = require("./pull");
+const remove_container_1 = require("./remove-container");
+const egress_proxy_1 = require("../network/egress-proxy");
+const port_relay_1 = require("../network/port-relay");
+const log_1 = require("../log");
+const sleep_1 = require("../utils/sleep");
+const remove_orphaned_containers_1 = require("./remove-orphaned-containers");
+const PROXY_READY_TIMEOUT_MS = 60000;
+const NETWORK_REMOVE_ATTEMPTS = 20;
+// the first release with com.docker.network.bridge.inhibit_ipv4, which keeps
+// an internal network from reaching the host through its gateway; it also
+// has the fix for CVE-2024-29018 (DNS forwarded from internal networks)
+const MIN_DOCKER_MAJOR = 28;
+// The internal network of a task or service with a limited network. Services
+// always have one when limited, since what needs them joins it; a task only
+// when it needs a service or allows hosts. The name carries the run, so
+// concurrent runs never share or remove each other's networks.
+function getItemNetworkName(item) {
+    return `hammerkit-${(0, work_instance_id_1.getWorkInstanceId)(item)}-${(0, run_id_1.getRunId)()}`;
+}
+exports.getItemNetworkName = getItemNetworkName;
+// On a server, the network the open tasks and services of a run share, in place of the
+// default bridge: Docker keeps user-defined bridges apart, so the containers of one run
+// can't reach those of another, or of another caller. What lies beyond the host's own
+// interfaces (the LAN, the cloud's metadata address) is the host firewall's to block.
+function getRunNetworkName() {
+    return `hammerkit-run-${(0, run_id_1.getRunId)()}`;
+}
+exports.getRunNetworkName = getRunNetworkName;
+const runNetworks = new WeakMap();
+function withRunNetwork(docker, fn) {
+    let state = runNetworks.get(docker);
+    if (!state) {
+        state = { users: 0, lock: Promise.resolve() };
+        runNetworks.set(docker, state);
+    }
+    const current = state;
+    const result = current.lock.then(() => fn(current));
+    current.lock = result.catch(() => undefined);
+    return result;
+}
+// The run's network for an open item, made by its first user and removed after the last
+// one: null off a server, for a daemon (it outlives the run) and on a daemon that has no
+// Linux bridges, where the item stays on the default bridge as before.
+async function acquireRunNetwork(docker, environment, options) {
+    if (!environment.hosted || options.daemon || (await getDaemonVersion(docker)).Os !== 'linux') {
+        return null;
+    }
+    return withRunNetwork(docker, async (state) => {
+        const name = getRunNetworkName();
+        if (state.users === 0) {
+            try {
+                await docker.createNetwork({
+                    Name: name,
+                    Labels: { app: 'hammerkit', ...(0, run_labels_1.getRunLabels)(), 'hammerkit-daemon': 'false' },
+                });
+            }
+            catch (e) {
+                // left by an item whose release failed
+                if (e?.statusCode !== 409) {
+                    throw e;
+                }
+            }
+        }
+        state.users++;
+        return name;
+    });
+}
+async function releaseRunNetwork(docker) {
+    await withRunNetwork(docker, async (state) => {
+        state.users--;
+        if (state.users === 0) {
+            await removeItemNetwork(docker, getRunNetworkName());
+        }
+    });
+}
+async function getAddressOn(docker, containerId, network) {
+    const info = await docker.getContainer(containerId).inspect();
+    const address = info.NetworkSettings.Networks[network]?.IPAddress;
+    if (!address) {
+        throw new Error(`${info.Name.replace(/^\//, '')} has no address on the network ${network} of the run`);
+    }
+    return address;
+}
+// The network between an `allow` item and its proxy, which nothing else joins:
+// the services the item needs don't get to use its proxy.
+function getProxyNetworkName(networkName) {
+    return `${networkName}-proxy`;
+}
+exports.getProxyNetworkName = getProxyNetworkName;
+const daemonVersions = new WeakMap();
+function getDaemonVersion(docker) {
+    let version = daemonVersions.get(docker);
+    if (!version) {
+        version = docker.version();
+        daemonVersions.set(docker, version);
+    }
+    return version;
+}
+// Limits rely on internal networks without a gateway on the host, which only
+// Linux containers on Docker 28 or later have. Anything else would run the
+// item with a hole, so it is refused.
+async function checkNetworkLimitSupport(docker, item) {
+    const { Version, Os } = await getDaemonVersion(docker);
+    const level = (0, work_network_1.getEffectiveNetwork)(item.data).type;
+    if (Os !== 'linux') {
+        throw new Error(`${item.name}: network ${level} needs Linux containers, the docker daemon runs ${Os} containers`);
+    }
+    const major = parseInt(Version, 10);
+    if (!(major >= MIN_DOCKER_MAJOR)) {
+        throw new Error(`${item.name}: network ${level} needs Docker ${MIN_DOCKER_MAJOR} or later, the daemon is ${Version}`);
+    }
+}
+exports.checkNetworkLimitSupport = checkNetworkLimitSupport;
+// Internal (no route out) and without an address on the host, so the host's
+// services can't be reached through it either.
+async function createInternalNetwork(docker, name, labels) {
+    await docker.createNetwork({
+        Name: name,
+        Internal: true,
+        Options: { 'com.docker.network.bridge.inhibit_ipv4': 'true' },
+        Labels: labels,
+    });
+}
+// What the proxy and relay may do: serve, nothing else.
+const HELPER_HOST_CONFIG = {
+    CapDrop: ['ALL'],
+    SecurityOpt: ['no-new-privileges'],
+    ReadonlyRootfs: true,
+    Memory: 256 * 1024 * 1024,
+    PidsLimit: 64,
+};
+// Where the container of a service is reached: on the default bridge (open)
+// or on its own internal network (limited), which may be of an earlier run
+// (`up --daemon`).
+async function getNeedNetwork(docker, need, containerId) {
+    const service = need.service;
+    if (!(0, work_service_1.isContainerWorkService)(service.data) || !(0, work_network_1.isLimitedNetwork)((0, work_network_1.getEffectiveNetwork)(service.data))) {
+        return null;
+    }
+    const info = await docker.getContainer(containerId).inspect();
+    return info.HostConfig.NetworkMode ?? null;
+}
+// Docker options for the network of a task or service. `open` is the default
+// bridge with links to the needed services, as before network limits, and on a
+// server the run's own network (see getRunNetworkName). `none`
+// and `allow` put the container on an internal network of its own (or none at
+// all for a task that needs nothing) that the needed services join under the
+// names it needs them by; `allow` adds the egress proxy to it.
+async function prepareItemNetwork(docker, item, environment, 
+// `publish`: the ports a service publishes on the host
+options) {
+    const network = (0, work_network_1.getEffectiveNetwork)(item.data);
+    const dns = (0, get_service_containers_1.getServiceContainers)(item.needs);
+    const hosts = [];
+    const containers = [];
+    for (const need of item.needs) {
+        const target = dns[need.name];
+        if (!target || (!(0, service_dns_1.isHostServiceDns)(target) && !target.containerId)) {
+            throw new Error(`service ${need.name} is not running`);
+        }
+        if ((0, service_dns_1.isHostServiceDns)(target)) {
+            hosts.push(`${need.name}:${target.host}`);
+        }
+        else {
+            containers.push({
+                need,
+                containerId: target.containerId,
+                network: await getNeedNetwork(docker, need, target.containerId),
+            });
+        }
+    }
+    if (!(0, work_network_1.isLimitedNetwork)(network)) {
+        const linked = containers.filter((c) => !c.network);
+        const shared = await acquireRunNetwork(docker, environment, options);
+        if (shared) {
+            try {
+                // the services it needs are on the same network, found under the names it needs them by
+                for (const target of linked) {
+                    hosts.push(`${target.need.name}:${await getAddressOn(docker, target.containerId, shared)}`);
+                }
+            }
+            catch (e) {
+                await releaseRunNetwork(docker);
+                throw e;
+            }
+        }
+        return {
+            options: shared
+                ? { mode: shared, hosts }
+                : { links: linked.map((c) => `${c.containerId}:${c.need.name}`), hosts },
+            // a limited service is on its internal network only, so an open item
+            // joins that network to reach it
+            async connect(container) {
+                for (const target of containers) {
+                    if (target.network) {
+                        // a link on a user-defined network resolves by container name only
+                        const info = await docker.getContainer(target.containerId).inspect();
+                        await docker.getNetwork(target.network).connect({
+                            Container: container.id,
+                            EndpointConfig: { Links: [`${info.Name.replace(/^\//, '')}:${target.need.name}`] },
+                        });
+                    }
+                }
+            },
+            async release() {
+                if (shared) {
+                    await releaseRunNetwork(docker);
+                }
+            },
+        };
+    }
+    if (network.type === 'none' && containers.length === 0 && !(0, work_service_1.isContainerWorkService)(item.data)) {
+        return {
+            options: { mode: 'none' },
+            async connect() { },
+            async release() { },
+        };
+    }
+    await checkNetworkLimitSupport(docker, item);
+    const name = getItemNetworkName(item);
+    const labels = {
+        app: 'hammerkit',
+        'hammerkit-network-of': (0, work_instance_id_1.getWorkInstanceId)(item),
+        ...(0, run_labels_1.getRunLabels)(),
+        'hammerkit-daemon': options.daemon ? 'true' : 'false',
+    };
+    await createInternalNetwork(docker, name, labels);
+    try {
+        for (const target of containers) {
+            await docker.getNetwork(name).connect({
+                Container: target.containerId,
+                EndpointConfig: { Aliases: [target.need.name] },
+            });
+        }
+        if (network.type === 'allow') {
+            await createInternalNetwork(docker, getProxyNetworkName(name), labels);
+            const proxyIp = await startEgressProxy(docker, item, environment, name, network, options, containers.length > 0);
+            // the item reaches each TCP forward under its host's name, at the proxy
+            for (const forward of (0, work_network_1.getTcpForwards)(network)) {
+                hosts.push(`${forward.host}:${proxyIp}`);
+            }
+        }
+    }
+    catch (e) {
+        await removeItemNetwork(docker, name);
+        throw e;
+    }
+    const publish = (options.publish ?? []).filter((port) => !!port.hostPort);
+    return {
+        options: {
+            mode: name,
+            hosts,
+            env: network.type === 'allow' ? (0, egress_proxy_1.getProxyClientEnvironment)(item.needs.map((need) => need.name)) : {},
+            relayPorts: (0, work_service_1.isContainerWorkService)(item.data),
+        },
+        async connect(container) {
+            try {
+                if (network.type === 'allow') {
+                    await docker.getNetwork(getProxyNetworkName(name)).connect({ Container: container.id });
+                }
+                if (publish.length > 0) {
+                    await startPortRelay(docker, item, environment, name, container, publish, options);
+                }
+            }
+            catch (e) {
+                await removeItemNetwork(docker, name);
+                throw e;
+            }
+        },
+        async release() {
+            await removeItemNetwork(docker, name);
+        },
+    };
+}
+exports.prepareItemNetwork = prepareItemNetwork;
+function getHelperLabels(networkName, options) {
+    return {
+        app: 'hammerkit',
+        // goes with the task or service it serves
+        'hammerkit-type': 'proxy',
+        'hammerkit-network': networkName,
+        ...(0, run_labels_1.getRunLabels)(),
+        'hammerkit-daemon': options.daemon ? 'true' : 'false',
+    };
+}
+// On the default bridge for the way out, and on the proxy network under
+// EGRESS_PROXY_HOST, the only address it listens on: neither the bridge nor
+// the services on the item's network reach it. It joins the item's network
+// to reach the needed services (for tools that ignore NO_PROXY), but doesn't
+// listen there. Refused hosts show up in the item's log. Returns its address
+// on the proxy network, where it also listens for the TCP forwards.
+async function startEgressProxy(docker, item, environment, networkName, network, options, reachNeeds) {
+    await (0, pull_1.pull)(item.status, docker, egress_proxy_1.EGRESS_PROXY_IMAGE, environment);
+    const createOptions = {
+        Image: egress_proxy_1.EGRESS_PROXY_IMAGE,
+        Cmd: (0, egress_proxy_1.getEgressProxyCommand)(),
+        User: 'node',
+        Env: Object.entries((0, egress_proxy_1.getItemProxyEnvironment)(item, egress_proxy_1.EGRESS_PROXY_HOST)).map(([key, value]) => `${key}=${value}`),
+        Labels: getHelperLabels(networkName, options),
+        // unprivileged, it still listens on the port of a TCP forward below 1024
+        HostConfig: { ...HELPER_HOST_CONFIG, Sysctls: { 'net.ipv4.ip_unprivileged_port_start': '0' } },
+    };
+    const container = await docker.createContainer(createOptions);
+    await docker.getNetwork(getProxyNetworkName(networkName)).connect({
+        Container: container.id,
+        EndpointConfig: { Aliases: [egress_proxy_1.EGRESS_PROXY_HOST] },
+    });
+    if (reachNeeds) {
+        await docker.getNetwork(networkName).connect({ Container: container.id });
+    }
+    const stream = await container.attach({ stream: true, stdout: true, stderr: true });
+    const stdout = new stream_1.PassThrough();
+    const stderr = new stream_1.PassThrough();
+    docker.modem.demuxStream(stream, stdout, stderr);
+    stream.on('end', () => {
+        stdout.end();
+        stderr.end();
+    });
+    stderr.on('data', (data) => item.status.write('debug', `proxy: ${data.toString().trim()}`));
+    const ready = new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error('the egress proxy did not start')), PROXY_READY_TIMEOUT_MS);
+        const lines = (0, readline_1.createInterface)({ input: stdout });
+        lines.on('line', (line) => {
+            const event = (0, egress_proxy_1.parseEgressProxyEvent)(line);
+            if (event?.type === 'ready') {
+                clearTimeout(timeout);
+                resolve();
+            }
+            else if (event) {
+                item.status.write('warn', (0, egress_proxy_1.formatEgressProxyEvent)(event));
+            }
+        });
+        lines.on('close', () => {
+            clearTimeout(timeout);
+            reject(new Error('the egress proxy stopped'));
+        });
+    });
+    // once ready, a later close only means the proxy was removed with the item
+    ready.catch(() => undefined);
+    await container.start();
+    await ready;
+    const ip = (await container.inspect()).NetworkSettings.Networks[getProxyNetworkName(networkName)]?.IPAddress;
+    if (!ip) {
+        throw new Error(`the egress proxy of ${item.name} has no address on its network`);
+    }
+    item.status.write('debug', `egress proxy allows ${network.hosts.join(', ') || 'no host'}`);
+    return ip;
+}
+// On the default bridge, where Docker publishes ports, and on the service's
+// network, from where it passes connections on to the service.
+async function startPortRelay(docker, item, environment, networkName, service, ports, options) {
+    await (0, pull_1.pull)(item.status, docker, egress_proxy_1.EGRESS_PROXY_IMAGE, environment);
+    // a container is found by its name on a user-defined network
+    const host = (await service.inspect()).Name.replace(/^\//, '');
+    const container = await docker.createContainer({
+        Image: egress_proxy_1.EGRESS_PROXY_IMAGE,
+        Cmd: (0, port_relay_1.getPortRelayCommand)(),
+        User: 'node',
+        Env: Object.entries((0, port_relay_1.getPortRelayEnvironment)(ports.map((port) => ({ port: port.containerPort, host, targetPort: port.containerPort })))).map(([key, value]) => `${key}=${value}`),
+        Labels: getHelperLabels(networkName, options),
+        ExposedPorts: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, {}])),
+        HostConfig: {
+            ...HELPER_HOST_CONFIG,
+            PortBindings: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, [{ HostPort: `${port.hostPort}` }]])),
+        },
+    });
+    await docker.getNetwork(networkName).connect({ Container: container.id });
+    await container.start();
+    item.status.write('debug', `ports ${ports.map((port) => port.hostPort).join(', ')} published by a relay`);
+}
+// The network, its proxy network, its proxy and relay, and every endpoint on
+// them: what an earlier run left behind, or what the item used.
+async function removeItemNetwork(docker, name) {
+    const helpers = await docker.listContainers({ all: true, filters: { label: [`hammerkit-network=${name}`] } });
+    for (const helper of helpers) {
+        await (0, remove_container_1.removeContainer)(docker.getContainer(helper.Id));
+    }
+    await removeNetwork(docker, getProxyNetworkName(name));
+    await removeNetwork(docker, name);
+}
+exports.removeItemNetwork = removeItemNetwork;
+async function removeNetwork(docker, name) {
+    const network = docker.getNetwork(name);
+    let info;
+    try {
+        info = await network.inspect();
+    }
+    catch (e) {
+        if (e?.statusCode === 404) {
+            return;
+        }
+        throw e;
+    }
+    for (const containerId of Object.keys(info.Containers ?? {})) {
+        try {
+            await network.disconnect({ Container: containerId, Force: true });
+        }
+        catch {
+            // gone already, or on its way out with its container
+        }
+    }
+    // a container being removed leaves its endpoints a moment later
+    for (let attempt = 1;; attempt++) {
+        try {
+            await network.remove();
+            return;
+        }
+        catch (e) {
+            if (e?.statusCode === 404) {
+                return;
+            }
+            if (attempt >= NETWORK_REMOVE_ATTEMPTS) {
+                throw new Error(`unable to remove network ${name}: ${(0, log_1.getErrorMessage)(e)}`);
+            }
+            await (0, sleep_1.sleep)(500);
+        }
+    }
+}
+function getNetworkHostConfig(network) {
+    return {
+        ...(network.mode ? { NetworkMode: network.mode } : {}),
+        Links: network.links ?? [],
+        ExtraHosts: network.hosts ?? [],
+    };
+}
+exports.getNetworkHostConfig = getNetworkHostConfig;
+// Every network of the item, of this run or another one: what `down` and
+// `clean` remove.
+async function removeItemNetworks(docker, item) {
+    const networks = await docker.listNetworks({
+        filters: { label: [`hammerkit-network-of=${(0, work_instance_id_1.getWorkInstanceId)(item)}`] },
+    });
+    for (const network of networks) {
+        await removeItemNetwork(docker, network.Name);
+    }
+}
+exports.removeItemNetworks = removeItemNetworks;
+// The internal networks of tasks and services a run left behind when it was
+// killed. Networks of `up --daemon` services stay until `down`.
+async function removeOrphanedNetworks(docker) {
+    let networks;
+    try {
+        networks = await docker.listNetworks({ filters: { label: ['app=hammerkit'] } });
+    }
+    catch (e) {
+        if ((0, remove_orphaned_containers_1.isDockerUnreachable)(e)) {
+            return [];
+        }
+        throw e;
+    }
+    const orphaned = networks.filter((network) => (0, remove_orphaned_containers_1.isOfStoppedRun)(network.Labels ?? {}, remove_orphaned_containers_1.isProcessAlive));
+    for (const network of orphaned) {
+        await removeItemNetwork(docker, network.Name);
+    }
+    return orphaned.map((network) => network.Name);
+}
+exports.removeOrphanedNetworks = removeOrphanedNetworks;
+//# sourceMappingURL=item-network.js.map

package/dist/docker/pull.js

@@ -19,11 +19,13 @@ async function pull(status, docker, imageName, environment) {
 exports.pull = pull;
 // A tag is present when an image carries it; an image pinned by digest
 // (`repo:tag@sha256:…`) when an image was pulled with that digest, whatever
-// tag it has now.
+// tag it has now. The containerd image store lists an image pulled by digest
+// under that name in RepoTags rather than in RepoDigests.
 function isImagePresent(images, imageName) {
     const { repository, tag, digest } = splitImageName(imageName);
     if (digest) {
-        return images.some((i) => i.RepoDigests?.some((repoDigest) => repoDigest === `${repository}@${digest}`));
+        const name = `${repository}@${digest}`;
+        return images.some((i) => i.RepoDigests?.includes(name) || i.RepoTags?.includes(name));
     }
     return images.some((i) => i.RepoTags?.some((repoTag) => repoTag === `${repository}:${tag ?? 'latest'}`));
 }