hammerkit 1.26.0 → 1.27.0-next.10

package/dist/docker/item-network.js

@@ -0,0 +1,471 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeOrphanedNetworks = exports.removeItemNetworks = exports.getNetworkHostConfig = exports.removeItemNetwork = exports.prepareItemNetwork = exports.checkNetworkLimitSupport = exports.getProxyNetworkName = exports.getRunNetworkName = exports.getItemNetworkName = void 0;
+const readline_1 = require("readline");
+const stream_1 = require("stream");
+const work_service_1 = require("../planner/work-service");
+const work_network_1 = require("../planner/work-network");
+const work_instance_id_1 = require("../planner/work-instance-id");
+const get_service_containers_1 = require("../executer/get-service-containers");
+const service_dns_1 = require("../executer/service-dns");
+const run_labels_1 = require("./run-labels");
+const run_id_1 = require("../utils/run-id");
+const pull_1 = require("./pull");
+const remove_container_1 = require("./remove-container");
+const egress_proxy_1 = require("../network/egress-proxy");
+const port_relay_1 = require("../network/port-relay");
+const log_1 = require("../log");
+const sleep_1 = require("../utils/sleep");
+const remove_orphaned_containers_1 = require("./remove-orphaned-containers");
+const PROXY_READY_TIMEOUT_MS = 60000;
+const NETWORK_REMOVE_ATTEMPTS = 20;
+// the first release with com.docker.network.bridge.inhibit_ipv4, which keeps
+// an internal network from reaching the host through its gateway; it also
+// has the fix for CVE-2024-29018 (DNS forwarded from internal networks)
+const MIN_DOCKER_MAJOR = 28;
+// The internal network of a task or service with a limited network. Services
+// always have one when limited, since what needs them joins it; a task only
+// when it needs a service or allows hosts. The name carries the run, so
+// concurrent runs never share or remove each other's networks.
+function getItemNetworkName(item) {
+    return `hammerkit-${(0, work_instance_id_1.getWorkInstanceId)(item)}-${(0, run_id_1.getRunId)()}`;
+}
+exports.getItemNetworkName = getItemNetworkName;
+// On a server, the network the open tasks and services of a run share, in place of the
+// default bridge: Docker keeps user-defined bridges apart, so the containers of one run
+// can't reach those of another, or of another caller. What lies beyond the host's own
+// interfaces (the LAN, the cloud's metadata address) is the host firewall's to block.
+function getRunNetworkName() {
+    return `hammerkit-run-${(0, run_id_1.getRunId)()}`;
+}
+exports.getRunNetworkName = getRunNetworkName;
+const runNetworks = new WeakMap();
+function withRunNetwork(docker, fn) {
+    let state = runNetworks.get(docker);
+    if (!state) {
+        state = { users: 0, lock: Promise.resolve() };
+        runNetworks.set(docker, state);
+    }
+    const current = state;
+    const result = current.lock.then(() => fn(current));
+    current.lock = result.catch(() => undefined);
+    return result;
+}
+// The run's network for an open item, made by its first user and removed after the last
+// one: null off a server, for a daemon (it outlives the run) and on a daemon that has no
+// Linux bridges, where the item stays on the default bridge as before.
+async function acquireRunNetwork(docker, environment, options) {
+    if (!environment.hosted || options.daemon || (await getDaemonVersion(docker)).Os !== 'linux') {
+        return null;
+    }
+    return withRunNetwork(docker, async (state) => {
+        const name = getRunNetworkName();
+        if (state.users === 0) {
+            try {
+                await docker.createNetwork({
+                    Name: name,
+                    Labels: { app: 'hammerkit', ...(0, run_labels_1.getRunLabels)(), 'hammerkit-daemon': 'false' },
+                });
+            }
+            catch (e) {
+                // left by an item whose release failed
+                if (e?.statusCode !== 409) {
+                    throw e;
+                }
+            }
+        }
+        state.users++;
+        return name;
+    });
+}
+async function releaseRunNetwork(docker) {
+    await withRunNetwork(docker, async (state) => {
+        state.users--;
+        if (state.users === 0) {
+            await removeItemNetwork(docker, getRunNetworkName());
+        }
+    });
+}
+async function getAddressOn(docker, containerId, network) {
+    const info = await docker.getContainer(containerId).inspect();
+    const address = info.NetworkSettings.Networks[network]?.IPAddress;
+    if (!address) {
+        throw new Error(`${info.Name.replace(/^\//, '')} has no address on the network ${network} of the run`);
+    }
+    return address;
+}
+// The network between an `allow` item and its proxy, which nothing else joins:
+// the services the item needs don't get to use its proxy.
+function getProxyNetworkName(networkName) {
+    return `${networkName}-proxy`;
+}
+exports.getProxyNetworkName = getProxyNetworkName;
+const daemonVersions = new WeakMap();
+function getDaemonVersion(docker) {
+    let version = daemonVersions.get(docker);
+    if (!version) {
+        version = docker.version();
+        daemonVersions.set(docker, version);
+    }
+    return version;
+}
+// Limits rely on internal networks without a gateway on the host, which only
+// Linux containers on Docker 28 or later have. Anything else would run the
+// item with a hole, so it is refused.
+async function checkNetworkLimitSupport(docker, item) {
+    const { Version, Os } = await getDaemonVersion(docker);
+    const level = (0, work_network_1.getEffectiveNetwork)(item.data).type;
+    if (Os !== 'linux') {
+        throw new Error(`${item.name}: network ${level} needs Linux containers, the docker daemon runs ${Os} containers`);
+    }
+    const major = parseInt(Version, 10);
+    if (!(major >= MIN_DOCKER_MAJOR)) {
+        throw new Error(`${item.name}: network ${level} needs Docker ${MIN_DOCKER_MAJOR} or later, the daemon is ${Version}`);
+    }
+}
+exports.checkNetworkLimitSupport = checkNetworkLimitSupport;
+// Internal (no route out) and without an address on the host, so the host's
+// services can't be reached through it either.
+async function createInternalNetwork(docker, name, labels) {
+    await docker.createNetwork({
+        Name: name,
+        Internal: true,
+        Options: { 'com.docker.network.bridge.inhibit_ipv4': 'true' },
+        Labels: labels,
+    });
+}
+// What the proxy and relay may do: serve, nothing else.
+const HELPER_HOST_CONFIG = {
+    CapDrop: ['ALL'],
+    SecurityOpt: ['no-new-privileges'],
+    ReadonlyRootfs: true,
+    Memory: 256 * 1024 * 1024,
+    PidsLimit: 64,
+};
+// Where the container of a service is reached: on the default bridge (open)
+// or on its own internal network (limited), which may be of an earlier run
+// (`up --daemon`).
+async function getNeedNetwork(docker, need, containerId) {
+    const service = need.service;
+    if (!(0, work_service_1.isContainerWorkService)(service.data) || !(0, work_network_1.isLimitedNetwork)((0, work_network_1.getEffectiveNetwork)(service.data))) {
+        return null;
+    }
+    const info = await docker.getContainer(containerId).inspect();
+    return info.HostConfig.NetworkMode ?? null;
+}
+// Docker options for the network of a task or service. `open` is the default
+// bridge with links to the needed services, as before network limits, and on a
+// server the run's own network (see getRunNetworkName). `none`
+// and `allow` put the container on an internal network of its own (or none at
+// all for a task that needs nothing) that the needed services join under the
+// names it needs them by; `allow` adds the egress proxy to it.
+async function prepareItemNetwork(docker, item, environment, 
+// `publish`: the ports a service publishes on the host
+options) {
+    const network = (0, work_network_1.getEffectiveNetwork)(item.data);
+    const dns = (0, get_service_containers_1.getServiceContainers)(item.needs);
+    const hosts = [];
+    const containers = [];
+    for (const need of item.needs) {
+        const target = dns[need.name];
+        if (!target || (!(0, service_dns_1.isHostServiceDns)(target) && !target.containerId)) {
+            throw new Error(`service ${need.name} is not running`);
+        }
+        if ((0, service_dns_1.isHostServiceDns)(target)) {
+            hosts.push(`${need.name}:${target.host}`);
+        }
+        else {
+            containers.push({
+                need,
+                containerId: target.containerId,
+                network: await getNeedNetwork(docker, need, target.containerId),
+            });
+        }
+    }
+    if (!(0, work_network_1.isLimitedNetwork)(network)) {
+        const linked = containers.filter((c) => !c.network);
+        const shared = await acquireRunNetwork(docker, environment, options);
+        if (shared) {
+            try {
+                // the services it needs are on the same network, found under the names it needs them by
+                for (const target of linked) {
+                    hosts.push(`${target.need.name}:${await getAddressOn(docker, target.containerId, shared)}`);
+                }
+            }
+            catch (e) {
+                await releaseRunNetwork(docker);
+                throw e;
+            }
+        }
+        return {
+            options: shared
+                ? { mode: shared, hosts }
+                : { links: linked.map((c) => `${c.containerId}:${c.need.name}`), hosts },
+            // a limited service is on its internal network only, so an open item
+            // joins that network to reach it
+            async connect(container) {
+                for (const target of containers) {
+                    if (target.network) {
+                        // a link on a user-defined network resolves by container name only
+                        const info = await docker.getContainer(target.containerId).inspect();
+                        await docker.getNetwork(target.network).connect({
+                            Container: container.id,
+                            EndpointConfig: { Links: [`${info.Name.replace(/^\//, '')}:${target.need.name}`] },
+                        });
+                    }
+                }
+            },
+            async release() {
+                if (shared) {
+                    await releaseRunNetwork(docker);
+                }
+            },
+        };
+    }
+    if (network.type === 'none' && containers.length === 0 && !(0, work_service_1.isContainerWorkService)(item.data)) {
+        return {
+            options: { mode: 'none' },
+            async connect() { },
+            async release() { },
+        };
+    }
+    await checkNetworkLimitSupport(docker, item);
+    const name = getItemNetworkName(item);
+    const labels = {
+        app: 'hammerkit',
+        'hammerkit-network-of': (0, work_instance_id_1.getWorkInstanceId)(item),
+        ...(0, run_labels_1.getRunLabels)(),
+        'hammerkit-daemon': options.daemon ? 'true' : 'false',
+    };
+    await createInternalNetwork(docker, name, labels);
+    try {
+        for (const target of containers) {
+            await docker.getNetwork(name).connect({
+                Container: target.containerId,
+                EndpointConfig: { Aliases: [target.need.name] },
+            });
+        }
+        if (network.type === 'allow') {
+            await createInternalNetwork(docker, getProxyNetworkName(name), labels);
+            const proxyIp = await startEgressProxy(docker, item, environment, name, network, options, containers.length > 0);
+            // the item reaches each TCP forward under its host's name, at the proxy
+            for (const forward of (0, work_network_1.getTcpForwards)(network)) {
+                hosts.push(`${forward.host}:${proxyIp}`);
+            }
+        }
+    }
+    catch (e) {
+        await removeItemNetwork(docker, name);
+        throw e;
+    }
+    const publish = (options.publish ?? []).filter((port) => !!port.hostPort);
+    return {
+        options: {
+            mode: name,
+            hosts,
+            env: network.type === 'allow' ? (0, egress_proxy_1.getProxyClientEnvironment)(item.needs.map((need) => need.name)) : {},
+            relayPorts: (0, work_service_1.isContainerWorkService)(item.data),
+        },
+        async connect(container) {
+            try {
+                if (network.type === 'allow') {
+                    await docker.getNetwork(getProxyNetworkName(name)).connect({ Container: container.id });
+                }
+                if (publish.length > 0) {
+                    await startPortRelay(docker, item, environment, name, container, publish, options);
+                }
+            }
+            catch (e) {
+                await removeItemNetwork(docker, name);
+                throw e;
+            }
+        },
+        async release() {
+            await removeItemNetwork(docker, name);
+        },
+    };
+}
+exports.prepareItemNetwork = prepareItemNetwork;
+function getHelperLabels(networkName, options) {
+    return {
+        app: 'hammerkit',
+        // goes with the task or service it serves
+        'hammerkit-type': 'proxy',
+        'hammerkit-network': networkName,
+        ...(0, run_labels_1.getRunLabels)(),
+        'hammerkit-daemon': options.daemon ? 'true' : 'false',
+    };
+}
+// On the default bridge for the way out, and on the proxy network under
+// EGRESS_PROXY_HOST, the only address it listens on: neither the bridge nor
+// the services on the item's network reach it. It joins the item's network
+// to reach the needed services (for tools that ignore NO_PROXY), but doesn't
+// listen there. Refused hosts show up in the item's log. Returns its address
+// on the proxy network, where it also listens for the TCP forwards.
+async function startEgressProxy(docker, item, environment, networkName, network, options, reachNeeds) {
+    await (0, pull_1.pull)(item.status, docker, egress_proxy_1.EGRESS_PROXY_IMAGE, environment);
+    const createOptions = {
+        Image: egress_proxy_1.EGRESS_PROXY_IMAGE,
+        Cmd: (0, egress_proxy_1.getEgressProxyCommand)(),
+        User: 'node',
+        Env: Object.entries((0, egress_proxy_1.getItemProxyEnvironment)(item, egress_proxy_1.EGRESS_PROXY_HOST)).map(([key, value]) => `${key}=${value}`),
+        Labels: getHelperLabels(networkName, options),
+        // unprivileged, it still listens on the port of a TCP forward below 1024
+        HostConfig: { ...HELPER_HOST_CONFIG, Sysctls: { 'net.ipv4.ip_unprivileged_port_start': '0' } },
+    };
+    const container = await docker.createContainer(createOptions);
+    await docker.getNetwork(getProxyNetworkName(networkName)).connect({
+        Container: container.id,
+        EndpointConfig: { Aliases: [egress_proxy_1.EGRESS_PROXY_HOST] },
+    });
+    if (reachNeeds) {
+        await docker.getNetwork(networkName).connect({ Container: container.id });
+    }
+    const stream = await container.attach({ stream: true, stdout: true, stderr: true });
+    const stdout = new stream_1.PassThrough();
+    const stderr = new stream_1.PassThrough();
+    docker.modem.demuxStream(stream, stdout, stderr);
+    stream.on('end', () => {
+        stdout.end();
+        stderr.end();
+    });
+    stderr.on('data', (data) => item.status.write('debug', `proxy: ${data.toString().trim()}`));
+    const ready = new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error('the egress proxy did not start')), PROXY_READY_TIMEOUT_MS);
+        const lines = (0, readline_1.createInterface)({ input: stdout });
+        lines.on('line', (line) => {
+            const event = (0, egress_proxy_1.parseEgressProxyEvent)(line);
+            if (event?.type === 'ready') {
+                clearTimeout(timeout);
+                resolve();
+            }
+            else if (event) {
+                item.status.write('warn', (0, egress_proxy_1.formatEgressProxyEvent)(event));
+            }
+        });
+        lines.on('close', () => {
+            clearTimeout(timeout);
+            reject(new Error('the egress proxy stopped'));
+        });
+    });
+    // once ready, a later close only means the proxy was removed with the item
+    ready.catch(() => undefined);
+    await container.start();
+    await ready;
+    const ip = (await container.inspect()).NetworkSettings.Networks[getProxyNetworkName(networkName)]?.IPAddress;
+    if (!ip) {
+        throw new Error(`the egress proxy of ${item.name} has no address on its network`);
+    }
+    item.status.write('debug', `egress proxy allows ${network.hosts.join(', ') || 'no host'}`);
+    return ip;
+}
+// On the default bridge, where Docker publishes ports, and on the service's
+// network, from where it passes connections on to the service.
+async function startPortRelay(docker, item, environment, networkName, service, ports, options) {
+    await (0, pull_1.pull)(item.status, docker, egress_proxy_1.EGRESS_PROXY_IMAGE, environment);
+    // a container is found by its name on a user-defined network
+    const host = (await service.inspect()).Name.replace(/^\//, '');
+    const container = await docker.createContainer({
+        Image: egress_proxy_1.EGRESS_PROXY_IMAGE,
+        Cmd: (0, port_relay_1.getPortRelayCommand)(),
+        User: 'node',
+        Env: Object.entries((0, port_relay_1.getPortRelayEnvironment)(ports.map((port) => ({ port: port.containerPort, host, targetPort: port.containerPort })))).map(([key, value]) => `${key}=${value}`),
+        Labels: getHelperLabels(networkName, options),
+        ExposedPorts: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, {}])),
+        HostConfig: {
+            ...HELPER_HOST_CONFIG,
+            PortBindings: Object.fromEntries(ports.map((port) => [`${port.containerPort}/tcp`, [{ HostPort: `${port.hostPort}` }]])),
+        },
+    });
+    await docker.getNetwork(networkName).connect({ Container: container.id });
+    await container.start();
+    item.status.write('debug', `ports ${ports.map((port) => port.hostPort).join(', ')} published by a relay`);
+}
+// The network, its proxy network, its proxy and relay, and every endpoint on
+// them: what an earlier run left behind, or what the item used.
+async function removeItemNetwork(docker, name) {
+    const helpers = await docker.listContainers({ all: true, filters: { label: [`hammerkit-network=${name}`] } });
+    for (const helper of helpers) {
+        await (0, remove_container_1.removeContainer)(docker.getContainer(helper.Id));
+    }
+    await removeNetwork(docker, getProxyNetworkName(name));
+    await removeNetwork(docker, name);
+}
+exports.removeItemNetwork = removeItemNetwork;
+async function removeNetwork(docker, name) {
+    const network = docker.getNetwork(name);
+    let info;
+    try {
+        info = await network.inspect();
+    }
+    catch (e) {
+        if (e?.statusCode === 404) {
+            return;
+        }
+        throw e;
+    }
+    for (const containerId of Object.keys(info.Containers ?? {})) {
+        try {
+            await network.disconnect({ Container: containerId, Force: true });
+        }
+        catch {
+            // gone already, or on its way out with its container
+        }
+    }
+    // a container being removed leaves its endpoints a moment later
+    for (let attempt = 1;; attempt++) {
+        try {
+            await network.remove();
+            return;
+        }
+        catch (e) {
+            if (e?.statusCode === 404) {
+                return;
+            }
+            if (attempt >= NETWORK_REMOVE_ATTEMPTS) {
+                throw new Error(`unable to remove network ${name}: ${(0, log_1.getErrorMessage)(e)}`);
+            }
+            await (0, sleep_1.sleep)(500);
+        }
+    }
+}
+function getNetworkHostConfig(network) {
+    return {
+        ...(network.mode ? { NetworkMode: network.mode } : {}),
+        Links: network.links ?? [],
+        ExtraHosts: network.hosts ?? [],
+    };
+}
+exports.getNetworkHostConfig = getNetworkHostConfig;
+// Every network of the item, of this run or another one: what `down` and
+// `clean` remove.
+async function removeItemNetworks(docker, item) {
+    const networks = await docker.listNetworks({
+        filters: { label: [`hammerkit-network-of=${(0, work_instance_id_1.getWorkInstanceId)(item)}`] },
+    });
+    for (const network of networks) {
+        await removeItemNetwork(docker, network.Name);
+    }
+}
+exports.removeItemNetworks = removeItemNetworks;
+// The internal networks of tasks and services a run left behind when it was
+// killed. Networks of `up --daemon` services stay until `down`.
+async function removeOrphanedNetworks(docker) {
+    let networks;
+    try {
+        networks = await docker.listNetworks({ filters: { label: ['app=hammerkit'] } });
+    }
+    catch (e) {
+        if ((0, remove_orphaned_containers_1.isDockerUnreachable)(e)) {
+            return [];
+        }
+        throw e;
+    }
+    const orphaned = networks.filter((network) => (0, remove_orphaned_containers_1.isOfStoppedRun)(network.Labels ?? {}, remove_orphaned_containers_1.isProcessAlive));
+    for (const network of orphaned) {
+        await removeItemNetwork(docker, network.Name);
+    }
+    return orphaned.map((network) => network.Name);
+}
+exports.removeOrphanedNetworks = removeOrphanedNetworks;
+//# sourceMappingURL=item-network.js.map

package/dist/docker/pull.js

@@ -19,11 +19,13 @@ async function pull(status, docker, imageName, environment) {
 exports.pull = pull;
 // A tag is present when an image carries it; an image pinned by digest
 // (`repo:tag@sha256:…`) when an image was pulled with that digest, whatever
-// tag it has now.
+// tag it has now. The containerd image store lists an image pulled by digest
+// under that name in RepoTags rather than in RepoDigests.
 function isImagePresent(images, imageName) {
     const { repository, tag, digest } = splitImageName(imageName);
     if (digest) {
-        return images.some((i) => i.RepoDigests?.some((repoDigest) => repoDigest === `${repository}@${digest}`));
+        const name = `${repository}@${digest}`;
+        return images.some((i) => i.RepoDigests?.includes(name) || i.RepoTags?.includes(name));
     }
     return images.some((i) => i.RepoTags?.some((repoTag) => repoTag === `${repository}:${tag ?? 'latest'}`));
 }

package/dist/docker/remove-orphaned-containers.js

@@ -1,12 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeOrphanedContainers = exports.isDockerUnreachable = exports.isProcessAlive = exports.isOrphanedContainer = void 0;
+exports.removeOrphanedContainers = exports.isDockerUnreachable = exports.isProcessAlive = exports.isOfStoppedRun = exports.isOrphanedContainer = void 0;
 const os_1 = require("os");
 const remove_container_1 = require("./remove-container");
 // A hammerkit container nothing will remove any more: a task container (they
-// never outlive their run) or a service of a run that is gone, started by a
-// hammerkit process of this machine that no longer exists, and the paused
-// state records hammerkit kept before 1.9. Services of `up --daemon` and
+// never outlive their run), a service or an egress proxy of a run that is
+// gone, started by a hammerkit process of this machine that no longer exists,
+// and the paused state records hammerkit kept before 1.9. Services of `up --daemon` and
 // containers of other machines or of running processes stay.
 function isOrphanedContainer(container, isProcessAlive) {
     const labels = container.Labels;
@@ -14,9 +14,16 @@ function isOrphanedContainer(container, isProcessAlive) {
     if (type === 'task' && container.State === 'paused') {
         return true;
     }
-    if (type !== 'task' && type !== 'service') {
+    // a proxy goes with the task or service it serves
+    if (type !== 'task' && type !== 'service' && type !== 'proxy') {
         return false;
     }
+    return isOfStoppedRun(labels, isProcessAlive);
+}
+exports.isOrphanedContainer = isOrphanedContainer;
+// Labeled with the run labels of a hammerkit process of this machine that no
+// longer exists, and not kept for `up --daemon`.
+function isOfStoppedRun(labels, isProcessAlive) {
     if (labels['hammerkit-daemon'] === 'true') {
         return false;
     }
@@ -29,7 +36,7 @@ function isOrphanedContainer(container, isProcessAlive) {
     }
     return !isProcessAlive(pid);
 }
-exports.isOrphanedContainer = isOrphanedContainer;
+exports.isOfStoppedRun = isOfStoppedRun;
 function isProcessAlive(pid) {
     try {
         process.kill(pid, 0);

package/dist/environment/replace-env-variables.js

@@ -6,9 +6,10 @@ const INTERPOLATION = /\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}/g;
 // The shell / process environment first, then the .env files. A variable set to
 // an empty string is set, as in a shell: CI providers leave many empty (a merge
 // request id on a branch pipeline).
-function resolveReference(name, environment, context) {
+function resolveReference(name, environment, context, fromProcess) {
     const processValue = environment.processEnvs[name];
     if (processValue !== undefined && processValue !== null) {
+        fromProcess?.add(name);
         return processValue;
     }
     for (const envFile of Object.values(context.envFiles)) {
@@ -22,10 +23,11 @@ function resolveReference(name, environment, context) {
 function buildEnvironmentVariables(envs, environment, context) {
     const replacements = [];
     const variables = {};
+    const fromProcess = new Set();
     for (const [key, value] of Object.entries(envs)) {
         if (value.startsWith('$') && !value.startsWith('${')) {
             const name = value.substring(1);
-            const nameValue = resolveReference(name, environment, context);
+            const nameValue = resolveReference(name, environment, context, fromProcess);
             if (nameValue !== null) {
                 replacements.push({ key, name, available: true, value: nameValue });
             }
@@ -34,7 +36,7 @@ function buildEnvironmentVariables(envs, environment, context) {
             }
         }
         else if (value.includes('${')) {
-            const interpolated = interpolate(value, envs, environment, context);
+            const interpolated = interpolate(value, envs, environment, context, fromProcess);
             if (interpolated.missing) {
                 replacements.push({ key, name: interpolated.missing, available: false, value: null });
             }
@@ -46,16 +48,16 @@ function buildEnvironmentVariables(envs, environment, context) {
             variables[key] = value;
         }
     }
-    return { variables, replacements };
+    return { variables, replacements, processNames: [...fromProcess].sort() };
 }
 exports.buildEnvironmentVariables = buildEnvironmentVariables;
 // Each `${NAME}` comes from the process environment or a .env file, else from a
 // literal value of the same build file / task `envs`, else from its
 // `:-default`. A reference without any of them is missing.
-function interpolate(value, envs, environment, context) {
+function interpolate(value, envs, environment, context, fromProcess) {
     let missing = null;
     const result = value.replace(INTERPOLATION, (match, name, fallback) => {
-        const resolved = resolveReference(name, environment, context) ?? getLiteral(envs, name) ?? fallback ?? null;
+        const resolved = resolveReference(name, environment, context, fromProcess) ?? getLiteral(envs, name) ?? fallback ?? null;
         if (resolved === null) {
             missing = missing ?? name;
             return match;

package/dist/executer/await-completed-dependencies.js

@@ -77,7 +77,8 @@ exports.awaitCompleted = awaitCompleted;
 async function awaitRunningNeed(work, dep, abort) {
     const state = await (0, state_resolver_1.isState)('await-running-need-' + work.name, dep.state, isRunningOrDone, abort);
     if (state.type !== 'running') {
-        throw new Error(`needed service ${dep.name} ${state.type === 'end' ? 'stopped' : 'failed'} before it was ready`);
+        const why = state.type === 'error' ? `: ${state.errorMessage}` : '';
+        throw new Error(`needed service ${dep.name} ${state.type === 'end' ? 'stopped' : 'failed'} before it was ready${why}`);
     }
     return state;
 }

package/dist/executer/check-readiness.js

@@ -2,8 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.checkReadiness = void 0;
 const execute_docker_1 = require("./execute-docker");
-async function checkReadiness(status, command, environment, container, user, abort) {
-    const result = await (0, execute_docker_1.execCommand)(status, environment, container, undefined, command, user, 2000, abort);
+async function checkReadiness(status, command, environment, container, user, abort, env) {
+    const result = await (0, execute_docker_1.execCommand)(status, environment, container, undefined, command, user, 2000, abort, env);
     if (result.type === 'timeout') {
         return false;
     }