hammerkit 1.7.0 → 1.9.0

package/README.md

@@ -1,6 +1,6 @@
 # About
 
-<figure><img src="docs/hammerkit-small.png" width="144" alt=""><figcaption></figcaption></figure>
+<img src="website/public/logo.png" width="144" alt="hammerkit">
 
 ## What is hammerkit?
 
@@ -9,7 +9,7 @@ Hammerkit is a build tool intended to build software projects in **containers**
 * the build tools come from the container image, so they are the same across all machines.
 * the build process is isolated since every file in the container has to be declared as a source or an output. Therefore, side effects can be reduced.
 
-For a fuller pitch — including how it compares to Make, npm scripts, Taskfile and Earthly — see [Why hammerkit](docs/why-hammerkit.md).
+For a fuller pitch — including how it compares to Make, npm scripts, Taskfile and Earthly — see [Why hammerkit](https://hammerkit.dev/docs/why-hammerkit).
 
 There are a lot of containerized CI systems that allow writing containerized builds today, but they have shortcomings that hammerkit tries to address:
 
@@ -23,4 +23,4 @@ Hammerkit, on the other hand, knows the source and output files of each build st
 
 ## Where to go next?
 
-Check out the [docs](https://no0dles.gitbook.io/hammerkit/) for more info about how hammerkit can be used.
+Check out the [docs on hammerkit.dev](https://hammerkit.dev) for more info about how hammerkit can be used.

package/build.schema.json

@@ -12,7 +12,7 @@
               "number"
             ]
           },
-          "description": "Environment values for the current build task\nhttps://no0dles.gitbook.io/hammerkit/build-file/environment-variables"
+          "description": "Environment values for the current build task\nhttps://hammerkit.dev/docs/build-file/environment-variables"
         },
         "tasks": {
           "type": "object",
@@ -61,7 +61,7 @@
                   },
                   "envs": {
                     "$ref": "#/definitions/hammerkit/properties/envs",
-                    "description": "Environment values for the current build task\nhttps://no0dles.gitbook.io/hammerkit/build-file/environment-variables"
+                    "description": "Environment values for the current build task\nhttps://hammerkit.dev/docs/build-file/environment-variables"
                   },
                   "labels": {
                     "type": "object",
@@ -280,7 +280,7 @@
                   },
                   "envs": {
                     "$ref": "#/definitions/hammerkit/properties/envs",
-                    "description": "Environment values for the current build task\nhttps://no0dles.gitbook.io/hammerkit/build-file/environment-variables"
+                    "description": "Environment values for the current build task\nhttps://hammerkit.dev/docs/build-file/environment-variables"
                   },
                   "mounts": {
                     "type": "array",
@@ -309,6 +309,12 @@
                   "cmd": {
                     "type": "string"
                   },
+                  "workdir": {
+                    "type": "string"
+                  },
+                  "shell": {
+                    "type": "string"
+                  },
                   "volumes": {
                     "type": "array",
                     "items": {
@@ -326,6 +332,9 @@
                     "properties": {
                       "cmd": {
                         "type": "string"
+                      },
+                      "timeout": {
+                        "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
                       }
                     },
                     "required": [
@@ -431,18 +440,11 @@
                       "kubeconfig": {
                         "type": "string"
                       },
-                      "ingresses": {
+                      "httpRoutes": {
                         "type": "array",
                         "items": {
                           "type": "object",
                           "properties": {
-                            "kind": {
-                              "type": "string",
-                              "enum": [
-                                "ingress",
-                                "httproute"
-                              ]
-                            },
                             "host": {
                               "type": "string"
                             },
@@ -464,10 +466,14 @@
                           },
                           "required": [
                             "host",
-                            "service"
+                            "service",
+                            "gateway"
                           ],
                           "additionalProperties": false
                         }
+                      },
+                      "ingresses": {
+                        "not": {}
                       }
                     },
                     "required": [
@@ -608,7 +614,7 @@
         }
       },
       "additionalProperties": false,
-      "description": "Build file with support for containerization\nhttps://no0dles.gitbook.io/hammerkit/build-file"
+      "description": "Build file with support for containerization\nhttps://hammerkit.dev/docs/build-file"
     }
   },
   "$schema": "http://json-schema.org/draft-07/schema#"

package/dist/docker/package.js

@@ -1,8 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.longestCommonPrefix = exports.packageWorkTree = exports.getDefaultPlatform = void 0;
+exports.getServiceInstructions = exports.longestCommonPrefix = exports.packageWorkTree = exports.getDefaultPlatform = void 0;
 const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
 const work_item_1 = require("../planner/work-item");
+const work_service_1 = require("../planner/work-service");
 const execute_docker_1 = require("../executer/execute-docker");
 const node_os_1 = require("node:os");
 const path_1 = require("path");
@@ -199,9 +200,17 @@ function getServiceInstructions(service, options) {
             ? service.data.volumes.map((v) => `RUN chown -R 1000:1000 ${toContainerPath((0, path_1.relative)(dependencyCwd, v.containerPath))}`)
             : []),
         options.overrideUser ? 'USER 1000:1000' : '',
+        // last, so the COPY destinations above stay relative to the build directory
+        service.data.workdir ? `WORKDIR ${service.data.workdir}` : '',
     ];
-    if (service.data.cmd) {
-        instructions.push(`CMD [${[service.data.cmd.parsed.command, ...service.data.cmd.parsed.args].map((p) => `"${p}"`).join(', ')}]`);
+    // exec form, JSON-encoded so arguments with quotes or a whole shell command
+    // survive intact
+    const command = (0, work_service_1.getServiceCommand)(service.data);
+    if (command.entrypoint) {
+        instructions.push(`ENTRYPOINT ${JSON.stringify(command.entrypoint)}`);
+    }
+    if (command.cmd) {
+        instructions.push(`CMD ${JSON.stringify(command.cmd)}`);
     }
     return {
         id: service.id(),
@@ -221,6 +230,7 @@ function getServiceInstructions(service, options) {
         ],
     };
 }
+exports.getServiceInstructions = getServiceInstructions;
 function getDependencyInstructions(cwd, task, taskIds) {
     return task.deps.map((dep) => {
         if (taskIds[dep.id()]) {

package/dist/docker/remove-orphaned-containers.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeOrphanedContainers = exports.isProcessAlive = exports.isOrphanedContainer = void 0;
+const os_1 = require("os");
+const remove_container_1 = require("./remove-container");
+// A hammerkit container nothing will remove any more: a task container (they
+// never outlive their run) or a service of a run that is gone, started by a
+// hammerkit process of this machine that no longer exists, and the paused
+// state records hammerkit kept before 1.9. Services of `up --daemon` and
+// containers of other machines or of running processes stay.
+function isOrphanedContainer(container, isProcessAlive) {
+    const labels = container.Labels;
+    const type = labels['hammerkit-type'];
+    if (type === 'task' && container.State === 'paused') {
+        return true;
+    }
+    if (type !== 'task' && type !== 'service') {
+        return false;
+    }
+    if (labels['hammerkit-daemon'] === 'true') {
+        return false;
+    }
+    if (labels['hammerkit-host'] !== (0, os_1.hostname)()) {
+        return false;
+    }
+    const pid = parseInt(labels['hammerkit-pid'] ?? '', 10);
+    if (!pid) {
+        return false;
+    }
+    return !isProcessAlive(pid);
+}
+exports.isOrphanedContainer = isOrphanedContainer;
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (e) {
+        // EPERM: it exists, it just belongs to another user
+        return e.code === 'EPERM';
+    }
+}
+exports.isProcessAlive = isProcessAlive;
+async function removeOrphanedContainers(docker) {
+    const containers = await docker.listContainers({ all: true, filters: { label: ['app=hammerkit'] } });
+    const orphaned = containers.filter((c) => isOrphanedContainer(c, isProcessAlive));
+    for (const container of orphaned) {
+        await (0, remove_container_1.removeContainer)(docker.getContainer(container.Id));
+    }
+    return orphaned.map((c) => c.Id);
+}
+exports.removeOrphanedContainers = removeOrphanedContainers;
+//# sourceMappingURL=remove-orphaned-containers.js.map

package/dist/docker/run-labels.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRunLabels = void 0;
+const os_1 = require("os");
+// Which hammerkit process on which machine started a container. `clean` uses
+// them to tell a container a killed run left behind from one still in use.
+function getRunLabels() {
+    return {
+        'hammerkit-pid': process.pid.toString(),
+        'hammerkit-host': (0, os_1.hostname)(),
+    };
+}
+exports.getRunLabels = getRunLabels;
+//# sourceMappingURL=run-labels.js.map

package/dist/docker/using-container.js

@@ -5,40 +5,23 @@ const remove_container_1 = require("./remove-container");
 const log_1 = require("../log");
 const execute_docker_1 = require("../executer/execute-docker");
 const work_instance_id_1 = require("../planner/work-instance-id");
-async function usingContainer(docker, item, createOptions, stateKey, callback) {
+const remove_orphaned_containers_1 = require("./remove-orphaned-containers");
+// Runs the callback in a fresh container and removes it afterwards, whatever
+// the outcome. What state the outputs hold is recorded outside docker (see
+// container-task-state.ts), so no container outlives the run.
+async function usingContainer(docker, item, createOptions, callback) {
+    await removeLeftoverContainers(docker, item);
     let container = null;
-    let succeeded = false;
-    let result;
     try {
         container = await docker.createContainer(createOptions);
         item.status.write('debug', `starting container with image ${item.data.image}`);
         await (0, execute_docker_1.startContainer)(item.status, container);
-        result = await callback(container);
-        succeeded = result !== false;
-        return result;
+        return await callback(container);
     }
     finally {
         try {
-            if (stateKey && succeeded) {
-                if (container) {
-                    await container.pause();
-                }
-                const containers = await docker.listContainers({
-                    all: true,
-                    filters: { label: [`hammerkit-id=${(0, work_instance_id_1.getWorkInstanceId)(item)}`] },
-                });
-                for (const container of containers) {
-                    item.status.write('debug', `found container ${container.Id}`);
-                    if (container.Labels['hammerkit-state'] != stateKey) {
-                        const oldContainer = docker.getContainer(container.Id);
-                        await (0, remove_container_1.removeContainer)(oldContainer);
-                    }
-                }
-            }
-            else {
-                if (container) {
-                    await (0, remove_container_1.removeContainer)(container);
-                }
+            if (container) {
+                await (0, remove_container_1.removeContainer)(container);
             }
         }
         catch (e) {
@@ -47,4 +30,20 @@ async function usingContainer(docker, item, createOptions, stateKey, callback) {
     }
 }
 exports.usingContainer = usingContainer;
+// Containers of this item nothing will remove: the paused state records
+// hammerkit kept before 1.9, and stopped ones of a hammerkit process that is
+// gone. A container of a live process (one another call of this run has just
+// created, say) stays.
+async function removeLeftoverContainers(docker, item) {
+    const containers = await docker.listContainers({
+        all: true,
+        filters: { label: [`hammerkit-id=${(0, work_instance_id_1.getWorkInstanceId)(item)}`] },
+    });
+    const leftovers = containers.filter((c) => c.State === 'paused' ||
+        (c.State !== 'running' && !(0, remove_orphaned_containers_1.isProcessAlive)(parseInt(c.Labels['hammerkit-pid'] ?? '', 10) || process.pid)));
+    for (const leftover of leftovers) {
+        item.status.write('debug', `remove leftover container ${leftover.Id}`);
+        await (0, remove_container_1.removeContainer)(docker.getContainer(leftover.Id));
+    }
+}
 //# sourceMappingURL=using-container.js.map

package/dist/executer/check-readiness.js

@@ -2,8 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.checkReadiness = void 0;
 const execute_docker_1 = require("./execute-docker");
-async function checkReadiness(status, healthCheck, environment, container, abort) {
-    const result = await (0, execute_docker_1.execCommand)(status, environment, container, undefined, [healthCheck.cmd.parsed.command, ...healthCheck.cmd.parsed.args], null, 2000, abort);
+async function checkReadiness(status, command, environment, container, user, abort) {
+    const result = await (0, execute_docker_1.execCommand)(status, environment, container, undefined, command, user, 2000, abort);
     if (result.type === 'timeout') {
         return false;
     }
@@ -12,11 +12,11 @@ async function checkReadiness(status, healthCheck, environment, container, abort
     }
     else {
         if (result.result.ExitCode === 0) {
-            status.write('debug', `healthcheck ${healthCheck.cmd} succeeded`);
+            status.write('debug', `healthcheck ${command.join(' ')} succeeded`);
             return true;
         }
         else {
-            status.write('debug', `healthcheck ${healthCheck.cmd} failed with ${result.result.ExitCode}`);
+            status.write('debug', `healthcheck ${command.join(' ')} failed with ${result.result.ExitCode}`);
             return false;
         }
     }

package/dist/executer/container-task-state.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeContainerTaskState = exports.writeContainerTaskState = exports.readContainerTaskState = exports.getContainerTaskStateFile = void 0;
+const path_1 = require("path");
+const get_cache_directory_1 = require("../optimizer/get-cache-directory");
+const work_instance_id_1 = require("../planner/work-instance-id");
+// The record says "the outputs of this container task belong to this state
+// key", like the state file of a local task. It lives in the hammerkit
+// directory, not the project, so writing it never touches a task's src.
+function getContainerTaskStateFile(task) {
+    return (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'state', (0, work_instance_id_1.getWorkInstanceId)(task));
+}
+exports.getContainerTaskStateFile = getContainerTaskStateFile;
+async function readContainerTaskState(environment, task) {
+    const stateFile = getContainerTaskStateFile(task);
+    if (!(await environment.file.exists(stateFile))) {
+        return null;
+    }
+    return (await environment.file.read(stateFile)).trim() || null;
+}
+exports.readContainerTaskState = readContainerTaskState;
+async function writeContainerTaskState(environment, task, stateKey) {
+    await environment.file.createDirectory((0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'state'));
+    await environment.file.writeFile(getContainerTaskStateFile(task), stateKey);
+}
+exports.writeContainerTaskState = writeContainerTaskState;
+async function removeContainerTaskState(environment, task) {
+    const stateFile = getContainerTaskStateFile(task);
+    if (await environment.file.exists(stateFile)) {
+        await environment.file.remove(stateFile);
+    }
+}
+exports.removeContainerTaskState = removeContainerTaskState;
+//# sourceMappingURL=container-task-state.js.map

package/dist/executer/docker-service.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dockerService = void 0;
+const work_service_1 = require("../planner/work-service");
+const run_labels_1 = require("../docker/run-labels");
 const abort_1 = require("./abort");
 const execute_docker_1 = require("./execute-docker");
 const stream_1 = require("../docker/stream");
@@ -8,11 +10,14 @@ const abort_event_1 = require("../utils/abort-event");
 const log_1 = require("../log");
 const remove_container_1 = require("../docker/remove-container");
 const check_readiness_1 = require("./check-readiness");
+const main_process_user_1 = require("./main-process-user");
 const execution_steps_1 = require("./execution-steps");
 const docker_task_1 = require("./docker-task");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const get_service_containers_1 = require("./get-service-containers");
 const work_instance_id_1 = require("../planner/work-instance-id");
+const work_healthcheck_1 = require("../planner/work-healthcheck");
+const HEALTHCHECK_INTERVAL_MS = 1000;
 async function dockerService(docker, item, options, environment) {
     let container = null;
     try {
@@ -26,22 +31,25 @@ async function dockerService(docker, item, options, environment) {
         const network = (0, docker_task_1.getNeedsNetwork)(serviceContainers, item.needs);
         item.status.write('debug', `create container with image ${item.data.image}`);
         const envs = (0, replace_env_variables_1.getEnvironmentVariables)(item.data.envs);
+        const command = (0, work_service_1.getServiceCommand)(item.data);
         container = await docker.createContainer({
             Image: item.data.image,
             Env: Object.keys(envs).map((k) => `${k}=${envs[k]}`),
             Labels: {
                 app: 'hammerkit',
                 'hammerkit-id': (0, work_instance_id_1.getWorkInstanceId)(item),
-                'hammerkit-pid': process.pid.toString(),
+                ...(0, run_labels_1.getRunLabels)(),
                 'hammerkit-type': 'service',
                 'hammerkit-state': options.stateKey,
+                'hammerkit-daemon': options.daemon ? 'true' : 'false',
             },
             ExposedPorts: item.data.ports.reduce((map, port) => {
                 map[`${port.containerPort}/tcp`] = {};
                 return map;
             }, {}),
-            Cmd: item.data.cmd ? [item.data.cmd.parsed.command, ...item.data.cmd.parsed.args] : undefined,
-            WorkingDir: item.data.cwd ? (0, execute_docker_1.convertToPosixPath)(item.data.cwd) : undefined,
+            Entrypoint: command.entrypoint ?? undefined,
+            Cmd: command.cmd ?? undefined,
+            WorkingDir: (0, execute_docker_1.convertToPosixPath)((0, work_service_1.getServiceWorkingDir)(item.data)),
             HostConfig: {
                 ExtraHosts: network.hosts,
                 Links: network.links,
@@ -70,11 +78,20 @@ async function dockerService(docker, item, options, environment) {
             });
         }
         else {
+            const healthcheck = item.data.healthcheck;
+            const deadline = Date.now() + healthcheck.timeout;
             let ready = false;
             do {
-                ready = await (0, check_readiness_1.checkReadiness)(item.status, item.data.healthcheck, environment, container, options.abort);
-                if (!ready) {
-                    await new Promise((resolve) => setTimeout(() => resolve(), 1000));
+                // The first check waits one interval too, so an entrypoint that drops
+                // privileges (`exec su-exec app …`) has done so before anything runs.
+                await new Promise((resolve) => setTimeout(() => resolve(), HEALTHCHECK_INTERVAL_MS));
+                if (options.abort.aborted) {
+                    break;
+                }
+                const user = await (0, main_process_user_1.getMainProcessUser)(container);
+                ready = await (0, check_readiness_1.checkReadiness)(item.status, (0, work_service_1.getHealthcheckCommand)(item.data) ?? [], environment, container, user, options.abort);
+                if (!ready && Date.now() >= deadline) {
+                    throw new Error((0, work_healthcheck_1.getHealthcheckTimeoutMessage)(item.name, healthcheck));
                 }
             } while (!ready && !options.abort.aborted);
             if (ready) {

package/dist/executer/docker-task.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.dockerTask = exports.getNeedsNetwork = void 0;
+exports.dockerTask = exports.buildCreateOptions = exports.getNeedsNetwork = void 0;
+const run_labels_1 = require("../docker/run-labels");
 const service_dns_1 = require("./service-dns");
 const execute_docker_1 = require("./execute-docker");
 const abort_1 = require("./abort");
@@ -13,6 +14,8 @@ const replace_env_variables_1 = require("../environment/replace-env-variables");
 const get_container_binds_1 = require("./get-container-binds");
 const get_service_containers_1 = require("./get-service-containers");
 const work_instance_id_1 = require("../planner/work-instance-id");
+const get_outputs_to_reset_1 = require("../planner/utils/get-outputs-to-reset");
+const container_task_state_1 = require("./container-task-state");
 function getNeedsNetwork(serviceContainers, needs) {
     const links = [];
     const hosts = [];
@@ -35,18 +38,22 @@ function buildCreateOptions(item, stateKey, serviceContainers, environment) {
     const network = getNeedsNetwork(serviceContainers, item.needs);
     const binds = (0, get_container_binds_1.getContainerBinds)(item);
     const envs = (0, replace_env_variables_1.getEnvironmentVariables)(item.data.envs);
+    // Running as the host's uid (Linux), the user has no home in the image and
+    // Docker sets HOME=/, which isn't writable: tools then fail to create their
+    // caches. Set on the container only, so it's not part of the cache key.
+    const home = item.data.user && !('HOME' in envs) ? { HOME: '/tmp' } : {};
     return {
         abortSignal: environment.abortCtrl.signal,
         Image: item.data.image,
         Tty: true,
         Entrypoint: [item.data.shell],
         Cmd: ['-c', 'sleep 3600'],
-        Env: Object.entries(envs).map(([key, value]) => `${key}=${value}`),
+        Env: Object.entries({ ...envs, ...home }).map(([key, value]) => `${key}=${value}`),
         WorkingDir: (0, execute_docker_1.convertToPosixPath)(item.data.cwd),
         Labels: {
             app: 'hammerkit',
             'hammerkit-id': (0, work_instance_id_1.getWorkInstanceId)(item),
-            'hammerkit-pid': process.pid.toString(),
+            ...(0, run_labels_1.getRunLabels)(),
             'hammerkit-type': 'task',
             'hammerkit-state': stateKey,
         },
@@ -58,9 +65,18 @@ function buildCreateOptions(item, stateKey, serviceContainers, environment) {
         },
     };
 }
+exports.buildCreateOptions = buildCreateOptions;
 async function dockerTask(docker, item, environment, options) {
     item.status.write('info', `execute ${item.name} in container`);
     try {
+        // A run that fails overwrites outputs, so the previous record goes first,
+        // and only a run that succeeded writes a new one.
+        await (0, container_task_state_1.removeContainerTaskState)(environment, item);
+        const outputsToReset = (0, get_outputs_to_reset_1.getOutputsToReset)(item.data);
+        // outputs on the host: file outputs (bind mounts) and exported directories
+        for (const generate of outputsToReset.filter((g) => g.isFile || g.export)) {
+            await environment.file.remove(generate.path);
+        }
         await (0, execution_steps_1.prepareMounts)(item, environment);
         (0, abort_1.checkForAbort)(options.abort);
         await (0, execution_steps_1.pullImage)(item, docker);
@@ -70,8 +86,11 @@ async function dockerTask(docker, item, environment, options) {
         const serviceContainers = (0, get_service_containers_1.getServiceContainers)(item.needs);
         const containerOptions = buildCreateOptions(item, options.stateKey, serviceContainers, environment);
         (0, print_container_options_1.printContainerOptions)(item.status, containerOptions);
-        await (0, using_container_1.usingContainer)(docker, item, containerOptions, options.stateKey, async (container) => {
+        const succeeded = await (0, using_container_1.usingContainer)(docker, item, containerOptions, async (container) => {
             await (0, execution_steps_1.setUserPermissions)(item, container, environment);
+            for (const generate of outputsToReset.filter((g) => !g.isFile)) {
+                await (0, execute_docker_1.clearContainerDirectory)(item.status, environment, container, generate.path);
+            }
             for (const cmd of item.data.cmds) {
                 (0, abort_1.checkForAbort)(options.abort);
                 item.status.write('info', `execute cmd "${cmd.cmd}" in container`);
@@ -112,6 +131,9 @@ async function dockerTask(docker, item, environment, options) {
             }
             return true;
         });
+        if (succeeded) {
+            await (0, container_task_state_1.writeContainerTaskState)(environment, item, options.stateKey);
+        }
     }
     catch (e) {
         if (e instanceof abort_1.AbortError) {

package/dist/executer/event-cache.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.cleanCache = exports.storeCache = exports.getArchivePaths = exports.restoreCache = void 0;
+const remove_orphaned_containers_1 = require("../docker/remove-orphaned-containers");
+const execute_docker_1 = require("./execute-docker");
 const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
 const get_cache_directory_1 = require("../optimizer/get-cache-directory");
 const path_1 = require("path");
@@ -90,6 +92,12 @@ async function cleanCache(workTree, environment, options) {
             await service.data.caching.backend.clear(service.id(), environment);
         }
     }
+    if (workTree.environment.type === 'docker') {
+        const removed = await (0, remove_orphaned_containers_1.removeOrphanedContainers)((0, execute_docker_1.getContainerCli)(workTree.environment));
+        for (const id of removed) {
+            environment.console.info(`removed container ${id.substring(0, 12)} left behind by a stopped run`);
+        }
+    }
 }
 exports.cleanCache = cleanCache;
 //# sourceMappingURL=event-cache.js.map

package/dist/executer/execute-command.js

@@ -6,6 +6,12 @@ const os_1 = require("os");
 const log_1 = require("../log");
 const abort_1 = require("./abort");
 const abort_event_1 = require("../utils/abort-event");
+const environment_config_1 = require("../utils/environment-config");
+// how long a local command's process group gets to exit after SIGTERM before
+// it is killed
+function getStopTimeout() {
+    return (0, environment_config_1.getEnvironmentConfig)('HAMMERKIT_STOP_TIMEOUT_MS', 10000);
+}
 async function executeCommand(status, abortSignal, cwd, command, envs, environment) {
     const windows = (0, os_1.platform)() === 'win32';
     return new Promise((resolve, reject) => {
@@ -48,14 +54,38 @@ async function executeCommand(status, abortSignal, cwd, command, envs, environme
                 ps.kill();
                 return;
             }
-            try {
-                process.kill(-ps.pid, 'SIGTERM');
-            }
-            catch {
-                // the group is already gone
-            }
+            stopGroup(ps.pid);
         });
     });
 }
 exports.executeCommand = executeCommand;
+// SIGTERM the whole group, then SIGKILL whatever is still in it after
+// HAMMERKIT_STOP_TIMEOUT_MS. The shell exiting doesn't end the wait: a child that ignores
+// SIGTERM would otherwise outlive hammerkit.
+function stopGroup(pid) {
+    if (!signalGroup(pid, 'SIGTERM')) {
+        return;
+    }
+    const deadline = Date.now() + getStopTimeout();
+    const poll = setInterval(() => {
+        if (!signalGroup(pid, 0)) {
+            clearInterval(poll);
+            return;
+        }
+        if (Date.now() >= deadline) {
+            signalGroup(pid, 'SIGKILL');
+            clearInterval(poll);
+        }
+    }, 100);
+}
+function signalGroup(pid, signal) {
+    try {
+        process.kill(-pid, signal);
+        return true;
+    }
+    catch {
+        // the group is already gone
+        return false;
+    }
+}
 //# sourceMappingURL=execute-command.js.map

package/dist/executer/execute-docker.js

@@ -3,16 +3,22 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.execCommand = exports.convertToPosixPath = exports.startContainer = exports.getContainerCli = void 0;
+exports.clearContainerDirectory = exports.execCommand = exports.convertToPosixPath = exports.startContainer = exports.getContainerCli = exports.createDockerClient = void 0;
 const stream_1 = require("../docker/stream");
 const dockerode_1 = __importDefault(require("dockerode"));
 const path_1 = require("path");
 const os_1 = require("os");
 const abort_event_1 = require("../utils/abort-event");
 let dockerInstance = null;
+// Without options dockerode reads DOCKER_HOST and the TLS variables; passing
+// `{ host: undefined }` would overwrite the host it parsed from DOCKER_HOST.
+function createDockerClient(host) {
+    return host ? new dockerode_1.default({ host }) : new dockerode_1.default();
+}
+exports.createDockerClient = createDockerClient;
 function getContainerCli(workEnvironment) {
     if (!dockerInstance) {
-        dockerInstance = new dockerode_1.default({ host: workEnvironment.host });
+        dockerInstance = createDockerClient(workEnvironment.host);
     }
     return dockerInstance;
 }
@@ -127,4 +133,14 @@ async function execCommand(status, environment, container, cwd, cmd, user, timeo
     });
 }
 exports.execCommand = execCommand;
+// Empty a directory inside a running container, as root. Volumes are emptied
+// this way rather than recreated: Docker refuses to remove a volume while any
+// container (a running dependent's, say) still mounts it.
+async function clearContainerDirectory(status, environment, container, path) {
+    const result = await execCommand(status, environment, container, '/', ['sh', '-c', 'rm -rf "$1"/* "$1"/.[!.]* "$1"/..?*', 'sh', convertToPosixPath(path)], null, undefined, environment.abortCtrl.signal);
+    if (result.type !== 'result' || result.result.ExitCode !== 0) {
+        throw new Error(`unable to empty ${path}`);
+    }
+}
+exports.clearContainerDirectory = clearContainerDirectory;
 //# sourceMappingURL=execute-docker.js.map

package/dist/executer/execute-work-service.js

@@ -32,8 +32,12 @@ async function executeWorkService(work, environment, options) {
             });
             await (0, await_completed_dependencies_1.awaitRunningNeeds)(work, work.needs.map((n) => n.service), abort);
             if (options.type === 'execute') {
+                // rejects when the run is aborted; unhandled, that rejection would
+                // crash hammerkit before the service container is removed
                 (0, await_completed_dependencies_1.awaitNoRequirements)(work, abort).then(() => {
                     stop();
+                }, () => {
+                    // aborted: the service is stopping anyway
                 });
             }
             await work.runtime.execute(environment, {