hammerkit 1.9.2 → 1.11.0

package/build.schema.json

@@ -141,7 +141,15 @@
                           "type": "object",
                           "properties": {
                             "export": {
-                              "type": "boolean"
+                              "anyOf": [
+                                {
+                                  "type": "boolean"
+                                },
+                                {
+                                  "type": "string",
+                                  "const": "always"
+                                }
+                              ]
                             },
                             "resetOnChange": {
                               "type": "boolean"
@@ -151,6 +159,9 @@
                             },
                             "name": {
                               "type": "string"
+                            },
+                            "readOnly": {
+                              "type": "boolean"
                             }
                           },
                           "required": [
@@ -170,6 +181,33 @@
                   "timeout": {
                     "type": "string"
                   },
+                  "secrets": {
+                    "type": "array",
+                    "items": {
+                      "type": "object",
+                      "properties": {
+                        "from": {
+                          "type": "string",
+                          "pattern": "^(env|file):.+$"
+                        },
+                        "env": {
+                          "type": "string",
+                          "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
+                        },
+                        "path": {
+                          "type": "string"
+                        },
+                        "cache": {
+                          "type": "boolean"
+                        }
+                      },
+                      "required": [
+                        "from"
+                      ],
+                      "additionalProperties": false,
+                      "description": "secret from env:NAME or file:path, injected as env or a read-only file"
+                    }
+                  },
                   "image": {
                     "type": "string"
                   },
@@ -228,6 +266,9 @@
                   "timeout": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
                   },
+                  "secrets": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/secrets"
+                  },
                   "platform": {
                     "type": "object",
                     "properties": {
@@ -341,6 +382,16 @@
                       "cmd"
                     ],
                     "additionalProperties": false
+                  },
+                  "init": {
+                    "type": "string",
+                    "description": "task run once the service is healthy, before dependents start"
+                  },
+                  "secrets": {
+                    "type": "array",
+                    "items": {
+                      "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/secrets/items"
+                    }
                   }
                 },
                 "required": [

package/dist/cache/backends/registry/registry-client.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.digestBuffer = exports.digestFile = exports.RegistryClient = exports.parseChallenge = exports.RegistryError = exports.OCI_LAYER_TAR = exports.OCI_CONFIG = exports.OCI_MANIFEST = void 0;
+exports.digestBuffer = exports.digestFile = exports.RegistryClient = exports.getTokenScope = exports.parseChallenge = exports.RegistryError = exports.OCI_LAYER_TAR = exports.OCI_CONFIG = exports.OCI_MANIFEST = void 0;
 const crypto_1 = require("crypto");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
@@ -32,6 +32,23 @@ function parseChallenge(header) {
     return { scheme, params };
 }
 exports.parseChallenge = parseChallenge;
+// Always ask for pull and push, so one token serves the whole backend, plus
+// what the challenge names for this repository: a DELETE is challenged for
+// `delete`, which a pull/push token lacks. A registry grants the subset the
+// credentials allow.
+function getTokenScope(repository, challengeScope) {
+    const actions = ['pull', 'push'];
+    const prefix = `repository:${repository}:`;
+    if (challengeScope?.startsWith(prefix)) {
+        for (const action of challengeScope.substring(prefix.length).split(',')) {
+            if (action && !actions.includes(action)) {
+                actions.push(action);
+            }
+        }
+    }
+    return `${prefix}${actions.join(',')}`;
+}
+exports.getTokenScope = getTokenScope;
 function basicHeader(credentials) {
     return `Basic ${Buffer.from(`${credentials.username}:${credentials.password}`).toString('base64')}`;
 }
@@ -76,9 +93,7 @@ class RegistryClient {
         if (challenge.params.service) {
             tokenUrl.searchParams.set('service', challenge.params.service);
         }
-        // always ask for push too: one token then serves the whole backend; a
-        // registry grants the subset the credentials allow
-        tokenUrl.searchParams.set('scope', `repository:${this.reference.repository}:pull,push`);
+        tokenUrl.searchParams.set('scope', getTokenScope(this.reference.repository, challenge.params.scope ?? null));
         const response = await fetch(tokenUrl, {
             headers: credentials ? { authorization: basicHeader(credentials) } : {},
         });

package/dist/cli.js

@@ -57,6 +57,7 @@ class Cli {
                         cacheReadOnly: options?.cacheReadOnly ?? false,
                         timeout: options?.timeout ?? null,
                         skipDeps: options?.skipDeps ?? true,
+                        wait: options?.wait ?? 'ready',
                     });
                 }
                 const result = (0, get_scheduler_execute_result_1.getSchedulerExecuteResult)(workTree);

package/dist/docker/pull.js

@@ -1,11 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitImageName = exports.isImagePresent = exports.pull = void 0;
+exports.normalizeArchitecture = exports.splitImageName = exports.isImagePresent = exports.pull = void 0;
 const registry_credentials_1 = require("../cache/backends/registry/registry-credentials");
 const registry_reference_1 = require("../cache/backends/registry/registry-reference");
 async function pull(status, docker, imageName, environment) {
     const images = await docker.listImages({});
     if (isImagePresent(images, imageName)) {
+        await warnOnForeignPlatform(status, docker, imageName);
         return;
     }
     status.write('debug', `pull image ${imageName}`);
@@ -49,4 +50,40 @@ async function getPullAuth(imageName, environment) {
     }
     return { ...credentials, serveraddress: credentialHost };
 }
+const daemonArchitectures = new WeakMap();
+const warnedImages = new Set();
+// A digest pins a multi-arch index; a copy pulled earlier for another
+// architecture (an amd64 one from a Compose file with `platform:`) carries the
+// same digest, so docker uses it and the image runs emulated instead of native.
+async function warnOnForeignPlatform(status, docker, imageName) {
+    const { repository, digest } = splitImageName(imageName);
+    if (!digest || warnedImages.has(imageName)) {
+        return;
+    }
+    try {
+        if (!daemonArchitectures.has(docker)) {
+            daemonArchitectures.set(docker, docker.info().then((info) => normalizeArchitecture(info.Architecture ?? '')));
+        }
+        const [image, daemonArchitecture] = await Promise.all([
+            docker.getImage(imageName).inspect(),
+            daemonArchitectures.get(docker),
+        ]);
+        const imageArchitecture = normalizeArchitecture(image.Architecture ?? '');
+        if (!imageArchitecture || !daemonArchitecture || imageArchitecture === daemonArchitecture) {
+            return;
+        }
+        warnedImages.add(imageName);
+        status.write('warn', `image ${imageName} is present for ${imageArchitecture} on this ${daemonArchitecture} docker host and runs ` +
+            `emulated; if the digest is a multi-arch index, remove the copy to pull the native one: ` +
+            `docker image rm ${repository}@${digest}`);
+    }
+    catch {
+        // only a hint: never fail a pull over it
+    }
+}
+function normalizeArchitecture(architecture) {
+    const aliases = { x86_64: 'amd64', aarch64: 'arm64' };
+    return aliases[architecture] ?? architecture;
+}
+exports.normalizeArchitecture = normalizeArchitecture;
 //# sourceMappingURL=pull.js.map

package/dist/environment/replace-env-variables.js

@@ -1,36 +1,45 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getEnvironmentVariables = exports.buildEnvironmentVariables = void 0;
+// `${NAME}` or `${NAME:-default}` anywhere in a value
+const INTERPOLATION = /\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}/g;
+// The shell / process environment first, then the .env files. A variable set to
+// an empty string is set, as in a shell: CI providers leave many empty (a merge
+// request id on a branch pipeline).
+function resolveReference(name, environment, context) {
+    const processValue = environment.processEnvs[name];
+    if (processValue !== undefined && processValue !== null) {
+        return processValue;
+    }
+    for (const envFile of Object.values(context.envFiles)) {
+        const fileValue = envFile[name];
+        if (fileValue !== undefined && fileValue !== null) {
+            return fileValue;
+        }
+    }
+    return null;
+}
 function buildEnvironmentVariables(envs, environment, context) {
     const replacements = [];
     const variables = {};
     for (const [key, value] of Object.entries(envs)) {
-        if (value.startsWith('$')) {
+        if (value.startsWith('$') && !value.startsWith('${')) {
             const name = value.substring(1);
-            let nameValue = environment.processEnvs[name] ?? null;
-            if (!nameValue) {
-                for (const envFile of Object.values(context.envFiles)) {
-                    nameValue = envFile[name] ?? null;
-                    if (nameValue) {
-                        break;
-                    }
-                }
+            const nameValue = resolveReference(name, environment, context);
+            if (nameValue !== null) {
+                replacements.push({ key, name, available: true, value: nameValue });
             }
-            if (nameValue) {
-                replacements.push({
-                    key,
-                    name,
-                    available: true,
-                    value: nameValue,
-                });
+            else {
+                replacements.push({ key, name, available: false, value: null });
+            }
+        }
+        else if (value.includes('${')) {
+            const interpolated = interpolate(value, envs, environment, context);
+            if (interpolated.missing) {
+                replacements.push({ key, name: interpolated.missing, available: false, value: null });
             }
             else {
-                replacements.push({
-                    key,
-                    name,
-                    available: false,
-                    value: null,
-                });
+                variables[key] = interpolated.value;
             }
         }
         else {
@@ -40,6 +49,28 @@ function buildEnvironmentVariables(envs, environment, context) {
     return { variables, replacements };
 }
 exports.buildEnvironmentVariables = buildEnvironmentVariables;
+// Each `${NAME}` comes from the process environment or a .env file, else from a
+// literal value of the same build file / task `envs`, else from its
+// `:-default`. A reference without any of them is missing.
+function interpolate(value, envs, environment, context) {
+    let missing = null;
+    const result = value.replace(INTERPOLATION, (match, name, fallback) => {
+        const resolved = resolveReference(name, environment, context) ?? getLiteral(envs, name) ?? fallback ?? null;
+        if (resolved === null) {
+            missing = missing ?? name;
+            return match;
+        }
+        return resolved;
+    });
+    return { value: result, missing };
+}
+function getLiteral(envs, name) {
+    const literal = envs[name];
+    if (literal === undefined || literal.includes('$')) {
+        return null;
+    }
+    return literal;
+}
 function getEnvironmentVariables(envs) {
     const result = { ...envs.variables };
     for (const replacement of envs.replacements) {

package/dist/executer/container-secrets.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeContainerSecrets = exports.resolveSecretEnvs = exports.prepareContainerSecrets = exports.getSecretDirectory = void 0;
+const path_1 = require("path");
+const promises_1 = require("fs/promises");
+const work_secret_1 = require("../planner/work-secret");
+const get_cache_directory_1 = require("../optimizer/get-cache-directory");
+const execute_docker_1 = require("./execute-docker");
+// Where values of env-sourced file targets are written while their container
+// runs: owner-only, outside the project, removed with the container.
+function getSecretDirectory(instanceId) {
+    return (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'secrets', instanceId);
+}
+exports.getSecretDirectory = getSecretDirectory;
+// Resolve every secret before the container starts, so a missing source fails
+// naming the secret and nothing has run yet.
+async function prepareContainerSecrets(secrets, instanceId, environment) {
+    const result = { env: {}, binds: [] };
+    for (const secret of secrets) {
+        if (secret.target.type === 'env') {
+            result.env[secret.target.name] = await (0, work_secret_1.resolveSecretValue)(secret, environment);
+            continue;
+        }
+        if (secret.source.type === 'file') {
+            await (0, work_secret_1.resolveSecretValue)(secret, environment);
+            result.binds.push(`${secret.source.path}:${(0, execute_docker_1.convertToPosixPath)(secret.target.path)}:ro`);
+            continue;
+        }
+        const directory = getSecretDirectory(instanceId);
+        await (0, promises_1.mkdir)(directory, { recursive: true, mode: 0o700 });
+        await (0, promises_1.chmod)(directory, 0o700);
+        const file = (0, path_1.join)(directory, getSecretFileName(secret));
+        await (0, promises_1.writeFile)(file, await (0, work_secret_1.resolveSecretValue)(secret, environment), { mode: 0o600 });
+        // the mode only applies on create, a leftover file is tightened too
+        await (0, promises_1.chmod)(file, 0o600);
+        result.binds.push(`${file}:${(0, execute_docker_1.convertToPosixPath)(secret.target.path)}:ro`);
+    }
+    return result;
+}
+exports.prepareContainerSecrets = prepareContainerSecrets;
+// A local task only takes env targets, file targets are refused when planning.
+async function resolveSecretEnvs(secrets, environment) {
+    const envs = {};
+    for (const secret of secrets) {
+        if (secret.target.type === 'env') {
+            envs[secret.target.name] = await (0, work_secret_1.resolveSecretValue)(secret, environment);
+        }
+    }
+    return envs;
+}
+exports.resolveSecretEnvs = resolveSecretEnvs;
+async function removeContainerSecrets(instanceId) {
+    await (0, promises_1.rm)(getSecretDirectory(instanceId), { recursive: true, force: true });
+}
+exports.removeContainerSecrets = removeContainerSecrets;
+function getSecretFileName(secret) {
+    return (0, work_secret_1.getSecretName)(secret).replace(/[^A-Za-z0-9._-]+/g, '_');
+}
+//# sourceMappingURL=container-secrets.js.map

package/dist/executer/deadline.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withDeadline = void 0;
+const abort_event_1 = require("../utils/abort-event");
+// An abort signal that follows `parent` and additionally fires after `timeout`
+// ms, so a task deadline reuses the runtimes' existing abort/cleanup path.
+function withDeadline(parent, timeout) {
+    const controller = new AbortController();
+    let expired = false;
+    const parentListener = (0, abort_event_1.listenOnAbort)(parent, () => controller.abort());
+    const timer = timeout === null
+        ? null
+        : setTimeout(() => {
+            expired = true;
+            controller.abort();
+        }, timeout);
+    return {
+        signal: controller.signal,
+        expired: () => expired,
+        clear() {
+            if (timer) {
+                clearTimeout(timer);
+            }
+            parentListener.close();
+        },
+    };
+}
+exports.withDeadline = withDeadline;
+//# sourceMappingURL=deadline.js.map

package/dist/executer/docker-service.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.dockerService = void 0;
+exports.dockerService = exports.buildServiceCreateOptions = void 0;
 const work_service_1 = require("../planner/work-service");
 const run_labels_1 = require("../docker/run-labels");
+const service_hostname_1 = require("../planner/utils/service-hostname");
+const service_definition_1 = require("../planner/service-definition");
 const abort_1 = require("./abort");
 const execute_docker_1 = require("./execute-docker");
 const stream_1 = require("../docker/stream");
@@ -13,11 +15,59 @@ const check_readiness_1 = require("./check-readiness");
 const main_process_user_1 = require("./main-process-user");
 const execution_steps_1 = require("./execution-steps");
 const docker_task_1 = require("./docker-task");
+const container_secrets_1 = require("./container-secrets");
 const replace_env_variables_1 = require("../environment/replace-env-variables");
 const get_service_containers_1 = require("./get-service-containers");
 const work_instance_id_1 = require("../planner/work-instance-id");
 const work_healthcheck_1 = require("../planner/work-healthcheck");
 const HEALTHCHECK_INTERVAL_MS = 1000;
+function buildServiceCreateOptions(item, options, network, secrets) {
+    const envs = (0, replace_env_variables_1.getEnvironmentVariables)(item.data.envs);
+    const command = (0, work_service_1.getServiceCommand)(item.data);
+    return {
+        Image: item.data.image,
+        Hostname: (0, service_hostname_1.getServiceHostname)(item.name),
+        Env: Object.entries({ ...envs, ...secrets.env }).map(([key, value]) => `${key}=${value}`),
+        Labels: {
+            app: 'hammerkit',
+            'hammerkit-id': (0, work_instance_id_1.getWorkInstanceId)(item),
+            ...(0, run_labels_1.getRunLabels)(),
+            'hammerkit-type': 'service',
+            'hammerkit-state': options.stateKey,
+            'hammerkit-daemon': options.daemon ? 'true' : 'false',
+            'hammerkit-definition': (0, service_definition_1.getServiceDefinitionHash)(item),
+        },
+        ExposedPorts: item.data.ports.reduce((map, port) => {
+            map[`${port.containerPort}/tcp`] = {};
+            return map;
+        }, {}),
+        Entrypoint: command.entrypoint ?? undefined,
+        Cmd: command.cmd ?? undefined,
+        WorkingDir: (0, execute_docker_1.convertToPosixPath)((0, work_service_1.getServiceWorkingDir)(item.data)),
+        HostConfig: {
+            ExtraHosts: network.hosts,
+            Links: network.links,
+            Binds: [
+                ...item.data.src.map((s) => `${s.absolutePath}:${s.absolutePath}`),
+                ...item.data.mounts.map((v) => `${v.localPath}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}${v.readOnly ? ':ro' : ''}`),
+                ...item.data.volumes.map((v) => `${v.name}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}${v.readOnly ? ':ro' : ''}`),
+                ...secrets.binds,
+            ],
+            // host ports are for `hammerkit up` and local tasks; container tasks
+            // reach the service over its link, and a run publishing them would
+            // collide with another run on the same host
+            PortBindings: options.publishPorts
+                ? item.data.ports
+                    .filter((p) => !!p.hostPort)
+                    .reduce((map, port) => {
+                    map[`${port.containerPort}/tcp`] = [{ HostPort: `${port.hostPort}` }];
+                    return map;
+                }, {})
+                : {},
+        },
+    };
+}
+exports.buildServiceCreateOptions = buildServiceCreateOptions;
 async function dockerService(docker, item, options, environment) {
     let container = null;
     try {
@@ -30,46 +80,12 @@ async function dockerService(docker, item, options, environment) {
         const serviceContainers = (0, get_service_containers_1.getServiceContainers)(item.needs);
         const network = (0, docker_task_1.getNeedsNetwork)(serviceContainers, item.needs);
         item.status.write('debug', `create container with image ${item.data.image}`);
-        const envs = (0, replace_env_variables_1.getEnvironmentVariables)(item.data.envs);
-        const command = (0, work_service_1.getServiceCommand)(item.data);
-        container = await docker.createContainer({
-            Image: item.data.image,
-            Env: Object.keys(envs).map((k) => `${k}=${envs[k]}`),
-            Labels: {
-                app: 'hammerkit',
-                'hammerkit-id': (0, work_instance_id_1.getWorkInstanceId)(item),
-                ...(0, run_labels_1.getRunLabels)(),
-                'hammerkit-type': 'service',
-                'hammerkit-state': options.stateKey,
-                'hammerkit-daemon': options.daemon ? 'true' : 'false',
-            },
-            ExposedPorts: item.data.ports.reduce((map, port) => {
-                map[`${port.containerPort}/tcp`] = {};
-                return map;
-            }, {}),
-            Entrypoint: command.entrypoint ?? undefined,
-            Cmd: command.cmd ?? undefined,
-            WorkingDir: (0, execute_docker_1.convertToPosixPath)((0, work_service_1.getServiceWorkingDir)(item.data)),
-            HostConfig: {
-                ExtraHosts: network.hosts,
-                Links: network.links,
-                Binds: [
-                    ...item.data.src.map((s) => `${s.absolutePath}:${s.absolutePath}`),
-                    ...item.data.mounts.map((v) => `${v.localPath}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}`),
-                    ...item.data.volumes.map((v) => `${v.name}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}`),
-                ],
-                PortBindings: item.data.ports
-                    .filter((p) => !!p.hostPort)
-                    .reduce((map, port) => {
-                    map[`${port.containerPort}/tcp`] = [{ HostPort: `${port.hostPort}` }];
-                    return map;
-                }, {}),
-            },
-        });
+        const secrets = await (0, container_secrets_1.prepareContainerSecrets)(item.data.secrets, (0, work_instance_id_1.getWorkInstanceId)(item), environment);
+        container = await docker.createContainer(buildServiceCreateOptions(item, options, network, secrets));
         const stream = await container.attach({ stream: true, stdout: true, stderr: true });
         (0, stream_1.logStream)(item.status, stream);
         await container.start();
-        if (!item.data.healthcheck) {
+        if (!item.data.healthcheck || !options.waitForReady) {
             options.state.set({
                 type: 'running',
                 dns: { containerId: container.id },
@@ -149,6 +165,10 @@ async function dockerService(docker, item, options, environment) {
                 item.status.write('error', `remove of container failed ${(0, log_1.getErrorMessage)(e)}`);
             }
         }
+        // a daemon keeps its secret files until `hammerkit down` stops it
+        if (!options.daemon) {
+            await (0, container_secrets_1.removeContainerSecrets)((0, work_instance_id_1.getWorkInstanceId)(item));
+        }
     }
 }
 exports.dockerService = dockerService;

package/dist/executer/docker-task.js

@@ -15,6 +15,7 @@ const get_container_binds_1 = require("./get-container-binds");
 const get_service_containers_1 = require("./get-service-containers");
 const work_instance_id_1 = require("../planner/work-instance-id");
 const get_outputs_to_reset_1 = require("../planner/utils/get-outputs-to-reset");
+const container_secrets_1 = require("./container-secrets");
 const container_task_state_1 = require("./container-task-state");
 function getNeedsNetwork(serviceContainers, needs) {
     const links = [];
@@ -34,7 +35,7 @@ function getNeedsNetwork(serviceContainers, needs) {
     return { links, hosts };
 }
 exports.getNeedsNetwork = getNeedsNetwork;
-function buildCreateOptions(item, stateKey, serviceContainers, environment) {
+function buildCreateOptions(item, stateKey, serviceContainers, environment, secrets) {
     const network = getNeedsNetwork(serviceContainers, item.needs);
     const binds = (0, get_container_binds_1.getContainerBinds)(item);
     const envs = (0, replace_env_variables_1.getEnvironmentVariables)(item.data.envs);
@@ -48,7 +49,7 @@ function buildCreateOptions(item, stateKey, serviceContainers, environment) {
         Tty: true,
         Entrypoint: [item.data.shell],
         Cmd: ['-c', 'sleep 3600'],
-        Env: Object.entries({ ...envs, ...home }).map(([key, value]) => `${key}=${value}`),
+        Env: Object.entries({ ...envs, ...home, ...secrets.env }).map(([key, value]) => `${key}=${value}`),
         WorkingDir: (0, execute_docker_1.convertToPosixPath)(item.data.cwd),
         Labels: {
             app: 'hammerkit',
@@ -58,7 +59,10 @@ function buildCreateOptions(item, stateKey, serviceContainers, environment) {
             'hammerkit-state': stateKey,
         },
         HostConfig: {
-            Binds: binds.map((b) => `${b.localPath}:${(0, execute_docker_1.convertToPosixPath)(b.containerPath)}`),
+            Binds: [
+                ...binds.map((b) => `${b.localPath}:${(0, execute_docker_1.convertToPosixPath)(b.containerPath)}${b.readOnly ? ':ro' : ''}`),
+                ...secrets.binds,
+            ],
             ExtraHosts: network.hosts,
             Links: network.links,
             AutoRemove: true,
@@ -84,7 +88,8 @@ async function dockerTask(docker, item, environment, options) {
         await (0, execution_steps_1.prepareVolume)(item, docker);
         (0, abort_1.checkForAbort)(options.abort);
         const serviceContainers = (0, get_service_containers_1.getServiceContainers)(item.needs);
-        const containerOptions = buildCreateOptions(item, options.stateKey, serviceContainers, environment);
+        const secrets = await (0, container_secrets_1.prepareContainerSecrets)(item.data.secrets, (0, work_instance_id_1.getWorkInstanceId)(item), environment);
+        const containerOptions = buildCreateOptions(item, options.stateKey, serviceContainers, environment, secrets);
         (0, print_container_options_1.printContainerOptions)(item.status, containerOptions);
         const succeeded = await (0, using_container_1.usingContainer)(docker, item, containerOptions, async (container) => {
             await (0, execution_steps_1.setUserPermissions)(item, container, environment);
@@ -102,6 +107,7 @@ async function dockerTask(docker, item, environment, options) {
                     throw new abort_1.AbortError();
                 }
                 if (result.result.ExitCode !== 0) {
+                    await exportGenerates(environment, container, item.data.generates.filter((g) => g.exportAlways));
                     options.state.set({
                         stateKey: options.stateKey,
                         type: 'crash',
@@ -110,25 +116,7 @@ async function dockerTask(docker, item, environment, options) {
                     return false;
                 }
             }
-            for (const generate of item.data.generates) {
-                if (!generate.export || generate.inherited || generate.isFile) {
-                    continue;
-                }
-                const readable = await container.getArchive({
-                    path: generate.path,
-                });
-                await environment.file.createDirectory(generate.path);
-                await new Promise((resolve, reject) => {
-                    readable
-                        .pipe((0, tar_1.extract)({
-                        cwd: generate.path,
-                        newer: true,
-                        stripComponents: 1,
-                    }))
-                        .on('close', () => resolve())
-                        .on('error', (err) => reject(err));
-                });
-            }
+            await exportGenerates(environment, container, item.data.generates.filter((g) => g.export));
             return true;
         });
         if (succeeded) {
@@ -150,6 +138,32 @@ async function dockerTask(docker, item, environment, options) {
             });
         }
     }
+    finally {
+        await (0, container_secrets_1.removeContainerSecrets)((0, work_instance_id_1.getWorkInstanceId)(item));
+    }
 }
 exports.dockerTask = dockerTask;
+// Copy exported directory outputs out of the container to the host. File
+// outputs are bind mounts and inherited outputs belong to their task.
+async function exportGenerates(environment, container, generates) {
+    for (const generate of generates) {
+        if (generate.inherited || generate.isFile) {
+            continue;
+        }
+        const readable = await container.getArchive({
+            path: generate.path,
+        });
+        await environment.file.createDirectory(generate.path);
+        await new Promise((resolve, reject) => {
+            readable
+                .pipe((0, tar_1.extract)({
+                cwd: generate.path,
+                newer: true,
+                stripComponents: 1,
+            }))
+                .on('close', () => resolve())
+                .on('error', (err) => reject(err));
+        });
+    }
+}
 //# sourceMappingURL=docker-task.js.map