hammerkit 1.27.0-next.3 → 1.27.0-next.30

package/dist/kubernetes/cluster-state.js

@@ -0,0 +1,245 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readClusterState = exports.describeEvent = exports.describePod = exports.getPodWaiting = exports.describeLastExit = exports.getPodFailure = exports.CRASHES_TO_FAIL = void 0;
+const log_1 = require("../log");
+const apply_1 = require("./apply");
+const job_placement_1 = require("./job-placement");
+const network_gate_1 = require("./network-gate");
+const resources_1 = require("./resources");
+// what a pod's or event's message is cut to
+const MAX_MESSAGE = 500;
+// how many events of a run are said, the newest
+const MAX_EVENTS = 20;
+// events the result of a task says better: a Job of a failed command
+const SAID_ELSEWHERE = ['BackoffLimitExceeded'];
+// A container restarted this often (ended three times) is taken as one that keeps crashing,
+// unless its service sets `restarts`: once may be a race with a dependency or a slow first start
+exports.CRASHES_TO_FAIL = 2;
+// what a pull error says when the image is not there to pull, or not for this cluster:
+// waiting does not change it. Others (a registry that limits, a token not there yet) may pass.
+const IMAGE_GONE = /not found|manifest unknown|unauthorized|denied|forbidden|invalid reference|no such host/i;
+/**
+ * Why a pod of a service will not become ready without a change, as the cluster says it:
+ * an image that can not be pulled, a container that keeps crashing or can not be created,
+ * an init container that failed. Null while it may still get there. A container fails once
+ * it was restarted `restarts` times and crashed again.
+ *
+ * Only what the cluster wrote is quoted: the termination message of a container is its
+ * own output (and may hold a value it read), except for the network gate's, which says why
+ * the policy did not take hold.
+ */
+function getPodFailure(pod, container, restarts = exports.CRASHES_TO_FAIL) {
+    for (const status of pod.status?.initContainerStatuses ?? []) {
+        const terminated = status.state?.terminated ?? status.lastState?.terminated;
+        if (!terminated || terminated.exitCode === 0) {
+            continue;
+        }
+        if (status.name === network_gate_1.NETWORK_GATE_CONTAINER) {
+            return {
+                text: `the network policy did not take hold: ${cut(terminated.message?.trim() || 'the gate failed')}`,
+                final: true,
+            };
+        }
+        if (status.restartCount >= exports.CRASHES_TO_FAIL) {
+            return { text: `its init container ${status.name} failed with exit code ${terminated.exitCode}`, final: true };
+        }
+    }
+    for (const status of mainStatuses(pod, container)) {
+        const failure = getContainerFailure(status, restartsEnded(pod, status), restarts);
+        if (failure) {
+            return failure;
+        }
+    }
+    return null;
+}
+exports.getPodFailure = getPodFailure;
+function getContainerFailure(status, restartsEnded, restarts) {
+    const waiting = status.state?.waiting;
+    const reason = waiting?.reason;
+    const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
+    switch (reason) {
+        // the first pull: final only when the registry said the image is not there, as older
+        // kubelets leave that out of the ImagePullBackOff that follows
+        case 'ErrImagePull':
+            return IMAGE_GONE.test(waiting?.message ?? '')
+                ? { text: `its image ${status.image} could not be pulled (${reason})${said}`, final: true }
+                : null;
+        case 'ImagePullBackOff':
+            return {
+                text: `its image ${status.image} could not be pulled (${reason})${said}`,
+                final: IMAGE_GONE.test(waiting?.message ?? ''),
+            };
+        case 'ErrImageNeverPull':
+        case 'InvalidImageName':
+            return { text: `its image ${status.image} could not be pulled (${reason})${said}`, final: true };
+        case 'CreateContainerConfigError':
+        case 'CreateContainerError':
+        case 'RunContainerError':
+            return { text: `its container could not be started (${reason})${said}`, final: false };
+        case 'CrashLoopBackOff':
+            return status.restartCount >= restarts
+                ? { text: `it keeps crashing: ${describeLastExit(status)}`, final: true }
+                : null;
+    }
+    // a kubelet may leave an instance that ended as it is until its back-off is over, and say
+    // CrashLoopBackOff only when it looks at the pod again, a minute later: the same crash
+    if (restartsEnded && status.state?.terminated && status.restartCount >= restarts) {
+        return { text: `it keeps crashing: ${describeLastExit(status)}`, final: true };
+    }
+    return null;
+}
+// How the last instance of a container ended, as the cluster says it.
+function describeLastExit(status) {
+    // the instance that ended now, else the one before the one that waits or runs
+    const last = status.state?.terminated ?? status.lastState?.terminated;
+    const restarts = `, restarted ${status.restartCount} time${status.restartCount === 1 ? '' : 's'}`;
+    if (!last) {
+        return `it ended${restarts}`;
+    }
+    if (last.reason === 'OOMKilled') {
+        return `it was killed for using more memory than its limit (OOMKilled)${restarts}`;
+    }
+    // the kubelet kills a container whose liveness probe (its healthcheck) keeps failing
+    const killed = last.exitCode === 137 || last.exitCode === 143 ? ', killed as when its healthcheck kept failing' : '';
+    return `it exited with code ${last.exitCode}${last.reason ? ` (${last.reason})` : ''}${killed}${restarts}`;
+}
+exports.describeLastExit = describeLastExit;
+/**
+ * What a pod of a service waits for, in a few words, while it is not ready: a node, its
+ * image, its container. Undefined when it waits for nothing the cluster names.
+ */
+function getPodWaiting(pod, container) {
+    const scheduled = pod.status?.conditions?.find((condition) => condition.type === 'PodScheduled');
+    if (scheduled?.status === 'False') {
+        return `waits for a node: ${cut(scheduled.message ?? scheduled.reason ?? 'not scheduled yet')}`;
+    }
+    for (const status of mainStatuses(pod, container)) {
+        const waiting = status.state?.waiting;
+        const said = waiting?.message ? `: ${cut(waiting.message)}` : '';
+        switch (waiting?.reason) {
+            case 'ErrImagePull':
+            case 'ImagePullBackOff':
+                return `can not pull its image ${status.image} yet${said}`;
+            case 'CreateContainerConfigError':
+            case 'CreateContainerError':
+            case 'RunContainerError':
+                return `can not start its container yet (${waiting.reason})${said}`;
+            case 'CrashLoopBackOff':
+                return `crashed and starts again: ${describeLastExit(status)}`;
+        }
+        const ended = status.state?.terminated;
+        if (ended && restartsEnded(pod, status)) {
+            return `${ended.exitCode === 0 ? 'ended' : 'crashed'} and starts again: ${describeLastExit(status)}`;
+        }
+        if (status.restartCount > 0 && status.state?.running && !status.ready) {
+            return `restarted: ${describeLastExit(status)}`;
+        }
+    }
+    return undefined;
+}
+exports.getPodWaiting = getPodWaiting;
+// Whether the kubelet starts a container that ended again: not one of a pod that finished (a
+// task's Job, an evicted pod, which its ReplicaSet replaces) or that never restarts.
+function restartsEnded(pod, status) {
+    const phase = pod.status?.phase;
+    if (phase === 'Succeeded' || phase === 'Failed') {
+        return false;
+    }
+    const policy = pod.spec?.restartPolicy ?? 'Always';
+    return policy === 'Always' || (policy === 'OnFailure' && status.state?.terminated?.exitCode !== 0);
+}
+function mainStatuses(pod, container) {
+    return (pod.status?.containerStatuses ?? []).filter((status) => !container || status.name === container);
+}
+// One line of a pod that is not as it should be, or null for one that runs, ready, or ended well.
+function describePod(pod) {
+    const name = pod.metadata?.name ?? '?';
+    const phase = pod.status?.phase ?? 'Unknown';
+    const failure = getPodFailure(pod)?.text;
+    const waiting = getPodWaiting(pod);
+    const unready = (pod.status?.containerStatuses ?? [])
+        .filter((status) => !status.ready && !status.state?.terminated)
+        .map((status) => status.name);
+    const crashed = (pod.status?.containerStatuses ?? []).filter((status) => (status.state?.terminated && status.state.terminated.exitCode !== 0) || status.restartCount > 0);
+    if (!failure && !waiting && crashed.length === 0 && (phase === 'Succeeded' || unready.length === 0)) {
+        return null;
+    }
+    const said = failure ?? waiting;
+    const parts = [phase, ...(said ? [said] : [])];
+    for (const status of crashed) {
+        if (!said?.includes(describeLastExit(status))) {
+            parts.push(`${status.name}: ${describeLastExit(status)}`);
+        }
+    }
+    if (!said && unready.length > 0) {
+        parts.push(`not ready: ${unready.join(', ')}`);
+    }
+    return `pod ${name}: ${parts.join(', ')}`;
+}
+exports.describePod = describePod;
+/**
+ * The message of a warning event, as far as it is the cluster's: a failed probe says the
+ * output of the command, which is the service's, and is left out.
+ */
+function describeEvent(event) {
+    const object = event.involvedObject;
+    const message = (event.message ?? '').replace(/^((Readiness|Liveness|Startup) probe (failed|errored)).*$/s, '$1');
+    const count = (event.count ?? 1) > 1 ? ` (${event.count} times)` : '';
+    return `${event.reason ?? 'Warning'} ${object.kind?.toLowerCase() ?? 'object'}/${object.name ?? '?'}: ${cut(message.trim())}${count}`;
+}
+exports.describeEvent = describeEvent;
+/**
+ * What the cluster said of a run that failed, into its log before the server removes its
+ * objects: its pods that are not as they should be, and the warning events of its objects.
+ * In the namespace of a service run everything is the run's; in a caller's namespace, which
+ * runs share, only the objects of the run's items (named by their key) are.
+ */
+async function readClusterState(instance, kubernetes, workTree, ownNamespace) {
+    const namespace = kubernetes.namespace;
+    const keys = [...Object.values(workTree.tasks), ...Object.values(workTree.services)].map((item) => (0, resources_1.getItemKey)(item, kubernetes));
+    const ours = (name) => ownNamespace || keys.some((key) => name?.includes(key));
+    const lines = [];
+    try {
+        const labels = (0, job_placement_1.getRunLabels)(kubernetes);
+        const labelSelector = Object.entries(labels)
+            .map(([key, value]) => `${key}=${value}`)
+            .join(',');
+        const pods = await instance.coreApi.listNamespacedPod({
+            namespace,
+            ...(labelSelector && !ownNamespace ? { labelSelector } : {}),
+        });
+        for (const pod of pods.items) {
+            const line = ours(pod.metadata?.name) ? describePod(pod) : null;
+            if (line) {
+                lines.push(line);
+            }
+        }
+    }
+    catch (e) {
+        lines.push(`its pods could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+    try {
+        const events = await instance.coreApi.listNamespacedEvent({ namespace, fieldSelector: 'type=Warning' });
+        const warnings = events.items
+            .filter((event) => ours(event.involvedObject.name) && !SAID_ELSEWHERE.includes(event.reason ?? ''))
+            .sort((a, b) => eventTime(a) - eventTime(b))
+            .slice(-MAX_EVENTS);
+        lines.push(...warnings.map((event) => `event ${describeEvent(event)}`));
+    }
+    catch (e) {
+        lines.push((0, apply_1.statusCodeOf)(e) === 403
+            ? `its events could not be read: the runRole of the server needs list on events (see the backend docs)`
+            : `its events could not be read: ${(0, log_1.getErrorMessage)(e)}`);
+    }
+    return lines;
+}
+exports.readClusterState = readClusterState;
+function eventTime(event) {
+    const time = event.lastTimestamp ?? event.eventTime ?? event.firstTimestamp ?? event.metadata?.creationTimestamp;
+    return time ? new Date(time).getTime() : 0;
+}
+function cut(text) {
+    const line = text.replace(/\s+/g, ' ');
+    return line.length > MAX_MESSAGE ? `${line.slice(0, MAX_MESSAGE)}…` : line;
+}
+//# sourceMappingURL=cluster-state.js.map

package/dist/kubernetes/ensure-http-route.js

@@ -36,7 +36,7 @@ async function ensureHttpRoute(instance, env, route, service) {
                     ],
                     backendRefs: [
                         {
-                            name: (0, resources_1.getResourceName)(service),
+                            name: (0, resources_1.getResourceName)(service, env),
                             port: route.servicePort ?? service.data.ports[0].containerPort,
                         },
                     ],

package/dist/kubernetes/ensure-kubernetes-deployment-exists.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
+exports.getServiceContainerName = exports.MIN_READY_SECONDS = exports.ensureKubernetesDeploymentExists = exports.DEFINITION_LABEL = void 0;
 const service_definition_1 = require("../planner/service-definition");
 const work_service_1 = require("../planner/work-service");
 const volumes_1 = require("./volumes");
@@ -17,11 +17,13 @@ const network_policy_1 = require("./network-policy");
 const egress_proxy_1 = require("../network/egress-proxy");
 // what the service was created from, to tell a changed one
 exports.DEFINITION_LABEL = 'hammerkit.dev/definition';
+// how often a starting container is checked: as the liveness probe
+const STARTUP_PERIOD_SECONDS = 5;
 async function ensureKubernetesDeploymentExists(instance, env, service, persistence, secrets, stateKey, 
 // the needed services by name, and the proxy of a limited network
 network) {
     const envs = (0, replace_env_variables_1.getEnvironmentVariables)(service.data.envs);
-    const name = (0, resources_1.getResourceName)(service);
+    const name = (0, resources_1.getResourceName)(service, env);
     const healthcheckCommand = (0, work_service_1.getHealthcheckCommand)(service.data);
     const serviceCommand = (0, work_service_1.getServiceCommand)(service.data, { kubernetes: true });
     const probe = healthcheckCommand
@@ -34,6 +36,16 @@ network) {
             failureThreshold: 3,
         }
         : undefined;
+    // the healthcheck's `timeout` for each start: until it passed once, the liveness probe
+    // doesn't kill a container that starts slowly, and the wait fails it with the timeout
+    const startupProbe = probe && service.data.healthcheck
+        ? {
+            ...probe,
+            // a check more than the timeout holds, and one more for the first check's offset: the
+            // wait fails it with the timeout before the kubelet restarts it
+            failureThreshold: Math.ceil(service.data.healthcheck.timeout / (STARTUP_PERIOD_SECONDS * 1000)) + 2,
+        }
+        : undefined;
     const server = env.server;
     const placement = server ? (0, job_placement_1.getPodPlacement)(server, undefined) : null;
     const workspace = (0, volumes_1.getWorkspacePodSpec)(env, persistence);
@@ -47,25 +59,24 @@ network) {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
             labels: {
-                'hammerkit.dev/id': service.id(),
+                ...(0, resources_1.getItemLabels)(service, env),
                 'hammerkit.dev/state': stateKey,
-                ...(0, job_placement_1.getRunLabels)(env),
                 [exports.DEFINITION_LABEL]: (0, service_definition_1.getServiceDefinitionHash)(service),
             },
         },
         spec: {
             selector: {
-                matchLabels: {
-                    'hammerkit.dev/id': service.id(),
-                },
+                matchLabels: (0, resources_1.getItemLabels)(service, env),
             },
             replicas: 1,
+            // without a healthcheck a pod is ready as soon as its container runs, also one that
+            // exits a second later: it counts once it ran this long without a restart
+            minReadySeconds: probe ? 0 : exports.MIN_READY_SECONDS,
             template: {
                 metadata: {
                     labels: {
-                        'hammerkit.dev/id': service.id(),
+                        ...(0, resources_1.getItemLabels)(service, env),
                         'hammerkit.dev/state': stateKey,
-                        ...(0, job_placement_1.getRunLabels)(env),
                         ...workspace.labels,
                         ...(0, network_policy_1.getNetworkLabels)(env, service),
                     },
@@ -87,7 +98,7 @@ network) {
                         : {}),
                     containers: [
                         {
-                            name: service.name.replace(/:/, '-'),
+                            name: getServiceContainerName(service),
                             image: service.data.image,
                             workingDir: (0, work_service_1.getServiceWorkingDir)(service.data),
                             env: [
@@ -104,6 +115,7 @@ network) {
                             })),
                             readinessProbe: probe,
                             livenessProbe: probe,
+                            startupProbe,
                             ...(0, container_resources_1.getKubernetesResources)(service.data.resources),
                             volumeMounts: [...persistence.mounts.map((m) => m.mount), ...secrets.mounts],
                             ...(0, job_placement_1.getContainerSecurity)(env),
@@ -123,18 +135,29 @@ network) {
         },
     };
     const deploy = await (0, apply_1.apply)(instance, deployment);
-    if (deploy.status?.readyReplicas ?? 0 > 0) {
+    if ((deploy.status?.availableReplicas ?? 0) > 0) {
         return;
     }
     await (0, await_running_state_1.awaitDeployRunningState)(instance, env, name, {
-        podSelector: `hammerkit.dev/id=${service.id()}`,
+        podSelector: (0, resources_1.getItemSelector)(service, env),
         timeout: service.data.healthcheck?.timeout ?? null,
         timeoutMessage: service.data.healthcheck
             ? (0, work_healthcheck_1.getHealthcheckTimeoutMessage)(service.name, service.data.healthcheck)
             : '',
+        container: getServiceContainerName(service),
+        subject: `service ${service.name}`,
+        onWaiting: (text) => service.status.write('warn', `${service.name} ${text}`),
+        ...(service.data.restarts !== null ? { restarts: service.data.restarts } : {}),
     });
 }
 exports.ensureKubernetesDeploymentExists = ensureKubernetesDeploymentExists;
+// how long the container of a service without a healthcheck has to run before it counts as up
+exports.MIN_READY_SECONDS = 10;
+// The container of a service's pod that runs its command, beside the `debug` one.
+function getServiceContainerName(service) {
+    return service.name.replace(/:/, '-');
+}
+exports.getServiceContainerName = getServiceContainerName;
 // Kubernetes `command` replaces the image entrypoint and `args` its command.
 // A shell service sets both; an exec-form `cmd` keeps the established mapping
 // (first token as `command`, the rest as `args`).

package/dist/kubernetes/ensure-kubernetes-persistent-volume-claim-exists.js

@@ -65,7 +65,7 @@ async function ensureKubernetesPersistentVolumeClaimExists(instance, env, volume
             },
         },
     };
-    service.status.console('stdout', `ensure volume ${volume.name}`);
+    service.status.write('debug', `ensure volume ${volume.name}`);
     // Don't block on the PVC reaching "Bound". Storage classes with
     // volumeBindingMode WaitForFirstConsumer (docker-desktop's "standard",
     // k3d/k3s local-path, most cloud defaults) keep the claim Pending until a pod

package/dist/kubernetes/ensure-kubernetes-service-exists.js

@@ -4,11 +4,10 @@ exports.ensureKubernetesServiceExists = void 0;
 const apply_1 = require("./apply");
 const resources_1 = require("./resources");
 const version_1 = require("../version");
-const job_placement_1 = require("./job-placement");
 // A service without ports gets a headless Service: a ClusterIP one needs a port,
 // and its tasks reach every port of the pod, as on docker.
 async function ensureKubernetesServiceExists(instance, env, service) {
-    const name = (0, resources_1.getResourceName)(service);
+    const name = (0, resources_1.getResourceName)(service, env);
     const headless = service.data.ports.length === 0;
     const svc = {
         kind: 'Service',
@@ -19,21 +18,19 @@ async function ensureKubernetesServiceExists(instance, env, service) {
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
-            labels: {
-                'hammerkit.dev/id': service.id(),
-                ...(0, job_placement_1.getRunLabels)(env),
-            },
+            labels: (0, resources_1.getItemLabels)(service, env),
         },
         spec: {
             type: 'ClusterIP',
             ...(headless ? { clusterIP: 'None' } : {}),
-            selector: {
-                'hammerkit.dev/id': service.id(),
-            },
+            // the pods of the service of this run, on a server where runs share the namespace
+            selector: (0, resources_1.getItemLabels)(service, env),
             ...(headless
                 ? {}
                 : {
+                    // a Service of more than one port needs a name for each
                     ports: service.data.ports.map((port) => ({
+                        name: `tcp-${port.hostPort ?? port.containerPort}`,
                         port: port.hostPort ?? port.containerPort,
                         targetPort: port.containerPort,
                     })),

package/dist/kubernetes/ensure-persistent-data.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensurePersistentData = exports.clearClaim = exports.getPodForPersistence = void 0;
+exports.ensurePersistentData = exports.packSource = exports.clearClaim = exports.getPodForPersistence = void 0;
 const apply_1 = require("./apply");
 const path_1 = require("path");
-const tar_1 = require("tar");
+const tar_1 = require("../utils/tar");
 const resources_1 = require("./resources");
 const await_running_state_1 = require("./await-running-state");
 const sleep_1 = require("../utils/sleep");
@@ -67,17 +67,14 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
     for (const volume of persistence.volumes) {
         await (0, ensure_kubernetes_persistent_volume_claim_exists_1.ensureKubernetesPersistentVolumeClaimExists)(instance, env, volume, service, persistence.sizes[volume.name], persistence.kept.includes(volume.name));
     }
-    const name = (0, resources_1.getResourceName)(service, type === 'write' ? '-upload' : '-download');
+    const name = (0, resources_1.getResourceName)(service, env, type === 'write' ? '-upload' : '-download');
     const podSpec = {
         kind: 'Pod',
         apiVersion: 'v1',
         metadata: {
             namespace: env.namespace,
             name,
-            labels: {
-                'hammerkit.dev/id': service.id(),
-                ...(0, job_placement_1.getRunLabels)(env),
-            },
+            labels: (0, resources_1.getItemLabels)(service, env),
             annotations: {
                 'hammerkit.dev/version': (0, version_1.getVersion)(),
             },
@@ -104,18 +101,18 @@ async function getPodForPersistence(instance, env, service, persistence, type, f
             volumes: persistence.volumes,
         },
     };
-    service.status.console('stdout', `create pod ${name}`);
+    service.status.write('debug', `create pod ${name}`);
     await removeStalePod(instance, env.namespace, name);
     const pod = await (0, apply_1.apply)(instance, podSpec);
     if (pod.status?.phase !== 'Running') {
-        service.status.console('stdout', `await running state ${name}`);
+        service.status.write('debug', `await running state ${name}`);
         await (0, await_running_state_1.awaitRunningState)(instance, env, name, 'Running');
     }
     try {
         await fn(name);
     }
     finally {
-        service.status.console('stdout', `delete pod ${name}`);
+        service.status.write('debug', `delete pod ${name}`);
         await instance.coreApi.deleteNamespacedPod({ name, namespace: env.namespace });
     }
 }
@@ -124,7 +121,7 @@ exports.getPodForPersistence = getPodForPersistence;
 // the task runs on it or outputs are restored into it, so it holds what a new one would
 async function clearClaim(instance, env, work, pod) {
     const targetPath = `/dev/hammerkit/${work.id()}`;
-    work.status.console('stdout', `clear what an earlier run left in ${targetPath}`);
+    work.status.write('info', `clear what an earlier run left in ${targetPath}`);
     let failure = null;
     const res = await instance.exec.exec(env.namespace, pod, 'volume', ['find', targetPath, '-mindepth', '1', '-maxdepth', '1', '-exec', 'rm', '-rf', '{}', '+'], process.stdout, process.stderr, null, false, (status) => {
         if (status.status !== 'Success') {
@@ -137,6 +134,25 @@ async function clearClaim(instance, env, work, pod) {
     });
 }
 exports.clearClaim = clearClaim;
+// The archive that uploads a source into the claim: its file or directory, at its place in
+// the claim (see getClaimSubPath) instead of under its own name
+function packSource(source) {
+    const fileItem = (0, path_1.basename)(source.localPath);
+    return stream_1.Readable.from((0, tar_1.packArchive)({
+        cwd: (0, path_1.dirname)(source.localPath),
+        filter(path) {
+            return source.matcher(path, source.localPath);
+        },
+        onWriteEntry(entry) {
+            entry.path = source.subPath + entry.path.slice(fileItem.length);
+            // the second of two hard-linked files links to the first by its path in the archive
+            if (entry.type === 'Link' && entry.linkpath?.startsWith(fileItem)) {
+                entry.linkpath = source.subPath + entry.linkpath.slice(fileItem.length);
+            }
+        },
+    }, [fileItem]));
+}
+exports.packSource = packSource;
 async function ensurePersistentData(instance, env, environment, service, persistence, 
 // the kept claim of the work holds what an earlier run left, which goes first
 options = {}) {
@@ -153,7 +169,7 @@ options = {}) {
     if (persistence.sources.length === 0 && !options.reset) {
         return;
     }
-    service.status.console('stdout', 'start upload container');
+    service.status.write('debug', 'start upload container');
     await getPodForPersistence(instance, env, service, persistence, 'write', async (name) => {
         if (options.reset) {
             await clearClaim(instance, env, service, name);
@@ -163,14 +179,8 @@ options = {}) {
             if (!(await environment.file.exists(source.localPath))) {
                 continue;
             }
-            const fileItem = (0, path_1.basename)(source.localPath);
-            const data = stream_1.Readable.from((0, tar_1.create)({
-                cwd: (0, path_1.dirname)(source.localPath),
-                filter(path) {
-                    return source.matcher(path, source.localPath);
-                },
-            }, [fileItem]));
-            service.status.console('stdout', `upload ${source.localPath}`);
+            const data = packSource(source);
+            service.status.write('info', `upload ${source.localPath}`);
             const targetPath = `/dev/hammerkit/${service.id()}`;
             const res = await instance.exec.exec(env.namespace, name, 'volume', ['tar', 'xf', '-', '-C', targetPath], process.stdout, process.stderr, data, false, (status) => {
                 service.status.write('debug', `upload ${source.localPath} to ${name}: ${status}`);