hammerkit 1.5.1 → 1.6.0-alpha.2

package/dist/kubernetes/remove-persistent-data.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removePersistentData = void 0;
+async function removePersistentData(instance, kubernetes, task) {
+    const configmaps = await instance.coreApi.listNamespacedConfigMap(kubernetes.namespace, undefined, undefined, undefined, undefined, `hammerkit.dev/id=${task.id()}`);
+    for (const configMap of configmaps.body.items) {
+        if (configMap.metadata?.name) {
+            await instance.coreApi.deleteNamespacedConfigMap(configMap.metadata.name, kubernetes.namespace);
+        }
+    }
+    const pvcs = await instance.coreApi.listNamespacedPersistentVolumeClaim(kubernetes.namespace, undefined, undefined, undefined, undefined, `hammerkit.dev/id=${task.id()}`);
+    for (const pvc of pvcs.body.items) {
+        if (pvc.metadata?.name) {
+            await instance.coreApi.deleteNamespacedPersistentVolumeClaim(pvc.metadata.name, kubernetes.namespace);
+        }
+    }
+}
+exports.removePersistentData = removePersistentData;
+//# sourceMappingURL=remove-persistent-data.js.map

package/dist/kubernetes/resolve-pod-name.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolvePodName = void 0;
+function formatSelector(matchLabels) {
+    return Object.entries(matchLabels)
+        .map(([k, v]) => `${k}=${v}`)
+        .join(',');
+}
+async function findReadyPod(coreApi, namespace, labelSelector) {
+    const pods = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, labelSelector);
+    const ready = pods.body.items.find((p) => p.status?.conditions?.some((c) => c.type === 'Ready' && c.status === 'True'));
+    const pod = ready ?? pods.body.items[0];
+    if (!pod?.metadata?.name) {
+        throw new Error(`no pod found for selector ${labelSelector} in namespace ${namespace}`);
+    }
+    return pod.metadata.name;
+}
+async function resolvePodName(coreApi, appsApi, namespace, selector) {
+    const type = selector.type.toLowerCase();
+    if (type === 'pod') {
+        return selector.name;
+    }
+    if (type === 'service') {
+        const svc = await coreApi.readNamespacedService(selector.name, namespace);
+        const matchLabels = svc.body.spec?.selector;
+        if (!matchLabels || Object.keys(matchLabels).length === 0) {
+            throw new Error(`service ${selector.name} has no selector`);
+        }
+        return findReadyPod(coreApi, namespace, formatSelector(matchLabels));
+    }
+    if (type === 'deployment') {
+        const deploy = await appsApi.readNamespacedDeployment(selector.name, namespace);
+        const matchLabels = deploy.body.spec?.selector?.matchLabels;
+        if (!matchLabels) {
+            throw new Error(`deployment ${selector.name} has no matchLabels`);
+        }
+        return findReadyPod(coreApi, namespace, formatSelector(matchLabels));
+    }
+    throw new Error(`unsupported selector type ${selector.type}`);
+}
+exports.resolvePodName = resolvePodName;
+//# sourceMappingURL=resolve-pod-name.js.map

package/dist/kubernetes/resolve-pod-name.spec.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const resolve_pod_name_1 = require("./resolve-pod-name");
+function podBody(items) {
+    return {
+        body: {
+            items: items.map((p) => ({
+                metadata: { name: p.name },
+                status: {
+                    conditions: [{ type: 'Ready', status: p.ready ? 'True' : 'False' }],
+                },
+            })),
+        },
+    };
+}
+function makeCoreApi(opts = {}) {
+    return {
+        readNamespacedService: vi.fn().mockResolvedValue({ body: { spec: { selector: opts.service?.selector } } }),
+        listNamespacedPod: vi.fn().mockResolvedValue(podBody(opts.pods ?? [])),
+    };
+}
+function makeAppsApi(opts = {}) {
+    return {
+        readNamespacedDeployment: vi
+            .fn()
+            .mockResolvedValue({ body: { spec: { selector: { matchLabels: opts.deployment?.matchLabels } } } }),
+    };
+}
+describe('resolvePodName', () => {
+    it('returns the name directly for selector type "pod"', async () => {
+        const core = makeCoreApi();
+        const apps = makeAppsApi();
+        await expect((0, resolve_pod_name_1.resolvePodName)(core, apps, 'ns', { type: 'pod', name: 'my-pod' })).resolves.toBe('my-pod');
+        expect(core.listNamespacedPod).not.toHaveBeenCalled();
+    });
+    it('resolves a service to a ready pod', async () => {
+        const core = makeCoreApi({
+            service: { selector: { app: 'api' } },
+            pods: [
+                { name: 'api-old', ready: false },
+                { name: 'api-new', ready: true },
+            ],
+        });
+        const apps = makeAppsApi();
+        await expect((0, resolve_pod_name_1.resolvePodName)(core, apps, 'ns', { type: 'service', name: 'api' })).resolves.toBe('api-new');
+        expect(core.listNamespacedPod).toHaveBeenCalledWith('ns', undefined, undefined, undefined, undefined, 'app=api');
+    });
+    it('falls back to the first pod if none are ready', async () => {
+        const core = makeCoreApi({
+            service: { selector: { app: 'api' } },
+            pods: [
+                { name: 'api-1', ready: false },
+                { name: 'api-2', ready: false },
+            ],
+        });
+        const apps = makeAppsApi();
+        await expect((0, resolve_pod_name_1.resolvePodName)(core, apps, 'ns', { type: 'service', name: 'api' })).resolves.toBe('api-1');
+    });
+    it('resolves a deployment via matchLabels', async () => {
+        const core = makeCoreApi({ pods: [{ name: 'p', ready: true }] });
+        const apps = makeAppsApi({ deployment: { matchLabels: { 'hammerkit.dev/id': 'x' } } });
+        await expect((0, resolve_pod_name_1.resolvePodName)(core, apps, 'ns', { type: 'deployment', name: 'api' })).resolves.toBe('p');
+        expect(core.listNamespacedPod).toHaveBeenCalledWith('ns', undefined, undefined, undefined, undefined, 'hammerkit.dev/id=x');
+    });
+    it('throws when a service has no selector', async () => {
+        const core = makeCoreApi({ service: { selector: undefined } });
+        const apps = makeAppsApi();
+        await expect((0, resolve_pod_name_1.resolvePodName)(core, apps, 'ns', { type: 'service', name: 'api' })).rejects.toThrow(/no selector/);
+    });
+    it('throws when no pods match', async () => {
+        const core = makeCoreApi({ service: { selector: { app: 'api' } }, pods: [] });
+        const apps = makeAppsApi();
+        await expect((0, resolve_pod_name_1.resolvePodName)(core, apps, 'ns', { type: 'service', name: 'api' })).rejects.toThrow(/no pod found/);
+    });
+    it('rejects unsupported selector types', async () => {
+        const core = makeCoreApi();
+        const apps = makeAppsApi();
+        await expect((0, resolve_pod_name_1.resolvePodName)(core, apps, 'ns', { type: 'statefulset', name: 'x' })).rejects.toThrow(/unsupported selector type/);
+    });
+});
+//# sourceMappingURL=resolve-pod-name.spec.js.map

package/dist/kubernetes/resources.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeInvalidCharacters = exports.getResourceName = void 0;
+function getResourceName(item, suffix) {
+    return removeInvalidCharacters(item.name) + '-' + item.id() + (suffix ?? '');
+}
+exports.getResourceName = getResourceName;
+function removeInvalidCharacters(val) {
+    return val.replace(/:/g, '-').replace(/\./g, '-');
+}
+exports.removeInvalidCharacters = removeInvalidCharacters;
+//# sourceMappingURL=resources.js.map

package/dist/kubernetes/restore-kubernetes-data.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.restoreKubernetesData = void 0;
+const event_cache_1 = require("../executer/event-cache");
+const log_1 = require("../log");
+const path_1 = require("path");
+const volumes_1 = require("./volumes");
+const ensure_persistent_data_1 = require("./ensure-persistent-data");
+async function restoreKubernetesData(work, kubernetes, instance, environment, path) {
+    const persistence = await (0, volumes_1.getKubernetesPersistence)(work);
+    await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'write', async (name) => {
+        for (const generatedArchive of (0, event_cache_1.getArchivePaths)(work.data, path)) {
+            if (!(await environment.file.exists(generatedArchive.filename))) {
+                continue;
+            }
+            const targetPath = `/dev/hammerkit/${work.id()}/${(0, path_1.basename)(generatedArchive.path)}`;
+            const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', ['tar', 'xzf', '-', '-C', targetPath], process.stdout, process.stderr, environment.file.readStream(generatedArchive.filename), false, (status) => {
+                work.status.write('debug', `upload ${generatedArchive.filename} to ${name}: ${status}`);
+            });
+            await new Promise((resolve, reject) => {
+                try {
+                    res.on('open', () => {
+                        work.status.write('debug', `start upload of ${generatedArchive.filename} to ${name}:${targetPath}`);
+                    });
+                    res.on('error', (err) => {
+                        work.status.write('error', `error during upload of ${generatedArchive.filename} to ${name}: ${(0, log_1.getErrorMessage)(err)}`);
+                        reject(err);
+                    });
+                    res.on('close', () => {
+                        work.status.write('debug', `uploaded ${generatedArchive.filename} to ${name}:${targetPath}`);
+                        resolve();
+                    });
+                }
+                catch (e) {
+                    reject(e);
+                }
+            });
+        }
+    });
+}
+exports.restoreKubernetesData = restoreKubernetesData;
+//# sourceMappingURL=restore-kubernetes-data.js.map

package/dist/kubernetes/store-kubernetes-data.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.storeKubernetesData = void 0;
+const log_1 = require("../log");
+const event_cache_1 = require("../executer/event-cache");
+const volumes_1 = require("./volumes");
+const ensure_persistent_data_1 = require("./ensure-persistent-data");
+async function storeKubernetesData(work, kubernetes, instance, environment, path) {
+    const persistence = await (0, volumes_1.getKubernetesPersistence)(work);
+    await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'read', async (name) => {
+        for (const generatedArchive of (0, event_cache_1.getArchivePaths)(work.data, path)) {
+            const writerStream = environment.file.createWriteStream(generatedArchive.filename);
+            const res = await instance.exec.exec(kubernetes.namespace, name, 'volume', ['tar', 'zcf', '-', '-C', generatedArchive.path, '.'], writerStream, process.stderr, null, false, (status) => {
+                work.status.write('debug', `archive ${generatedArchive.path} of ${name}: ${status}`);
+                writerStream.close();
+            });
+            await new Promise((resolve, reject) => {
+                try {
+                    res.on('open', () => {
+                        work.status.write('debug', `start download ${generatedArchive.path} of ${name}`);
+                    });
+                    res.on('error', (err) => {
+                        work.status.write('error', `download during upload ${generatedArchive.path} of ${name}: ${(0, log_1.getErrorMessage)(err)}`);
+                        reject(err);
+                    });
+                    res.on('close', () => {
+                        work.status.write('debug', `downloaded ${generatedArchive.path} of ${name}`);
+                        resolve();
+                    });
+                }
+                catch (e) {
+                    reject(e);
+                }
+            });
+        }
+    });
+}
+exports.storeKubernetesData = storeKubernetesData;
+//# sourceMappingURL=store-kubernetes-data.js.map

package/dist/kubernetes/volumes.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendVolume = exports.getKubernetesPersistence = exports.getVolumeName = void 0;
+const path_1 = require("path");
+function getVolumeName(service) {
+    return 'hammerkit-' + service.id();
+}
+exports.getVolumeName = getVolumeName;
+async function getKubernetesPersistence(work) {
+    const persistence = { volumes: [], sources: [], mounts: [], initCommands: [] };
+    for (const mount of work.data.mounts) {
+        appendVolume(work, persistence, mount.containerPath, mount.localPath, mount.isFile, true);
+    }
+    for (const src of work.data.src) {
+        if (src.inherited) {
+            continue;
+        }
+        appendVolume(work, persistence, src.absolutePath, src.absolutePath, src.isFile, true); // TODO matcher
+    }
+    if (work.data.type === 'container-task') {
+        for (const generate of work.data.generates) {
+            appendVolume(generate.inherited ?? work, persistence, generate.path, null, false, !!generate.inherited);
+        }
+    }
+    else {
+        for (const volume of work.data.volumes) {
+            appendVolume(volume.inherited ?? work, persistence, volume.containerPath, null, false, !!volume.inherited);
+        }
+    }
+    return persistence;
+}
+exports.getKubernetesPersistence = getKubernetesPersistence;
+function appendVolume(sourceWork, persistence, containerPath, localPath, isFile, readOnly) {
+    const volumeName = getVolumeName(sourceWork);
+    const existVolume = persistence.volumes.some((v) => v.name === volumeName);
+    if (!existVolume) {
+        persistence.volumes.push({
+            name: volumeName,
+            persistentVolumeClaim: {
+                claimName: volumeName,
+            },
+        });
+    }
+    persistence.mounts.push({
+        mount: {
+            readOnly,
+            mountPath: containerPath,
+            name: volumeName,
+            subPath: (0, path_1.basename)(containerPath),
+        },
+        uploadPath: `/dev/hammerkit/${sourceWork.id()}`,
+        cwd: isFile ? (0, path_1.dirname)(containerPath) : containerPath,
+    });
+    if (localPath) {
+        persistence.sources.push({
+            localPath: localPath,
+            matcher: () => true, // TODO
+            containerPath,
+            stateKey: '', // TODO
+        });
+    }
+}
+exports.appendVolume = appendVolume;
+//# sourceMappingURL=volumes.js.map

package/dist/log.js

@@ -3,9 +3,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.showCursor = exports.hideCursor = exports.writeWorkTreeStatus = exports.formatDate = exports.writeNodeLogToConsole = exports.getNodeName = exports.getNodeNameLengthForWorkTree = exports.getNodeNameLength = exports.printTitle = exports.printItem = exports.printProperty = exports.getNodeNameLengthForSchedulerState = exports.printWorkTreeResult = exports.getType = exports.isVerbose = exports.consoleContext = exports.getErrorMessage = exports.getLogLevel = exports.getLogs = void 0;
+exports.showCursor = exports.hideCursor = exports.writeWorkTreeStatus = exports.formatDate = exports.writeWorkItemLogToConsole = exports.getNodeName = exports.getMaxNameLength = exports.printTitle = exports.printItem = exports.printProperty = exports.getWorkItemMaxLength = exports.printWorkTreeResult = exports.getType = exports.isVerbose = exports.consoleContext = exports.getErrorMessage = exports.getLogLevel = exports.getLogs = void 0;
 const readline_1 = require("readline");
-const plan_work_nodes_1 = require("./planner/utils/plan-work-nodes");
+const plan_work_tasks_1 = require("./planner/utils/plan-work-tasks");
 const colors_1 = __importDefault(require("colors"));
 const fail_never_1 = require("./utils/fail-never");
 function getLogs(chunk) {
@@ -66,50 +66,59 @@ function getType(type) {
 }
 exports.getType = getType;
 async function printWorkTreeResult(schedulerState, env) {
-    const maxNodeNameLength = getNodeNameLengthForSchedulerState(schedulerState);
+    const maxNodeNameLength = getWorkItemMaxLength(schedulerState);
     for (const log of env.status.read()) {
         if (exports.isVerbose || (log.type === 'status' && log.level === 'error')) {
             env.stdout.write(`${formatDate(log.date)} ${getType(log.context.type)} ${getNodeName(log.context.name, maxNodeNameLength)} - ${colors_1.default.white(log.message)}\n`);
         }
     }
-    for (const state of (0, plan_work_nodes_1.iterateWorkServices)(schedulerState.service)) {
-        if (state.type === 'end' && state.reason === 'crash') {
-            const message = `${getType('service')} ${getNodeName(state.service.name, maxNodeNameLength)}`;
-            for (const log of env.status.service(state.service).logs()) {
+    for (const service of (0, plan_work_tasks_1.iterateWorkServices)(schedulerState)) {
+        if (service.state.current.type === 'end' && service.state.current.reason === 'crash') {
+            const message = `${getType('service')} ${getNodeName(service.name, maxNodeNameLength)}`;
+            for (const log of service.status.logs()) {
                 env.stdout.write(`${formatDate(log.date)} ${message} - ${log.console}: ${log.message}\n`);
             }
         }
+        else if (service.state.current.type === 'running') {
+            const message = `${getType('service')} ${getNodeName(service.name, maxNodeNameLength)}`;
+            if (service.state.current.remote) {
+                env.stdout.write(`${formatDate(new Date())} ${message} - already started\n`);
+            }
+            else {
+                env.stdout.write(`${formatDate(new Date())} ${message} - sucessfull started\n`);
+            }
+        }
     }
-    for (const state of (0, plan_work_nodes_1.iterateWorkNodes)(schedulerState.node)) {
-        if (state.type === 'error' || state.type === 'crash') {
-            const message = `${getType('task')} ${getNodeName(state.node.name, maxNodeNameLength)}`;
-            for (const log of env.status.task(state.node).logs()) {
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(schedulerState)) {
+        if (task.state.current.type === 'error' || task.state.current.type === 'crash') {
+            const message = `${getType('task')} ${getNodeName(task.name, maxNodeNameLength)}`;
+            for (const log of task.status.logs()) {
                 env.stdout.write(`${formatDate(log.date)} ${message} - ${log.console}: ${log.message}\n`);
             }
         }
     }
-    for (const state of (0, plan_work_nodes_1.iterateWorkServices)(schedulerState.service)) {
-        let message = `${formatDate(new Date())} ${getType('service')} ${getNodeName(state.service.name, maxNodeNameLength)} - ${getStateText(state)}`;
-        if (state.type === 'end') {
-            if (state.reason === 'crash') {
+    for (const service of (0, plan_work_tasks_1.iterateWorkServices)(schedulerState)) {
+        let message = `${formatDate(new Date())} ${getType('service')} ${getNodeName(service.name, maxNodeNameLength)} - ${getStateText(service.state.current)}`;
+        if (service.state.current.type === 'end') {
+            if (service.state.current.reason === 'crash') {
                 message += ` crashed`;
             }
             env.stdout.write(`${message}\n`);
         }
     }
-    for (const state of (0, plan_work_nodes_1.iterateWorkNodes)(schedulerState.node)) {
-        let message = `${formatDate(new Date())} ${getType('task')} ${getNodeName(state.node.name, maxNodeNameLength)} - ${getStateText(state)}`;
-        if (state.type === 'completed') {
-            message += ` in ${state.duration}ms`;
-            if (state.cached) {
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(schedulerState)) {
+        let message = `${formatDate(new Date())} ${getType('task')} ${getNodeName(task.name, maxNodeNameLength)} - ${getStateText(task.state.current)}`;
+        if (task.state.current.type === 'completed') {
+            message += ` in ${task.state.current.duration}ms`;
+            if (task.state.current.cached) {
                 message += ' [CACHED]';
             }
         }
-        if (state.type === 'crash') {
-            message += ` exited with ${state.exitCode}`;
+        if (task.state.current.type === 'crash') {
+            message += ` exited with ${task.state.current.exitCode}`;
         }
-        if (state.type === 'error') {
-            message += ` with message ${state.errorMessage}`;
+        if (task.state.current.type === 'error') {
+            message += ` with message ${task.state.current.errorMessage}`;
         }
         env.stdout.write(`${message}\n`);
     }
@@ -142,21 +151,21 @@ function getStateText(state) {
         return colors_1.default.blue(state.type);
     }
     else {
-        (0, fail_never_1.failNever)(state, 'unknown node state');
+        (0, fail_never_1.failNever)(state, 'unknown task state');
     }
 }
-function getNodeNameLengthForSchedulerState(schedulerState) {
+function getWorkItemMaxLength(schedulerState) {
     function* Names() {
-        for (const node of (0, plan_work_nodes_1.iterateWorkNodes)(schedulerState.node)) {
-            yield node.node.name;
+        for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(schedulerState)) {
+            yield task.name;
         }
-        for (const service of (0, plan_work_nodes_1.iterateWorkServices)(schedulerState.service)) {
-            yield service.service.name;
+        for (const service of (0, plan_work_tasks_1.iterateWorkServices)(schedulerState)) {
+            yield service.name;
         }
     }
-    return getNodeNameLength(Names);
+    return getMaxNameLength(Names);
 }
-exports.getNodeNameLengthForSchedulerState = getNodeNameLengthForSchedulerState;
+exports.getWorkItemMaxLength = getWorkItemMaxLength;
 function printProperty(env, name, value) {
     env.stdout.write(`   ${colors_1.default.grey(`${name}:`)} ${value}\n`);
 }
@@ -169,7 +178,7 @@ function printTitle(env, name) {
     env.stdout.write(`${colors_1.default.bgWhite(name + ':\n')}`);
 }
 exports.printTitle = printTitle;
-function getNodeNameLength(names) {
+function getMaxNameLength(names) {
     let maxNodeNameLength = 0;
     for (const name of names()) {
         if (name.length > maxNodeNameLength) {
@@ -178,24 +187,12 @@ function getNodeNameLength(names) {
     }
     return maxNodeNameLength;
 }
-exports.getNodeNameLength = getNodeNameLength;
-function getNodeNameLengthForWorkTree(nodes, services) {
-    function* Names() {
-        for (const node of (0, plan_work_nodes_1.iterateWorkNodes)(nodes)) {
-            yield node.node.name;
-        }
-        for (const service of (0, plan_work_nodes_1.iterateWorkServices)(services)) {
-            yield service.service.name;
-        }
-    }
-    return getNodeNameLength(Names);
-}
-exports.getNodeNameLengthForWorkTree = getNodeNameLengthForWorkTree;
+exports.getMaxNameLength = getMaxNameLength;
 function getNodeName(name, maxNodeNameLength) {
     return colors_1.default.white(name) + ' '.repeat(Math.max(0, maxNodeNameLength - name.length));
 }
 exports.getNodeName = getNodeName;
-function writeNodeLogToConsole(env, log, maxNodeNameLength) {
+function writeWorkItemLogToConsole(env, log, maxNodeNameLength) {
     if (log.type === 'status') {
         if (log.level === 'debug' && !exports.isVerbose) {
             return;
@@ -208,7 +205,7 @@ function writeNodeLogToConsole(env, log, maxNodeNameLength) {
         : `${getLogLevel(log.level)} - ${log.message}`;
     env.stdout.write(`${formatDate(log.date)} ${colors_1.default.grey(log.context.type + ':')} ${getNodeName(log.context.name, maxNodeNameLength)} - ${log.date.toLocaleTimeString()} - ${content}\n`);
 }
-exports.writeNodeLogToConsole = writeNodeLogToConsole;
+exports.writeWorkItemLogToConsole = writeWorkItemLogToConsole;
 function formatDate(date) {
     return colors_1.default.grey(`[${date.getFullYear()}-${(date.getMonth() + 1).toString().padStart(2, '0')}-${date
         .getDay()
@@ -224,23 +221,26 @@ function formatDate(date) {
 exports.formatDate = formatDate;
 function writeWorkTreeStatus(schedulerState, env, ticker) {
     (0, readline_1.clearScreenDown)(env.stdout);
-    const maxNodeNameLength = getNodeNameLengthForSchedulerState(schedulerState);
+    const maxNodeNameLength = getWorkItemMaxLength(schedulerState);
     let count = 0;
-    for (const state of (0, plan_work_nodes_1.iterateWorkServices)(schedulerState.service)) {
-        let message = `${getType('service')} ${getNodeName(state.service.name, maxNodeNameLength)} - ${getStateText(state)}`;
-        const currentMessage = env.status.service(state.service).current();
+    for (const service of (0, plan_work_tasks_1.iterateWorkServices)(schedulerState)) {
+        let message = `${getType('service')} ${getNodeName(service.name, maxNodeNameLength)} - ${getStateText(service.state.current)}`;
+        if (service.state.current.type === 'running' && service.state.current.remote) {
+            message += ` [REMOTE]`;
+        }
+        const currentMessage = service.status.currentLog();
         if (currentMessage) {
             message += ` ${currentMessage.message}`;
         }
         count += printWithTruncate(env, message);
     }
-    for (const state of (0, plan_work_nodes_1.iterateWorkNodes)(schedulerState.node)) {
-        let message = `${getType('task')} ${getNodeName(state.node.name, maxNodeNameLength)} - ${getStateText(state)}`;
-        if (state.type === 'pending') {
-            const totalDepCount = state.node.deps.length;
-            const totalServiceCount = state.node.needs.length;
-            const completedDepCount = state.node.deps.filter((d) => schedulerState.node[d.id]?.type === 'completed').length;
-            const runningServiceCount = state.node.needs.filter((n) => schedulerState.service[n.id]?.type === 'ready').length;
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(schedulerState)) {
+        let message = `${getType('task')} ${getNodeName(task.name, maxNodeNameLength)} - ${getStateText(task.state.current)}`;
+        if (task.state.current.type === 'pending') {
+            const totalDepCount = task.deps.length;
+            const totalServiceCount = task.needs.length;
+            const completedDepCount = task.deps.filter((d) => d.state.current.type === 'completed').length;
+            const runningServiceCount = task.needs.filter((n) => n.service.state.current.type === 'ready').length;
             if (totalDepCount > 0 && completedDepCount !== totalDepCount) {
                 message += ` | ${colors_1.default.grey(` awaiting dependencies (${completedDepCount}/${totalDepCount})`)}`;
             }
@@ -248,28 +248,28 @@ function writeWorkTreeStatus(schedulerState, env, ticker) {
                 message += ` | ${colors_1.default.grey(` awaiting service (${runningServiceCount}/${totalServiceCount})`)}`;
             }
         }
-        if (state.type === 'starting' || state.type === 'ready') {
+        if (task.state.current.type === 'starting' || task.state.current.type === 'ready') {
             message += ` | ${spinner[ticker % spinner.length]}`;
-            const currentMessage = env.status.task(state.node).current();
+            const currentMessage = task.status.current();
             if (currentMessage) {
                 message += ` ${currentMessage.message}`;
             }
         }
-        else if (state.type === 'running') {
+        else if (task.state.current.type === 'running') {
             message += ` | ${spinner[ticker % spinner.length]}`;
-            const currentMessage = env.status.task(state.node).currentLog();
+            const currentMessage = task.status.currentLog();
             if (currentMessage) {
                 message += ` ${currentMessage.message}`;
             }
         }
-        else if (state.type === 'completed') {
-            message += ` in ${state.duration}ms`;
-            if (state.cached) {
+        else if (task.state.current.type === 'completed') {
+            message += ` in ${task.state.current.duration}ms`;
+            if (task.state.current.cached) {
                 message += ' [CACHED]';
             }
         }
-        else if (state.type === 'error') {
-            message += ` ${state.errorMessage}`;
+        else if (task.state.current.type === 'error') {
+            message += ` ${task.state.current.errorMessage}`;
         }
         count += printWithTruncate(env, message);
     }
@@ -290,7 +290,7 @@ function printWithTruncate(env, message) {
     let index = 0;
     do {
         const newLineCount = countChar(message, '\n');
-        env.stdout.write(`${message.substr(index, env.stdoutColumns)}\n`);
+        env.stdout.write(`${message.substring(index, index + env.stdoutColumns)}\n`);
         index += env.stdoutColumns;
         lineCount++;
         lineCount += newLineCount;