hammerkit 1.4.0 → 1.5.0-alpha.1

package/dist/executer/create-scheduler-state.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSchedulerState = void 0;
+const plan_work_nodes_1 = require("../planner/utils/plan-work-nodes");
+const check_for_loop_1 = require("./scheduler/check-for-loop");
+const state_1 = require("./state");
+function createSchedulerState(input) {
+    const state = {
+        service: {},
+        node: {},
+        cacheMethod: input.cacheMethod,
+        watch: input.watch,
+    };
+    for (const node of (0, plan_work_nodes_1.iterateWorkNodes)(input.nodes)) {
+        state.node[node.id] = {
+            type: 'pending',
+            node: node,
+            stateKey: null,
+        };
+    }
+    for (const service of (0, plan_work_nodes_1.iterateWorkServices)(input.services)) {
+        state.service[service.id] = {
+            type: 'pending',
+            service,
+            stateKey: null,
+        };
+    }
+    (0, check_for_loop_1.checkForLoop)(state);
+    return new state_1.State(state);
+}
+exports.createSchedulerState = createSchedulerState;
+//# sourceMappingURL=create-scheduler-state.js.map

package/dist/executer/docker-node.js

@@ -0,0 +1,123 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dockerNode = void 0;
+const service_dns_1 = require("./service-dns");
+const execute_docker_1 = require("./execute-docker");
+const abort_1 = require("./abort");
+const write_work_node_cache_1 = require("../optimizer/write-work-node-cache");
+const log_1 = require("../log");
+const states_1 = require("./states");
+const execution_steps_1 = require("./execution-steps");
+const using_container_1 = require("../docker/using-container");
+const print_container_options_1 = require("./print-container-options");
+function buildCreateOptions(node, serviceContainers) {
+    const links = [];
+    const hosts = [];
+    for (const need of node.needs) {
+        const dns = serviceContainers[need.id];
+        if ((0, service_dns_1.isHostServiceDns)(dns)) {
+            hosts.push(`${need.name}:${dns.host}`);
+        }
+        else {
+            if (!dns.containerId) {
+                throw new Error(`service ${need.name} is not running`);
+            }
+            links.push(`${dns.containerId}:${need.name}`);
+        }
+    }
+    return {
+        Image: node.image,
+        Tty: true,
+        Entrypoint: node.shell,
+        Cmd: ['-c', 'sleep 3600'],
+        Env: Object.keys(node.envs).map((k) => `${k}=${node.envs[k]}`),
+        WorkingDir: (0, execute_docker_1.convertToPosixPath)(node.cwd),
+        Labels: { app: 'hammerkit', 'hammerkit-id': node.id, 'hammerkit-type': 'task' },
+        HostConfig: {
+            Binds: [
+                ...node.mounts.map((v) => `${v.localPath}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}`),
+                ...node.volumes.map((v) => `${v.name}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}`),
+            ],
+            PortBindings: node.ports.reduce((map, port) => {
+                map[`${port.containerPort}/tcp`] = [{ HostPort: `${port.hostPort}` }];
+                return map;
+            }, {}),
+            ExtraHosts: hosts,
+            Links: links,
+            AutoRemove: true,
+        },
+        ExposedPorts: node.ports.reduce((map, port) => {
+            map[`${port.containerPort}/tcp`] = {};
+            return map;
+        }, {}),
+    };
+}
+function dockerNode(node, stateKey, serviceContainers, state, environment) {
+    return async (abort, started) => {
+        const status = environment.status.task(node);
+        status.write('info', `execute ${node.name} in container`);
+        try {
+            await (0, execution_steps_1.prepareMounts)(node, environment);
+            (0, abort_1.checkForAbort)(abort.signal);
+            await (0, execution_steps_1.pullImage)(node, environment);
+            (0, abort_1.checkForAbort)(abort.signal);
+            await (0, execution_steps_1.prepareVolume)(node, environment);
+            (0, abort_1.checkForAbort)(abort.signal);
+            const containerOptions = buildCreateOptions(node, serviceContainers);
+            (0, print_container_options_1.printContainerOptions)(status, containerOptions);
+            const success = await (0, using_container_1.usingContainer)(environment, node, containerOptions, async (container) => {
+                await (0, execution_steps_1.setUserPermissions)(node, container, environment);
+                for (const cmd of node.cmds) {
+                    (0, abort_1.checkForAbort)(abort.signal);
+                    status.write('info', `execute cmd "${cmd.cmd}" in container`);
+                    const result = await (0, execute_docker_1.execCommand)(status, environment, container, (0, execute_docker_1.convertToPosixPath)(cmd.path), [node.shell, '-c', cmd.cmd], node.user, undefined, abort.signal);
+                    if (result.type === 'timeout') {
+                        throw new Error(`command ${cmd.cmd} timed out`);
+                    }
+                    if (result.type === 'canceled') {
+                        throw new abort_1.AbortError();
+                    }
+                    if (result.result.ExitCode !== 0) {
+                        state.patchNode({
+                            node,
+                            stateKey,
+                            type: 'crash',
+                            exitCode: result.result.ExitCode ?? 1,
+                        }, stateKey);
+                        return false;
+                    }
+                }
+                return true;
+            });
+            if (success) {
+                await (0, write_work_node_cache_1.writeWorkNodeCache)(node, environment);
+                state.patchNode({
+                    node,
+                    stateKey,
+                    type: 'completed',
+                    cached: false,
+                    duration: (0, states_1.getDuration)(started),
+                }, stateKey);
+            }
+        }
+        catch (e) {
+            if (e instanceof abort_1.AbortError) {
+                state.patchNode({
+                    node,
+                    stateKey,
+                    type: 'canceled',
+                }, stateKey);
+            }
+            else {
+                state.patchNode({
+                    node,
+                    stateKey,
+                    type: 'error',
+                    errorMessage: (0, log_1.getErrorMessage)(e),
+                }, stateKey);
+            }
+        }
+    };
+}
+exports.dockerNode = dockerNode;
+//# sourceMappingURL=docker-node.js.map

package/dist/executer/docker-service.js

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dockerService = void 0;
+const abort_1 = require("./abort");
+const execute_docker_1 = require("./execute-docker");
+const stream_1 = require("../docker/stream");
+const abort_event_1 = require("../utils/abort-event");
+const log_1 = require("../log");
+const remove_container_1 = require("../docker/remove-container");
+const check_readiness_1 = require("./check-readiness");
+const execution_steps_1 = require("./execution-steps");
+function dockerService(service, stateKey, state, environment) {
+    return async (abort) => {
+        const status = environment.status.service(service);
+        let container = null;
+        await (0, execution_steps_1.prepareMounts)(service, environment);
+        (0, abort_1.checkForAbort)(abort.signal);
+        await (0, execution_steps_1.pullImage)(service, environment);
+        (0, abort_1.checkForAbort)(abort.signal);
+        await (0, execution_steps_1.prepareVolume)(service, environment);
+        (0, abort_1.checkForAbort)(abort.signal);
+        try {
+            (0, abort_1.checkForAbort)(abort.signal);
+            status.write('debug', `create container with image ${service.image}`);
+            container = await environment.docker.createContainer({
+                Image: service.image,
+                Env: Object.keys(service.envs).map((k) => `${k}=${service.envs[k]}`),
+                Labels: { app: 'hammerkit', 'hammerkit-id': service.id, 'hammerkit-type': 'service' },
+                ExposedPorts: service.ports.reduce((map, port) => {
+                    map[`${port.containerPort}/tcp`] = {};
+                    return map;
+                }, {}),
+                Cmd: service.cmd ? service.cmd.split(' ') : undefined,
+                HostConfig: {
+                    Binds: [
+                        ...service.mounts.map((v) => `${v.localPath}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}`),
+                        ...service.volumes.map((v) => `${v.name}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}`),
+                    ],
+                    PortBindings: service.ports.reduce((map, port) => {
+                        map[`${port.containerPort}/tcp`] = [{ HostPort: `${port.hostPort}` }];
+                        return map;
+                    }, {}),
+                },
+            });
+            const stream = await container.attach({ stream: true, stdout: true, stderr: true });
+            (0, stream_1.logStream)(status, stream);
+            await container.start();
+            // await setUserPermissions(service, container, environment)
+            if (!service.healthcheck) {
+                state.patchService({
+                    type: 'running',
+                    service,
+                    dns: { containerId: container.id },
+                    stateKey,
+                });
+            }
+            else {
+                let ready = false;
+                do {
+                    ready = await (0, check_readiness_1.checkReadiness)(status, service.healthcheck, environment, container, abort.signal);
+                    if (!ready) {
+                        await new Promise((resolve) => setTimeout(() => resolve(), 1000));
+                    }
+                } while (!ready && !abort.signal.aborted);
+                if (ready) {
+                    state.patchService({
+                        type: 'running',
+                        service,
+                        dns: { containerId: container.id },
+                        stateKey,
+                    });
+                }
+            }
+            await (0, abort_event_1.waitOnAbort)(abort.signal);
+            state.patchService({
+                type: 'end',
+                service,
+                stateKey,
+                reason: 'terminated',
+            });
+        }
+        catch (e) {
+            if (e instanceof abort_1.AbortError) {
+                state.patchService({
+                    type: 'canceled',
+                    service,
+                    stateKey,
+                });
+            }
+            else {
+                status.write('error', (0, log_1.getErrorMessage)(e));
+                state.patchService({
+                    type: 'end',
+                    service,
+                    reason: 'crash',
+                    stateKey,
+                });
+            }
+        }
+        finally {
+            if (container) {
+                try {
+                    await (0, remove_container_1.removeContainer)(container);
+                }
+                catch (e) {
+                    status.write('error', `remove of container failed ${(0, log_1.getErrorMessage)(e)}`);
+                }
+            }
+        }
+    };
+}
+exports.dockerService = dockerService;
+//# sourceMappingURL=docker-service.js.map

package/dist/executer/environment-mock.js

@@ -1,3 +1,20 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.environmentMock = void 0;
+const work_node_status_1 = require("../planner/work-node-status");
+const console_context_mock_1 = require("../console/console-context-mock");
+const get_file_context_1 = require("../file/get-file-context");
+const execute_docker_1 = require("./execute-docker");
+function environmentMock(cwd) {
+    return {
+        cwd,
+        file: (0, get_file_context_1.getFileContext)(cwd),
+        console: (0, console_context_mock_1.consoleContextMock)(),
+        abortCtrl: new AbortController(),
+        processEnvs: {},
+        status: (0, work_node_status_1.statusConsole)(),
+        docker: (0, execute_docker_1.getContainerCli)(),
+    };
+}
+exports.environmentMock = environmentMock;
 //# sourceMappingURL=environment-mock.js.map

package/dist/executer/event-cache.js

@@ -0,0 +1,165 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cleanCache = exports.storeCache = exports.restoreCache = void 0;
+const plan_work_nodes_1 = require("../planner/utils/plan-work-nodes");
+const get_cache_directory_1 = require("../optimizer/get-cache-directory");
+const path_1 = require("path");
+const move_files_1 = require("../file/move-files");
+const execute_docker_1 = require("./execute-docker");
+const get_docker_executor_1 = require("./get-docker-executor");
+const tar_1 = require("tar");
+const work_node_1 = require("../planner/work-node");
+const using_container_1 = require("../docker/using-container");
+const work_service_1 = require("../planner/work-service");
+const plan_work_volume_1 = require("../planner/utils/plan-work-volume");
+async function restoreCache(path, workTree, environment) {
+    for (const node of (0, plan_work_nodes_1.iterateWorkNodes)(workTree.nodes)) {
+        const cachePath = (0, get_cache_directory_1.getCacheDirectory)(node.id);
+        const sourceCacheDir = (0, path_1.join)(path, node.id);
+        await (0, move_files_1.moveFiles)(node, environment, function* () {
+            yield { from: (0, path_1.join)(sourceCacheDir, 'stats.json'), to: (0, path_1.join)(cachePath, 'stats.json') };
+            yield { from: (0, path_1.join)(sourceCacheDir, 'description.json'), to: (0, path_1.join)(cachePath, 'description.json') };
+        });
+        if ((0, work_node_1.isContainerWorkNode)(node)) {
+            await restoreContainer(environment, node, sourceCacheDir);
+        }
+        else {
+            await restoreLocal(environment, node, sourceCacheDir);
+        }
+    }
+}
+exports.restoreCache = restoreCache;
+async function restoreLocal(environment, node, path) {
+    for (const generate of getGenerates(node, path)) {
+        if (await environment.file.exists(generate.filename)) {
+            await (0, tar_1.extract)({
+                file: generate.filename,
+                cwd: node.cwd,
+            });
+        }
+    }
+}
+async function restoreContainer(environment, node, path) {
+    await (0, using_container_1.usingContainer)(environment, node, {
+        Image: node.image,
+        Tty: true,
+        Entrypoint: 'sh',
+        Cmd: ['-c', 'sleep 3600'],
+        WorkingDir: (0, execute_docker_1.convertToPosixPath)(node.cwd),
+        Labels: { app: 'hammerkit', 'hammerkit-id': node.id, 'hammerkit-type': 'task' },
+        HostConfig: {
+            AutoRemove: true,
+            Binds: [...node.volumes.map((v) => `${v.name}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}`)],
+        },
+    }, async (container) => {
+        for (const generate of getGenerates(node, path)) {
+            if (await environment.file.exists(generate.filename)) {
+                await container.putArchive(environment.file.readStream(generate.filename), {
+                    path: (0, path_1.dirname)(generate.path),
+                });
+            }
+        }
+    });
+}
+function* getGenerates(node, path) {
+    const generates = node.generates.filter((g) => !g.inherited);
+    for (const generate of generates) {
+        const generateId = (0, path_1.relative)(node.cwd, generate.path).split(path_1.sep).join('-');
+        yield {
+            filename: (0, path_1.join)(path, generateId + '-generates.tgz'),
+            path: generate.path,
+        };
+    }
+}
+async function archiveContainer(environment, node, path) {
+    await (0, using_container_1.usingContainer)(environment, node, {
+        Image: node.image,
+        Tty: true,
+        Entrypoint: 'sh',
+        Cmd: ['-c', 'sleep 3600'],
+        WorkingDir: (0, execute_docker_1.convertToPosixPath)(node.cwd),
+        Labels: { app: 'hammerkit', 'hammerkit-id': node.id, 'hammerkit-type': 'task' },
+        HostConfig: {
+            AutoRemove: true,
+            Binds: [...node.volumes.map((v) => `${v.name}:${(0, execute_docker_1.convertToPosixPath)(v.containerPath)}`)],
+        },
+    }, async (container) => {
+        for (const generatedArchive of getGenerates(node, path)) {
+            const readable = await container.getArchive({
+                path: generatedArchive.path,
+            });
+            await environment.file.writeStream(generatedArchive.filename, readable);
+        }
+    });
+}
+async function archiveLocal(environment, node, path) {
+    for (const generatedArchive of getGenerates(node, path)) {
+        await environment.file.writeStream(generatedArchive.filename, (0, tar_1.create)({
+            cwd: node.cwd,
+            gzip: true,
+        }, [(0, path_1.relative)(node.cwd, generatedArchive.path)]));
+    }
+}
+async function storeCache(path, workTree, environment) {
+    for (const node of (0, plan_work_nodes_1.iterateWorkNodes)(workTree.nodes)) {
+        const cachePath = (0, get_cache_directory_1.getCacheDirectory)(node.id);
+        const sourceCacheDir = (0, path_1.join)(path, node.id);
+        await (0, move_files_1.moveFiles)(node, environment, function* () {
+            yield { from: (0, path_1.join)(cachePath, 'stats.json'), to: (0, path_1.join)(sourceCacheDir, 'stats.json') };
+            yield { from: (0, path_1.join)(cachePath, 'description.json'), to: (0, path_1.join)(sourceCacheDir, 'description.json') };
+        });
+        if ((0, work_node_1.isContainerWorkNode)(node)) {
+            await archiveContainer(environment, node, sourceCacheDir);
+        }
+        else {
+            await archiveLocal(environment, node, sourceCacheDir);
+        }
+    }
+}
+exports.storeCache = storeCache;
+async function cleanCache(workTree, environment, options) {
+    for (const node of (0, plan_work_nodes_1.iterateWorkNodes)(workTree.nodes)) {
+        const nodeStatus = environment.status.task(node);
+        for (const generate of node.generates) {
+            if (generate.inherited) {
+                continue;
+            }
+            const volumeName = (0, plan_work_volume_1.getVolumeName)(generate.path);
+            const volumeExists = await (0, get_docker_executor_1.existsVolume)(environment, volumeName);
+            if (volumeExists) {
+                nodeStatus.write('info', `remove volume ${volumeName}`);
+                const volume = await environment.docker.getVolume(volumeName);
+                await volume.remove();
+            }
+            else {
+                nodeStatus.write('info', `generate ${generate} has no volume ${volumeName}`);
+            }
+            nodeStatus.write('info', `remove local directory ${generate.path}`);
+            await environment.file.remove(generate.path);
+        }
+        const cachePath = (0, get_cache_directory_1.getCacheDirectory)(node.id);
+        if (await environment.file.exists(cachePath)) {
+            nodeStatus.write('info', `remove cache ${cachePath}`);
+            await environment.file.remove(cachePath);
+        }
+    }
+    if (options.service) {
+        for (const service of (0, plan_work_nodes_1.iterateWorkServices)(workTree.services)) {
+            if (!(0, work_service_1.isContainerWorkService)(service)) {
+                continue;
+            }
+            const nodeStatus = environment.status.service(service);
+            for (const volume of service.volumes) {
+                const volumeExists = await (0, get_docker_executor_1.existsVolume)(environment, volume.name);
+                if (!volumeExists) {
+                    continue;
+                }
+                nodeStatus.write('info', `remove volume ${volume.name}`);
+                const containerVolume = await environment.docker.getVolume(volume.name);
+                await containerVolume.remove();
+            }
+        }
+    }
+}
+exports.cleanCache = cleanCache;
+//# sourceMappingURL=event-cache.js.map

package/dist/executer/execute-command.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeCommand = void 0;
+const child_process_1 = require("child_process");
+const os_1 = require("os");
+const log_1 = require("../log");
+const abort_1 = require("./abort");
+const abort_event_1 = require("../utils/abort-event");
+async function executeCommand(status, abortSignal, cwd, command, envs) {
+    return new Promise((resolve, reject) => {
+        const ps = (0, child_process_1.exec)(command, {
+            env: envs,
+            cwd,
+            shell: (0, os_1.platform)() === 'win32' ? 'powershell.exe' : undefined,
+        });
+        ps.stdout?.on('data', async (data) => {
+            for (const log of (0, log_1.getLogs)(data)) {
+                status.console('stdout', log);
+            }
+        });
+        ps.stderr?.on('data', async (data) => {
+            for (const log of (0, log_1.getLogs)(data)) {
+                status.console('stderr', log);
+            }
+        });
+        ps.on('error', (err) => {
+            reject(err);
+        });
+        ps.on('close', (code) => {
+            if (abortSignal.aborted) {
+                reject(new abort_1.AbortError());
+                return;
+            }
+            resolve(code ?? 0);
+        });
+        (0, abort_event_1.listenOnAbort)(abortSignal, () => {
+            ps.kill();
+        });
+    });
+}
+exports.executeCommand = executeCommand;
+//# sourceMappingURL=execute-command.js.map