hammerkit 1.4.0-beta.6 → 1.5.0-alpha.0

package/dist/executer/execute-docker.js

@@ -3,125 +3,55 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.executeDocker = exports.getContainerMounts = exports.getContainerVolumes = void 0;
+exports.execCommand = exports.convertToPosixPath = exports.startContainer = exports.getContainerCli = void 0;
 const stream_1 = require("../docker/stream");
 const dockerode_1 = __importDefault(require("dockerode"));
-const pull_1 = require("../docker/pull");
 const path_1 = require("path");
 const os_1 = require("os");
-const template_value_1 = require("../planner/utils/template-value");
-const defer_1 = require("../utils/defer");
-const crypto_1 = require("crypto");
-async function getContainerVolumes(node, context) {
-    const volumes = [];
-    for (const generate of node.generates) {
-        const name = `hammerkit-${crypto_1.createHash('sha1').update(generate).digest('hex')}`;
-        volumes.push({
-            name,
-            containerPath: generate,
-        });
-    }
-    return volumes;
-}
-exports.getContainerVolumes = getContainerVolumes;
-async function getContainerMounts(node, context) {
-    const result = [];
-    for (const source of node.src) {
-        const exists = await context.file.exists(source.absolutePath);
-        if (exists) {
-            result.push({
-                localPath: source.absolutePath,
-                containerPath: source.absolutePath,
-            });
-        }
-        else {
-            if (path_1.extname(source.absolutePath)) {
-                await context.file.writeFile(source.absolutePath, '');
-            }
-            else {
-                await context.file.createDirectory(source.absolutePath);
-                result.push({
-                    localPath: source.absolutePath,
-                    containerPath: source.absolutePath,
-                });
-            }
-        }
-    }
-    for (const mount of node.mounts) {
-        result.push(mount);
-    }
-    for (const volume of result) {
-        const otherVolumes = result.filter((v) => v.containerPath === volume.containerPath && v.localPath !== volume.localPath);
-        if (otherVolumes.length > 0) {
-            throw new Error(`duplicate container mount with different sources ${[
-                volume.localPath,
-                ...otherVolumes.map((ov) => ov.localPath),
-            ].join(', ')}`);
-        }
-    }
-    return result.filter((v, i) => result.findIndex((iv) => iv.containerPath == v.containerPath) === i);
-}
-exports.getContainerMounts = getContainerMounts;
-class Lock {
-    constructor(total) {
-        this.total = total;
-        this.count = 0;
-        this.queue = [];
-    }
-    enqueueQueue() {
-        const item = this.queue.pop();
-        if (item) {
-            item.resolve(this.lease());
-        }
-    }
-    lease() {
-        return {
-            close: () => {
-                this.count--;
-                this.enqueueQueue();
-            },
-        };
-    }
-    async acquire() {
-        if (this.count < this.total) {
-            this.count++;
-            return this.lease();
-        }
-        else {
-            const defer = new defer_1.Defer();
-            this.queue.push(defer);
-            return defer.promise;
-        }
-    }
-}
+const abort_event_1 = require("../utils/abort-event");
 let dockerInstance = null;
-const instanceLock = new Lock(2);
-function getDocker() {
+function getContainerCli() {
     if (!dockerInstance) {
         dockerInstance = new dockerode_1.default();
     }
     return dockerInstance;
 }
-async function useDocker(fn) {
-    const lease = await instanceLock.acquire();
-    try {
-        const docker = getDocker();
-        await fn(docker);
-    }
-    finally {
-        lease.close();
-    }
-}
-async function startContainer(container) {
-    const defer = new defer_1.Defer();
-    // TODO watch if start takes too long to finish, report warning
-    container.start().then(() => {
-        defer.resolve();
+exports.getContainerCli = getContainerCli;
+async function startContainer(status, container) {
+    return new Promise((resolve, reject) => {
+        const handle = setTimeout(() => {
+            status.write('warn', 'start of container is potentially stuck on start');
+        }, 1000);
+        container
+            .start()
+            .then(() => {
+            clearTimeout(handle);
+            resolve();
+        })
+            .catch((e) => {
+            clearTimeout(handle);
+            if ('json' in e && 'message' in e.json) {
+                const errorMessage = e.json.message;
+                const portBlocked = /Bind for .*:\d+ failed: port is already allocated/.exec(errorMessage);
+                if (portBlocked) {
+                    reject(new Error(portBlocked[0]));
+                }
+                else {
+                    reject(new Error(errorMessage));
+                }
+            }
+            else if ('json' in e && e.json instanceof Buffer) {
+                reject(new Error(e.json.toString()));
+            }
+            else {
+                reject(e);
+            }
+        });
     });
-    return defer.promise;
 }
+exports.startContainer = startContainer;
 function convertToPosixPath(path) {
-    if (os_1.platform() === 'win32') {
+    if ((0, os_1.platform)() === 'win32') {
         return path
             .split(path_1.sep)
             .map((value, index) => (index === 0 && value.endsWith(':') ? '/' + value.substr(0, value.length - 1) : value))
@@ -129,101 +59,8 @@ function convertToPosixPath(path) {
     }
     return path;
 }
-async function executeDocker(node, context, cancelDefer) {
-    node.status.console.write('internal', 'debug', `execute ${node.name} as docker task`);
-    await useDocker(async (docker) => {
-        const volumes = await getContainerVolumes(node, context.environment);
-        const mounts = await getContainerMounts(node, context.environment);
-        await pull_1.pull(node, docker, node.image);
-        for (const volume of volumes) {
-            try {
-                const existing = await docker.getVolume(volume.name);
-                await existing.inspect();
-            }
-            catch (e) {
-                await docker.createVolume({
-                    Name: volume.name,
-                    Driver: 'local',
-                    Labels: { app: 'hammerkit' },
-                });
-            }
-        }
-        node.status.console.write('internal', 'debug', `create container with image ${node.image} with ${node.shell}`);
-        const container = await docker.createContainer({
-            Image: node.image,
-            Tty: true,
-            Entrypoint: node.shell,
-            Env: Object.keys(node.envs).map((k) => `${k}=${node.envs[k]}`),
-            WorkingDir: convertToPosixPath(node.cwd),
-            Labels: { app: 'hammerkit' },
-            HostConfig: {
-                Binds: [
-                    ...mounts.map((v) => `${v.localPath}:${convertToPosixPath(v.containerPath)}`),
-                    ...volumes.map((v) => `${v.name}:${convertToPosixPath(v.containerPath)}`),
-                ],
-                PortBindings: node.ports.reduce((map, port) => {
-                    map[`${port.containerPort}/tcp`] = [{ HostPort: `${port.hostPort}` }];
-                    return map;
-                }, {}),
-            },
-            ExposedPorts: node.ports.reduce((map, port) => {
-                map[`${port.containerPort}/tcp`] = {};
-                return map;
-            }, {}),
-        });
-        for (const mount of mounts) {
-            node.status.console.write('internal', 'debug', `bind mount ${mount.localPath}:${mount.containerPath}`);
-        }
-        for (const volume of volumes) {
-            node.status.console.write('internal', 'debug', `volume mount ${volume.name}:${volume.containerPath}`);
-        }
-        cancelDefer.promise.then(() => {
-            container.remove({ force: true }).catch(() => {
-                node.status.console.write('internal', 'debug', `remove of container failed`);
-            });
-        });
-        const user = os_1.platform() === 'linux' || os_1.platform() === 'freebsd' || os_1.platform() === 'openbsd' || os_1.platform() === 'sunos'
-            ? `${process.getuid()}:${process.getgid()}`
-            : undefined;
-        try {
-            node.status.console.write('internal', 'info', `starting container with image ${node.image}`);
-            await startContainer(container);
-            if (user) {
-                const setUserPermission = async (directory) => {
-                    node.status.console.write('internal', 'debug', 'set permission on ' + directory);
-                    const result = await execCommand(node, docker, container, '/', ['chown', user, directory], undefined);
-                    if (result.ExitCode !== 0) {
-                        node.status.console.write('internal', 'warn', `unable to set permissions for ${directory}`);
-                    }
-                };
-                await setUserPermission(node.cwd);
-                for (const mount of mounts) {
-                    await setUserPermission(mount.containerPath);
-                }
-            }
-            for (const cmd of node.cmds) {
-                if (cancelDefer.isResolved) {
-                    return;
-                }
-                const command = template_value_1.templateValue(cmd.cmd, node.envs);
-                node.status.console.write('internal', 'info', `execute cmd ${command} in container`);
-                const result = await execCommand(node, docker, container, convertToPosixPath(cmd.path), [node.shell, '-c', command], user);
-                if (result.ExitCode !== 0) {
-                    node.status.console.write('internal', 'error', `command ${command} failed with ${result.ExitCode}`);
-                    throw new Error(`command ${command} failed with ${result.ExitCode}`);
-                }
-            }
-        }
-        finally {
-            node.status.console.write('internal', 'debug', `remove container`);
-            await container.remove({ force: true }).catch(() => {
-                node.status.console.write('internal', 'debug', `remove of container failed`);
-            });
-        }
-    });
-}
-exports.executeDocker = executeDocker;
-async function execCommand(node, docker, container, cwd, cmd, user) {
+exports.convertToPosixPath = convertToPosixPath;
+async function execCommand(status, environment, container, cwd, cmd, user, timeout, abort) {
     const exec = await container.exec({
         Cmd: cmd,
         WorkingDir: cwd,
@@ -231,32 +68,45 @@ async function execCommand(node, docker, container, cwd, cmd, user) {
         AttachStdout: true,
         AttachStdin: true,
         AttachStderr: true,
-        User: user,
+        User: user ?? undefined,
     });
-    const defer = new defer_1.Defer();
-    node.status.console.write('internal', 'debug', `received exec id ${exec.id}`);
+    status.write('debug', `received exec id ${exec.id}`);
     const stream = await exec.start({ stdin: true, hijack: true, Detach: false, Tty: false });
-    stream_1.awaitStream(node, docker, stream).then(async () => {
-        if (!defer.isResolved) {
-            const result = await exec.inspect();
-            if (!defer.isResolved) {
-                defer.resolve(result);
+    return new Promise((resolve, reject) => {
+        let resolved = false;
+        if (abort) {
+            (0, abort_event_1.listenOnAbort)(abort, () => {
+                if (resolved) {
+                    return;
+                }
+                resolved = true;
+                resolve({ type: 'canceled' });
+            });
+        }
+        (0, stream_1.awaitStream)(status, stream)
+            .then(() => {
+            if (resolved) {
+                return;
             }
+            return exec.inspect().then((result) => {
+                if (resolved) {
+                    return;
+                }
+                resolve({ type: 'result', result });
+                resolved = true;
+            });
+        })
+            .catch(reject);
+        if (timeout) {
+            setTimeout(() => {
+                if (resolved) {
+                    return;
+                }
+                resolve({ type: 'timeout' });
+                resolved = true;
+            }, timeout);
         }
     });
-    pollStatus(exec, defer);
-    return defer.promise;
-}
-function pollStatus(exec, defer) {
-    async function inspect() {
-        const result = await exec.inspect();
-        if (!result.Running && !defer.isResolved) {
-            defer.resolve(result);
-        }
-        if (!defer.isResolved) {
-            setTimeout(() => inspect(), 50);
-        }
-    }
-    inspect();
 }
+exports.execCommand = execCommand;
 //# sourceMappingURL=execute-docker.js.map

package/dist/executer/execute.spec.js

@@ -1,46 +1,46 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const create_work_tree_1 = require("../testing/create-work-tree");
-const execute_1 = require("./execute");
-const get_execution_context_mock_1 = require("./get-execution-context-mock");
+const get_test_suite_1 = require("../testing/get-test-suite");
+const expect_1 = require("../testing/expect");
 describe('execute', () => {
-    let ctx;
-    let workTree;
-    let nodeId;
-    beforeEach(async () => {
-        ctx = get_execution_context_mock_1.getExecutionContextMock();
-        ctx.watch = true;
-        workTree = await create_work_tree_1.createWorkTree(ctx.environment, {
-            tasks: {
-                api: {
-                    cmds: ['node index.js'],
-                    src: ['index.js', 'package.json'],
-                },
-            },
-        }, 'api');
-        nodeId = workTree.rootNode.id;
-        await ctx.environment.file.createDirectory(ctx.environment.cwd);
-        await ctx.environment.file.writeFile(`${ctx.environment.cwd}/index.js`, "console.log('hello')");
-        await ctx.environment.file.writeFile(`${ctx.environment.cwd}/package.json`, '{}');
-    });
+    const suite = (0, get_test_suite_1.getTestSuite)('hello-world-node', ['build.yaml', 'package.json', 'index.js']);
+    afterAll(() => suite.close());
     it('should restart watching task if once completed', async () => {
-        const resultPromise = execute_1.execute(workTree, ctx);
-        const runningApi = await ctx.executor.waitFor(nodeId);
-        runningApi.resolve();
-        await ctx.environment.file.appendFile(`${ctx.environment.cwd}/index.js`, '\n');
-        await ctx.executor.waitFor(nodeId);
-        await ctx.environment.cancelDefer.resolve();
-        const result = await resultPromise;
-        expect(result.success).toBeFalsy();
+        const { cli, environment } = await suite.setup({ taskName: 'api' });
+        const node = cli.node('api');
+        const exec = cli.execWatch({ watch: true });
+        let count = 0;
+        exec.state.on((state) => {
+            if (state.node[node.id].type === 'completed') {
+                count++;
+                if (count === 1) {
+                    environment.file.appendFile(`${environment.cwd}/index.js`, '\n');
+                }
+                else if (count === 2) {
+                    environment.abortCtrl.abort();
+                }
+            }
+        });
+        const result = await exec.start();
+        await (0, expect_1.expectSuccessfulResult)(result, environment);
     });
     it('should restart watching task if once failed', async () => {
-        const resultPromise = execute_1.execute(workTree, ctx);
-        const runningApi = await ctx.executor.waitFor(nodeId);
-        runningApi.reject(new Error('runtime error'));
-        await ctx.environment.file.appendFile(`${ctx.environment.cwd}/index.js`, '\n');
-        await ctx.executor.waitFor(nodeId);
-        await ctx.environment.cancelDefer.resolve();
-        const result = await resultPromise;
+        const { cli, environment } = await suite.setup({ taskName: 'api_crashing' });
+        const node = cli.node('api_crashing');
+        const exec = cli.execWatch({ watch: true });
+        let count = 0;
+        exec.state.on((state) => {
+            if (state.node[node.id].type === 'crash') {
+                count++;
+                if (count === 1) {
+                    environment.file.appendFile(`${environment.cwd}/index.js`, '\n');
+                }
+                else if (count === 2) {
+                    environment.abortCtrl.abort();
+                }
+            }
+        });
+        const result = await exec.start();
         expect(result.success).toBeFalsy();
     });
 });

package/dist/executer/execution-steps.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setUserPermissions = exports.prepareMounts = exports.prepareVolume = exports.pullImage = void 0;
+const work_node_1 = require("../planner/work-node");
+const pull_1 = require("../docker/pull");
+const get_docker_executor_1 = require("./get-docker-executor");
+const work_service_1 = require("../planner/work-service");
+const path_1 = require("path");
+const set_user_permission_1 = require("./set-user-permission");
+async function pullImage(node, environment) {
+    if ((0, work_node_1.isContainerWorkNode)(node)) {
+        await (0, pull_1.pull)(environment.status.task(node), environment, node.image);
+    }
+    else if ((0, work_service_1.isContainerWorkService)(node)) {
+        await (0, pull_1.pull)(environment.status.service(node), environment, node.image);
+    }
+}
+exports.pullImage = pullImage;
+async function prepareVolume(node, environment) {
+    if ((0, work_node_1.isContainerWorkNode)(node) || (0, work_service_1.isContainerWorkService)(node)) {
+        for (const volume of node.volumes) {
+            await (0, get_docker_executor_1.ensureVolumeExists)(environment, volume.name);
+        }
+    }
+}
+exports.prepareVolume = prepareVolume;
+async function prepareMounts(node, environment) {
+    for (const mount of node.mounts) {
+        const exists = await environment.file.exists(mount.localPath);
+        if (exists) {
+            continue;
+        }
+        if ((0, path_1.extname)(mount.localPath)) {
+            await environment.file.writeFile(mount.localPath, '');
+        }
+        else {
+            await environment.file.createDirectory(mount.localPath);
+        }
+    }
+}
+exports.prepareMounts = prepareMounts;
+async function setUserPermissions(node, container, environment) {
+    const status = environment.status.from(node);
+    if (node.user) {
+        if ((0, work_node_1.isContainerWorkNode)(node)) {
+            await (0, set_user_permission_1.setUserPermission)(node.cwd, status, environment, container, node.user);
+        }
+        for (const volume of node.volumes) {
+            await (0, set_user_permission_1.setUserPermission)(volume.containerPath, status, environment, container, node.user);
+        }
+        for (const mount of node.mounts) {
+            await (0, set_user_permission_1.setUserPermission)(mount.containerPath, status, environment, container, node.user);
+        }
+    }
+}
+exports.setUserPermissions = setUserPermissions;
+//# sourceMappingURL=execution-steps.js.map

package/dist/executer/get-docker-executor.js

@@ -1,24 +1,25 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDockerExecutor = void 0;
-const work_node_1 = require("../planner/work-node");
-const execute_docker_1 = require("./execute-docker");
-const get_local_executor_1 = require("./get-local-executor");
-const replace_env_variables_1 = require("../environment/replace-env-variables");
-function getDockerExecutor() {
-    const localExec = get_local_executor_1.getLocalExecutor();
-    return {
-        async exec(node, context, cancelDefer) {
-            if (!work_node_1.isContainerWorkNode(node)) {
-                return localExec.exec(node, context, cancelDefer);
-            }
-            const envs = replace_env_variables_1.replaceEnvVariables(node, context.environment.processEnvs);
-            await execute_docker_1.executeDocker({
-                ...node,
-                envs,
-            }, context, cancelDefer);
-        },
-    };
+exports.ensureVolumeExists = exports.existsVolume = void 0;
+async function existsVolume(environment, volumeName) {
+    try {
+        const volume = await environment.docker.getVolume(volumeName);
+        return await volume.inspect();
+    }
+    catch (e) {
+        return false;
+    }
 }
-exports.getDockerExecutor = getDockerExecutor;
+exports.existsVolume = existsVolume;
+async function ensureVolumeExists(environment, volumeName) {
+    const volumeExists = await existsVolume(environment, volumeName);
+    if (!volumeExists) {
+        await environment.docker.createVolume({
+            Name: volumeName,
+            Driver: 'local',
+            Labels: { app: 'hammerkit' },
+        });
+    }
+}
+exports.ensureVolumeExists = ensureVolumeExists;
 //# sourceMappingURL=get-docker-executor.js.map

package/dist/executer/hierarchy-scheduler.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.schedule = void 0;
+const work_service_1 = require("../planner/work-service");
+const docker_service_1 = require("./docker-service");
+const kubernetes_service_1 = require("./kubernetes-service");
+const work_node_1 = require("../planner/work-node");
+const docker_node_1 = require("./docker-node");
+const local_node_1 = require("./local-node");
+const work_node_status_1 = require("../planner/work-node-status");
+async function schedule(processManager, state, environment) {
+    state.on((currentState) => {
+        for (const [, nodeState] of Object.entries(currentState.node)) {
+            if (nodeState.type === 'pending') {
+                const endedNeeds = nodeState.node.needs
+                    .map((need) => currentState.service[need.id])
+                    .filter((service) => service.type === 'end');
+                if (endedNeeds.length > 0) {
+                    state.patchNode({
+                        type: 'canceled',
+                        node: nodeState.node,
+                    });
+                    continue;
+                }
+                const pendingNeeds = nodeState.node.needs
+                    .map((need) => currentState.service[need.id])
+                    .filter((service) => service.type === 'pending');
+                const hasOpenDeps = nodeState.node.deps.some((dep) => currentState.node[dep.id].type !== 'completed');
+                if (hasOpenDeps) {
+                    continue;
+                }
+                if (pendingNeeds.length > 0) {
+                    // TODO do not start, if needs are cached
+                    for (const pendingNeed of pendingNeeds) {
+                        const ctx = (0, work_node_status_1.logContext)('service', pendingNeed.service);
+                        state.patchService({
+                            type: 'running',
+                            service: pendingNeed.service,
+                            abortController: processManager.background(ctx, (0, work_service_1.isContainerWorkService)(pendingNeed.service)
+                                ? (0, docker_service_1.dockerService)(pendingNeed.service, state, environment)
+                                : (0, kubernetes_service_1.kubernetesService)(pendingNeed.service, state, environment)),
+                        });
+                    }
+                    continue;
+                }
+                const hasNotReadyNeeds = nodeState.node.needs.some((need) => currentState.service[need.id].type !== 'ready');
+                if (hasNotReadyNeeds) {
+                    continue;
+                }
+                const serviceContainers = {};
+                for (const need of nodeState.node.needs) {
+                    const serviceState = currentState.service[need.id];
+                    if (serviceState.type === 'ready') {
+                        serviceContainers[need.id] = serviceState.dns;
+                    }
+                }
+                const ctx = (0, work_node_status_1.logContext)('task', nodeState.node);
+                state.patchNode({
+                    type: 'starting',
+                    node: nodeState.node,
+                    started: new Date(),
+                    abortController: (0, work_node_1.isContainerWorkNode)(nodeState.node)
+                        ? processManager.task(ctx, (0, docker_node_1.dockerNode)(nodeState.node, serviceContainers, state, environment))
+                        : processManager.task(ctx, (0, local_node_1.localNode)(nodeState.node, state, environment)),
+                });
+            }
+        }
+        for (const [serviceId, serviceState] of Object.entries(currentState.service)) {
+            if (serviceState.type !== 'ready' && serviceState.type !== 'running') {
+                continue;
+            }
+            let hasNeed = false;
+            for (const nodeState of Object.values(currentState.node)) {
+                if (nodeState.type === 'running' || nodeState.type === 'starting' || nodeState.type === 'pending') {
+                    if (nodeState.node.needs.some((n) => n.id === serviceId)) {
+                        hasNeed = true;
+                    }
+                }
+            }
+            if (!hasNeed) {
+                environment.status.service(serviceState.service).write('info', 'stop unused service');
+                serviceState.abortController.abort();
+            }
+        }
+    });
+    await processManager.onComplete();
+    const success = !Object.values(state.current.node).some((n) => n.type !== 'completed');
+    return {
+        state: state.current,
+        success,
+    };
+}
+exports.schedule = schedule;
+//# sourceMappingURL=hierarchy-scheduler.js.map