hammerkit 1.4.0 → 1.5.0-alpha.1

package/.devcontainer/Dockerfile

@@ -0,0 +1,5 @@
+FROM node:18-alpine
+
+RUN apk add git npm
+
+USER root

package/.devcontainer/devcontainer.json

@@ -0,0 +1,40 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.195.0/containers/javascript-node
+{
+	"name": "Hammerkit",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			// Set *default* container specific settings.json values on container create.
+			"settings": {},
+			
+			// Add the IDs of extensions you want installed when the container is created.
+			"extensions": [
+				"dbaeumer.vscode-eslint"
+			]
+		}
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [3000],
+
+	// Use 'portsAttributes' to set default properties for specific forwarded ports. More info: https://code.visualstudio.com/docs/remote/devcontainerjson-reference.
+	"portsAttributes": {
+	},
+
+	// Use 'otherPortsAttributes' to configure any ports that aren't configured using 'portsAttributes'.
+	// "otherPortsAttributes": {
+	// 		"onAutoForward": "silent"
+	// },
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "npm install",
+
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "node"
+}

package/README.md

@@ -1,97 +1,24 @@
-<h1 align="center">
-  <img src="docs/hammerkit.png" alt="hammerkit" width="250">
-</h1>
-<h2 align="center">Hammerkit</h2>
+# About
 
-<p align='center'>
-  <img alt='build' src='https://img.shields.io/github/workflow/status/no0dles/hammerkit/master'>
-  <img alt='license' src='https://img.shields.io/npm/l/hammerkit'>
-  <img alt='npm' src='https://img.shields.io/npm/v/hammerkit'>
-</p>
+<figure><img src="docs/hammerkit-small.png" width="144" alt=""><figcaption></figcaption></figure>
 
-## About
+## What is hammerkit?
 
-Hammerkit is a build tool intended to build software projects in containers locally and in a CI. The advantage of containerized builds are:
-- the build tools used from the container image are the same across all machines.
-- the build process is isolated since every file in the container has to be declared as a source or an output. Therefore side effect should never occur.
+Hammerkit is a build tool intended to build software projects in **containers** locally and in a CI. The advantage of containerized builds are:
+
+* the build tools used from the container image are the same across all machines.
+* the build process is isolated since every file in the container has to be declared as a source or an output. Therefore side effects can be reduced.
 
 There are a lot of containerized CI systems that allow writing containerized builds today, but they lack some features hammerkit tries to solve:
-- they are often not usable during development on your local machine, so you have either maintain two build scripts or wrap one of them in the other.
-- they mount the entire repository into the container and can therefore side effect can reduce reliability
-- some of them do not allow switching images during build steps. Requiring to maintain a large build image, that contains all tools needed to build to software.
+
+* they are often not **usable during development** on your local machine, so you have either maintain two build scripts or wrap one of them in the other.
+* they mount the entire repository into the container and can therefore **side effect** can reduce reliability
+* some of them do not allow **switching images** during build steps. Requiring to maintain a large build image, that contains all tools needed to build to software.
 
 Additionally hammerkit tries to reduce complexity of build caching in CI systems. Usually you define a directory that is being cached and restored before and after the CI build. That's totally fine for smaller project where there are not that many directories to cache. The bigger the project gets, the more effort it is, to keep all directories up to date.
 
 Hammerkit on the other hand knows the source and output files of each build step and can therefore compact the build results into a single directory. Making it easy to being cached. Once restored, hammerkit can detect what's changed since last time and only build the changes.
 
-<p align='center'>
-    <a href="https://no0dles.gitbook.io/hammerkit">
-        <img src="docs/read_the_docs.png" alt="docs">
-    </a>
-</p>
-
-## Installation
-
-### Yarn/npm
-```npm i -g hammerkit```
-
-### Homebrew
-```
-brew tap no0dles/hammerkit
-brew install hammerkit
-```
-
-### Gitlab CI
-```
-variables:
-  DOCKER_DRIVER: overlay2
-
-services:
-  - docker:19.03.0-dind
-
-build:
-  image: no0dles/hammerkit
-  script:
-    - hammerkit build
-```
-
-### Github Action
-```
-jobs:
-  build:
-    runs-on: ubuntu-18.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - uses: no0dles/hammerkit-github-action@v1.3
-```
-
-## Roadmap
-The following topics are planned to be addressed in the future.
-
-### Services
-Task sometimes require some service to run. 
-For example a database, which is required to run an api or an integration test.
-The goal will be to define those and hammerkit spins them up for the task and shuts them down if not needed. 
-
-### Distributed caching
-Distribute local cache, so other developers and the CI can use already build up caches.
-The goal will be general performance improvements on CI and local development.
-
-### Distributed computing
-Offload complex and compute intense tasks to remote hardware.
-The goal will be use the power of cloud computing, with very less configuration.
-
-### Platform requirements
-Define tasks that require a specific platform for local tasks.
-The goal will be to either skip local tasks where platform requirements are not met or execute them on a remote machine.
-
+## Where to go next?
 
-## Release
-```
-npm version patch|minor|major|prerelease
-git push origin --tags
-```
+Check out the [docs](https://no0dles.gitbook.io/hammerkit/) for more info about how hammerkit can be used.

package/SUMMARY.md

@@ -0,0 +1,43 @@
+# Table of contents
+
+* [About](README.md)
+* [Installation](docs/installation.md)
+* [Getting started](docs/getting-started.md)
+* [Build file](docs/build-file/README.md)
+  * [Environment Variables](docs/build-file/environment-variables.md)
+  * [References](docs/build-file/references.md)
+  * [Includes](docs/build-file/includes.md)
+* [Task](docs/task/README.md)
+  * [Dependencies](docs/task/dependencies.md)
+  * [Needs](docs/task/needs.md)
+  * [Caching](docs/task/caching.md)
+  * [Container](docs/task/container.md)
+  * [Watching](docs/task/watching.md)
+  * [Extending](docs/task/extending.md)
+* [Service](docs/service/README.md)
+  * [Container](docs/service/container.md)
+  * [Kubernetes](docs/service/kubernetes.md)
+* [Labels](docs/labels/README.md) 
+* [CLI](docs/cli/README.md)
+  * [Init](docs/cli/init.md)
+  * [ls](docs/cli/ls.md)
+  * [Execute](docs/cli/execute.md)
+  * [Store / Restore](docs/cli/store-restore.md)
+  * [Clean](docs/cli/clean.md)
+  * [Validate](docs/cli/validate.md)
+  * [Help](docs/cli/help.md)
+
+## External Links
+
+* [Docker Hub](https://hub.docker.com/r/no0dles/hammerkit)
+* [Homebrew Repository](https://github.com/no0dles/homebrew-hammerkit)
+
+## Release Blog
+
+* [Release 1.4.0](docs/release-blog/release-1.4.0.md)
+* [Release 1.5.0](docs/release-blog/release-1.5.0.md)
+
+## Contribution
+
+* [Roadmap](docs/contribution/roadmap.md)
+* [Publishing](docs/contribution/publish.md)

package/build.schema.json

@@ -15,6 +15,13 @@
         "$ref": "#/definitions/task"
       }
     },
+    "services": {
+      "type": "object",
+      "description": "A map of services",
+      "additionalProperties": {
+        "$ref": "#/definitions/service"
+      }
+    },
     "includes": {
       "$ref": "#/definitions/include"
     },
@@ -23,6 +30,11 @@
     }
   },
   "definitions": {
+    "labels": {
+      "description": "A map of label values",
+      "type": "object",
+      "additionalProperties": {"type": "string"}
+    },
     "envMap": {
       "description": "A map of environment values",
       "type": "object",
@@ -88,7 +100,7 @@
         "type": "string"
       }
     },
-    "taskMount": {
+    "mount": {
       "type": "array",
       "description": "A list of file paths that get mounted into the container",
       "items": {
@@ -115,6 +127,9 @@
           "type": "string",
           "description": "Name of task that gets used a template"
         },
+        "labels": {
+          "$ref": "#/definitions/labels"
+        },
         "envs": {
           "$ref": "#/definitions/envMap"
         },
@@ -149,6 +164,9 @@
           "type": "string",
           "description": "A description of the task"
         },
+        "labels": {
+          "$ref": "#/definitions/labels"
+        },
         "extend": {
           "type": "string",
           "description": "Name of task that gets used a template"
@@ -175,7 +193,7 @@
           "description": "Define the container image the commands run in"
         },
         "mounts": {
-          "$ref": "#/definitions/taskMount"
+          "$ref": "#/definitions/mount"
         },
         "src": {
           "$ref": "#/definitions/taskSource"
@@ -198,6 +216,42 @@
       }, {
         "$ref": "#/definitions/localTask"
       }]
+    },
+    "service": {
+      "description": "A daemon in the background",
+      "required": ["image"],
+      "additionalProperties": false,
+      "properties": {
+        "image": {
+          "type": "string",
+          "description": "Define the container image the commands run in"
+        },
+        "labels": {
+          "$ref": "#/definitions/labels"
+        },
+        "envs": {
+          "$ref": "#/definitions/envMap"
+        },
+        "ports": {
+          "$ref": "#/definitions/taskPort"
+        },
+        "healthcheck": {
+          "$ref": "#/definitions/serviceHealthcheck"
+        },
+        "mounts": {
+          "$ref": "#/definitions/mount"
+        }
+      }
+    },
+    "serviceHealthcheck": {
+      "additionalItems": false,
+      "description": "A check to detect if the service is started and ready",
+      "properties": {
+        "cmd": {
+          "description": "task to check if the service is healthy",
+          "type": "string"
+        }
+      }
     }
   }
 }

package/dist/cli.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCli = exports.isCliService = exports.isCliTask = void 0;
+const plan_work_nodes_1 = require("./planner/utils/plan-work-nodes");
+const create_scheduler_state_1 = require("./executer/create-scheduler-state");
+const ci_1 = require("./utils/ci");
+const get_logger_1 = require("./console/get-logger");
+const schedule_1 = require("./executer/schedule");
+const event_cache_1 = require("./executer/event-cache");
+const validate_1 = require("./planner/validate");
+const process_manager_1 = require("./executer/process-manager");
+const remove_container_1 = require("./docker/remove-container");
+const isCliTask = (val) => val.type === 'task';
+exports.isCliTask = isCliTask;
+const isCliService = (val) => val.type === 'service';
+exports.isCliService = isCliService;
+function getCli(workTree, environment) {
+    return {
+        execWatch(options) {
+            const processManager = new process_manager_1.ProcessManager(environment, options?.workers ?? 0);
+            const state = (0, create_scheduler_state_1.createSchedulerState)({
+                services: workTree.services,
+                nodes: workTree.nodes,
+                watch: options?.watch ?? false,
+                logMode: options?.logMode ?? ci_1.isCI ? 'live' : 'interactive',
+                cacheMethod: options?.cacheDefault ?? 'checksum',
+            });
+            const logMode = options?.logMode ?? (ci_1.isCI ? 'live' : 'interactive');
+            const logger = (0, get_logger_1.getLogger)(logMode, state, environment);
+            return {
+                state,
+                processManager,
+                start: async () => {
+                    const result = await (0, schedule_1.schedule)(processManager, state, environment);
+                    await logger.complete(result, environment);
+                    return result;
+                },
+            };
+        },
+        async shutdown() {
+            for (const node of (0, plan_work_nodes_1.iterateWorkNodes)(workTree.nodes)) {
+                const containers = await environment.docker.listContainers({
+                    all: true,
+                    filters: {
+                        label: [`hammerkit-id=${node.id}`],
+                    },
+                });
+                for (const container of containers) {
+                    await (0, remove_container_1.removeContainer)(environment.docker.getContainer(container.Id));
+                }
+            }
+            for (const service of (0, plan_work_nodes_1.iterateWorkServices)(workTree.services)) {
+                const containers = await environment.docker.listContainers({
+                    all: true,
+                    filters: {
+                        label: [`hammerkit-id=${service.id}`],
+                    },
+                });
+                for (const container of containers) {
+                    await (0, remove_container_1.removeContainer)(environment.docker.getContainer(container.Id));
+                }
+            }
+        },
+        async exec(options) {
+            return this.execWatch(options).start();
+        },
+        async clean(options) {
+            await (0, event_cache_1.cleanCache)(workTree, environment, {
+                service: options?.service ?? false,
+            });
+        },
+        async restore(path) {
+            await (0, event_cache_1.restoreCache)(path, workTree, environment);
+        },
+        async store(path) {
+            await (0, event_cache_1.storeCache)(path, workTree, environment);
+        },
+        ls() {
+            return [
+                ...Array.from((0, plan_work_nodes_1.iterateWorkServices)(workTree.services)).map((item) => ({ item, type: 'service' })),
+                ...Array.from((0, plan_work_nodes_1.iterateWorkNodes)(workTree.nodes)).map((item) => ({ item, type: 'task' })),
+            ];
+        },
+        validate() {
+            return (0, validate_1.validate)(workTree, environment);
+        },
+        node(name) {
+            const node = Object.values(workTree.nodes).find((n) => n.name == name);
+            if (!node) {
+                throw new Error(`unable to find node ${name}`);
+            }
+            return node;
+        },
+    };
+}
+exports.getCli = getCli;
+//# sourceMappingURL=cli.js.map

package/dist/cli.spec.js

@@ -0,0 +1,5 @@
+"use strict";
+describe('cli', () => {
+    it('should warn about empty cli', () => { });
+});
+//# sourceMappingURL=cli.spec.js.map

package/dist/console/console-context-mock.js

@@ -1,3 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.consoleContextMock = void 0;
+/* eslint-disable @typescript-eslint/no-empty-function */
+function consoleContextMock() {
+    return {
+        warn() { },
+        info() { },
+        error() { },
+        debug() { },
+    };
+}
+exports.consoleContextMock = consoleContextMock;
 //# sourceMappingURL=console-context-mock.js.map

package/dist/console/get-logger.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getLogger = void 0;
+const grouped_logger_1 = require("../logging/grouped-logger");
+const live_logger_1 = require("../logging/live-logger");
+const interactive_logger_1 = require("../logging/interactive-logger");
+const fail_never_1 = require("../utils/fail-never");
+function getLogger(mode, state, env) {
+    if (mode === 'grouped') {
+        return (0, grouped_logger_1.groupedLogger)(state, env);
+    }
+    else if (mode === 'live') {
+        return (0, live_logger_1.liveLogger)(state, env);
+    }
+    else if (mode === 'interactive') {
+        return (0, interactive_logger_1.interactiveLogger)(state, env);
+    }
+    else {
+        (0, fail_never_1.failNever)(mode, 'unknown log mode');
+    }
+}
+exports.getLogger = getLogger;
+//# sourceMappingURL=get-logger.js.map

package/dist/docker/pull.js

@@ -1,19 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.pull = void 0;
-async function pull(node, docker, imageName) {
+async function pull(status, environment, imageName) {
     let searchImageName = imageName;
     if (imageName.indexOf(':') === -1) {
         searchImageName += ':latest';
     }
-    const images = await docker.listImages({});
+    const images = await environment.docker.listImages({});
     if (images.some((i) => i.RepoTags?.some((repoTag) => repoTag === searchImageName))) {
         return;
     }
-    node.status.console.write('internal', 'debug', `pull image ${imageName}`);
-    const image = await docker.pull(imageName);
+    status.write('debug', `pull image ${imageName}`);
+    const image = await environment.docker.pull(imageName);
     await new Promise((resolve, reject) => {
-        docker.modem.followProgress(image, (err, res) => (err ? reject(err) : resolve(res)));
+        environment.docker.modem.followProgress(image, (err, res) => (err ? reject(err) : resolve(res)));
     });
 }
 exports.pull = pull;

package/dist/docker/remove-container.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeContainer = void 0;
+const sleep_1 = require("../utils/sleep");
+async function removeContainer(container) {
+    try {
+        await container.remove({ force: true });
+    }
+    catch (e) {
+        if (e.statusCode === 404) {
+            return;
+        }
+        else if (e.statusCode === 409) {
+            await (0, sleep_1.sleep)(500);
+            return removeContainer(container);
+        }
+        throw e;
+    }
+}
+exports.removeContainer = removeContainer;
+//# sourceMappingURL=remove-container.js.map

package/dist/docker/stream.js

@@ -1,23 +1,27 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.awaitStream = void 0;
+exports.logStream = exports.awaitStream = void 0;
 const log_1 = require("../log");
-async function awaitStream(node, docker, stream) {
+async function awaitStream(status, stream) {
     return new Promise((resolve, reject) => {
-        function writeLog(leve) {
-            return (buffer) => {
-                for (const log of log_1.getLogs(buffer)) {
-                    node.status.console.write('process', leve, log.endsWith('\n') ? log.substr(0, log.length - 1) : log);
-                }
-            };
-        }
+        logStream(status, stream);
         stream.on('error', reject);
         stream.on('end', resolve);
         stream.on('close', resolve);
-        demuxStream(stream, writeLog('info'), writeLog('error'));
     });
 }
 exports.awaitStream = awaitStream;
+function logStream(status, stream) {
+    demuxStream(stream, writeLog(status, 'stdout'), writeLog(status, 'stderr'));
+}
+exports.logStream = logStream;
+function writeLog(status, level) {
+    return async (buffer) => {
+        for (const log of (0, log_1.getLogs)(buffer)) {
+            status.console(level, log.endsWith('\n') ? log.substr(0, log.length - 1) : log);
+        }
+    };
+}
 function demuxStream(stream, stdoutFn, stderrFn) {
     let nextDataType = null;
     let nextDataLength = null;

package/dist/docker/using-container.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.usingContainer = void 0;
+const remove_container_1 = require("./remove-container");
+const log_1 = require("../log");
+const execute_docker_1 = require("../executer/execute-docker");
+async function usingContainer(environment, node, createOptions, callback) {
+    const status = environment.status.from(node);
+    let container = null;
+    try {
+        container = await environment.docker.createContainer(createOptions);
+        status.write('debug', `starting container with image ${node.image}`);
+        await (0, execute_docker_1.startContainer)(status, container);
+        return await callback(container);
+    }
+    finally {
+        if (container) {
+            try {
+                await (0, remove_container_1.removeContainer)(container);
+            }
+            catch (e) {
+                status.write('error', `remove of container failed ${(0, log_1.getErrorMessage)(e)}`);
+            }
+        }
+    }
+}
+exports.usingContainer = usingContainer;
+//# sourceMappingURL=using-container.js.map

package/dist/environment/get-process-env.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getProcessEnvs = void 0;
-function getProcessEnvs(envs, ctx) {
+exports.extendHostVariables = void 0;
+function extendHostVariables(envs, ctx) {
     const result = { ...envs };
-    for (const key of Object.keys(ctx.processEnvs)) {
+    for (const key of ['PATH']) {
         const value = ctx.processEnvs[key];
         if (!result[key] && value) {
             result[key] = value;
@@ -11,5 +11,5 @@ function getProcessEnvs(envs, ctx) {
     }
     return result;
 }
-exports.getProcessEnvs = getProcessEnvs;
+exports.extendHostVariables = extendHostVariables;
 //# sourceMappingURL=get-process-env.js.map

package/dist/environment/replace-env-variables.js

@@ -1,14 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.replaceEnvVariables = void 0;
-function replaceEnvVariables(node, processEnv) {
-    const result = { ...node.envs };
+function replaceEnvVariables(envs, processEnv) {
+    const result = { ...envs };
     for (const key of Object.keys(result)) {
         const value = result[key];
         if (value.startsWith('$')) {
             const processEnvValue = processEnv[value.substr(1)];
             if (processEnvValue) {
-                node.status.console.write('internal', 'debug', `use process env ${value.substr(1)}`);
                 result[key] = processEnvValue;
             }
             else {

package/dist/executer/abort.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbortError = exports.checkForAbort = void 0;
+function checkForAbort(abortSignal) {
+    if (abortSignal.aborted) {
+        throw new AbortError();
+    }
+}
+exports.checkForAbort = checkForAbort;
+class AbortError extends Error {
+}
+exports.AbortError = AbortError;
+//# sourceMappingURL=abort.js.map

package/dist/executer/check-readiness.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.checkReadiness = void 0;
+const execute_docker_1 = require("./execute-docker");
+async function checkReadiness(status, healthCheck, environment, container, abort) {
+    const result = await (0, execute_docker_1.execCommand)(status, environment, container, undefined, healthCheck.cmd.split(' '), null, 2000, abort);
+    if (result.type === 'timeout') {
+        return false;
+    }
+    else if (result.type === 'canceled') {
+        return false;
+    }
+    else {
+        if (result.result.ExitCode === 0) {
+            status.write('debug', `healthcheck ${healthCheck.cmd} succeeded`);
+            return true;
+        }
+        else {
+            status.write('debug', `healthcheck ${healthCheck.cmd} failed with ${result.result.ExitCode}`);
+            return false;
+        }
+    }
+}
+exports.checkReadiness = checkReadiness;
+//# sourceMappingURL=check-readiness.js.map