hammerkit 1.6.0-alpha.3 → 1.7.0

package/README.md

@@ -4,22 +4,22 @@
 
 ## What is hammerkit?
 
-Hammerkit is a build tool intended to build software projects in **containers** locally and in a CI. The advantages of containerized builds are:
+Hammerkit is a build tool intended to build software projects in **containers** locally and in CI. The advantages of containerized builds are:
 
 * the build tools come from the container image, so they are the same across all machines.
 * the build process is isolated since every file in the container has to be declared as a source or an output. Therefore, side effects can be reduced.
 
 For a fuller pitch — including how it compares to Make, npm scripts, Taskfile and Earthly — see [Why hammerkit](docs/why-hammerkit.md).
 
-There are a lot of containerized CI systems that allow writing containerized builds today, but they lack some features hammerkit tries to solve:
+There are a lot of containerized CI systems that allow writing containerized builds today, but they have shortcomings that hammerkit tries to address:
 
 * they are often not **usable during development** on your local machine, so you either have to maintain two build scripts or wrap one of them in the other.
 * they mount the entire repository into the container, so uncontrolled **side effects** can reduce reliability.
 * some of them do not allow **switching images** between build steps, requiring you to maintain one large build image that contains every tool needed to build the software.
 
-Additionally, hammerkit tries to reduce complexity of build caching in CI systems. Usually you define a directory that is being cached and restored before and after the CI build. That's totally fine for smaller project where there are not that many directories to cache. The bigger the project gets, the more effort it is, to keep all directories up to date.
+Additionally, hammerkit tries to reduce the complexity of build caching in CI systems. Usually you define a directory that is restored before and cached after the CI build. That's fine for smaller projects with only a few directories to cache. The bigger the project gets, the more effort it takes to keep all of those directories up to date.
 
-Hammerkit on the other hand knows the source and output files of each build step and can therefore compact the build results into a single directory. Making it easy to being cached. Once restored, hammerkit can detect what's changed since last time and only build the changes.
+Hammerkit, on the other hand, knows the source and output files of each build step and can therefore compact the build results into a single directory, which is easy to cache — or share them through a remote cache. Once restored, hammerkit detects what has changed since the last run and only builds the changes.
 
 ## Where to go next?
 

package/build.schema.json

@@ -164,6 +164,12 @@
                   "shell": {
                     "type": "string"
                   },
+                  "continuous": {
+                    "type": "boolean"
+                  },
+                  "timeout": {
+                    "type": "string"
+                  },
                   "image": {
                     "type": "string"
                   },
@@ -216,6 +222,12 @@
                   "shell": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/shell"
                   },
+                  "continuous": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/continuous"
+                  },
+                  "timeout": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
+                  },
                   "platform": {
                     "type": "object",
                     "properties": {
@@ -323,8 +335,7 @@
                   }
                 },
                 "required": [
-                  "image",
-                  "ports"
+                  "image"
                 ],
                 "additionalProperties": false,
                 "description": "container service"
@@ -545,8 +556,44 @@
                       "bucket"
                     ],
                     "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "type": {
+                        "type": "string",
+                        "const": "registry"
+                      },
+                      "repository": {
+                        "type": "string"
+                      },
+                      "insecure": {
+                        "type": "boolean"
+                      }
+                    },
+                    "required": [
+                      "type",
+                      "repository"
+                    ],
+                    "additionalProperties": false
                   }
                 ]
+              },
+              "retention": {
+                "type": "object",
+                "properties": {
+                  "maxAge": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout"
+                  },
+                  "maxSize": {
+                    "type": "string"
+                  },
+                  "keepPerTask": {
+                    "type": "integer",
+                    "exclusiveMinimum": 0
+                  }
+                },
+                "additionalProperties": false
               }
             },
             "required": [

package/dist/cache/backends/local-cache-backend.js

@@ -3,8 +3,43 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createLocalCacheBackend = void 0;
 const path_1 = require("path");
 const os_1 = require("os");
+const promises_1 = require("fs/promises");
+// Hammerkit-owned last-use marker inside each entry directory, updated on every
+// pull. Filesystem atime is unreliable (relatime/noatime), so retention reads
+// this instead. Never copied out of the cache.
+const LAST_ACCESS_FILE = '.last-access';
+async function directories(path) {
+    try {
+        return (await (0, promises_1.readdir)(path, { withFileTypes: true })).filter((d) => d.isDirectory()).map((d) => d.name);
+    }
+    catch {
+        return [];
+    }
+}
 function createLocalCacheBackend(spec) {
     const root = spec.path ?? (0, path_1.join)((0, os_1.homedir)(), '.hammerkit', 'remote-cache');
+    async function readEntry(taskId, stateKey) {
+        const dir = (0, path_1.join)(root, taskId, stateKey);
+        let created;
+        try {
+            // stats.json is written last, so its presence marks a complete entry
+            created = (await (0, promises_1.stat)((0, path_1.join)(dir, 'stats.json'))).mtimeMs;
+        }
+        catch {
+            return null;
+        }
+        let size = 0;
+        let lastAccessedAt = null;
+        for (const file of await (0, promises_1.readdir)(dir)) {
+            if (file === LAST_ACCESS_FILE) {
+                const value = parseInt(await (0, promises_1.readFile)((0, path_1.join)(dir, file), 'utf8'), 10);
+                lastAccessedAt = Number.isNaN(value) ? null : value;
+                continue;
+            }
+            size += (await (0, promises_1.stat)((0, path_1.join)(dir, file))).size;
+        }
+        return { taskId, stateKey, size, createdAt: Math.round(created), lastAccessedAt };
+    }
     return {
         type: 'local',
         async has(taskId, stateKey, environment) {
@@ -25,8 +60,12 @@ function createLocalCacheBackend(spec) {
             await environment.file.createDirectory(into);
             try {
                 for (const file of await environment.file.listFiles(remoteDir)) {
+                    if (file === LAST_ACCESS_FILE) {
+                        continue;
+                    }
                     await environment.file.copy((0, path_1.join)(remoteDir, file), (0, path_1.join)(into, file));
                 }
+                await (0, promises_1.writeFile)((0, path_1.join)(remoteDir, LAST_ACCESS_FILE), `${Date.now()}`);
                 return true;
             }
             catch (e) {
@@ -37,7 +76,7 @@ function createLocalCacheBackend(spec) {
             const remoteDir = (0, path_1.join)(root, taskId, stateKey);
             await environment.file.createDirectory(remoteDir);
             // Write data files first; stats.json last so a partial push is invisible to has()/pull().
-            const files = await environment.file.listFiles(from);
+            const files = (await environment.file.listFiles(from)).filter((f) => f !== LAST_ACCESS_FILE);
             const ordered = [
                 ...files.filter((f) => f !== 'stats.json' && f !== 'description.json'),
                 ...files.filter((f) => f === 'description.json'),
@@ -50,6 +89,24 @@ function createLocalCacheBackend(spec) {
         async clear(taskId, environment) {
             await environment.file.remove((0, path_1.join)(root, taskId));
         },
+        async list() {
+            const entries = [];
+            for (const taskId of await directories(root)) {
+                for (const stateKey of await directories((0, path_1.join)(root, taskId))) {
+                    const entry = await readEntry(taskId, stateKey);
+                    if (entry) {
+                        entries.push(entry);
+                    }
+                }
+            }
+            return entries;
+        },
+        async remove(taskId, stateKey, environment) {
+            await environment.file.remove((0, path_1.join)(root, taskId, stateKey));
+            if ((await directories((0, path_1.join)(root, taskId))).length === 0) {
+                await environment.file.remove((0, path_1.join)(root, taskId));
+            }
+        },
     };
 }
 exports.createLocalCacheBackend = createLocalCacheBackend;

package/dist/cache/backends/registry/registry-client.js

@@ -0,0 +1,264 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.digestBuffer = exports.digestFile = exports.RegistryClient = exports.parseChallenge = exports.RegistryError = exports.OCI_LAYER_TAR = exports.OCI_CONFIG = exports.OCI_MANIFEST = void 0;
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+const stream_1 = require("stream");
+const promises_2 = require("stream/promises");
+const registry_credentials_1 = require("./registry-credentials");
+exports.OCI_MANIFEST = 'application/vnd.oci.image.manifest.v1+json';
+exports.OCI_CONFIG = 'application/vnd.oci.image.config.v1+json';
+exports.OCI_LAYER_TAR = 'application/vnd.oci.image.layer.v1.tar';
+const MANIFEST_ACCEPT = [exports.OCI_MANIFEST, 'application/vnd.docker.distribution.manifest.v2+json'].join(', ');
+class RegistryError extends Error {
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+    }
+}
+exports.RegistryError = RegistryError;
+// Parse a `WWW-Authenticate` header, e.g.
+// `Bearer realm="https://ghcr.io/token",service="ghcr.io",scope="repository:o/r:pull"`
+function parseChallenge(header) {
+    const space = header.indexOf(' ');
+    const scheme = (space < 0 ? header : header.substring(0, space)).toLowerCase();
+    const params = {};
+    const regex = /([a-zA-Z_]+)="([^"]*)"/g;
+    let match;
+    while ((match = regex.exec(header)) !== null) {
+        params[match[1]] = match[2];
+    }
+    return { scheme, params };
+}
+exports.parseChallenge = parseChallenge;
+function basicHeader(credentials) {
+    return `Basic ${Buffer.from(`${credentials.username}:${credentials.password}`).toString('base64')}`;
+}
+// A minimal OCI distribution-spec client: just the manifest/blob operations the
+// cache backend needs, with the Basic and Bearer-token auth handshakes every
+// mainstream registry (Docker Hub, GHCR, ECR, GAR, Artifactory, registry:2)
+// uses. Talks HTTP directly so no docker daemon or extra binary is involved.
+class RegistryClient {
+    constructor(reference, environment) {
+        this.reference = reference;
+        this.environment = environment;
+        this.authorization = null;
+        this.credentials = undefined;
+    }
+    url(path) {
+        const scheme = this.reference.insecure ? 'http' : 'https';
+        return `${scheme}://${this.reference.host}/v2/${this.reference.repository}/${path}`;
+    }
+    async getCredentials() {
+        if (this.credentials === undefined) {
+            this.credentials = await (0, registry_credentials_1.resolveRegistryCredentials)(this.reference.credentialHost, this.environment);
+        }
+        return this.credentials;
+    }
+    async authenticate(challengeHeader) {
+        if (!challengeHeader) {
+            return false;
+        }
+        const challenge = parseChallenge(challengeHeader);
+        const credentials = await this.getCredentials();
+        if (challenge.scheme === 'basic') {
+            if (!credentials) {
+                return false;
+            }
+            this.authorization = basicHeader(credentials);
+            return true;
+        }
+        if (challenge.scheme !== 'bearer' || !challenge.params.realm) {
+            return false;
+        }
+        const tokenUrl = new URL(challenge.params.realm);
+        if (challenge.params.service) {
+            tokenUrl.searchParams.set('service', challenge.params.service);
+        }
+        // always ask for push too: one token then serves the whole backend; a
+        // registry grants the subset the credentials allow
+        tokenUrl.searchParams.set('scope', `repository:${this.reference.repository}:pull,push`);
+        const response = await fetch(tokenUrl, {
+            headers: credentials ? { authorization: basicHeader(credentials) } : {},
+        });
+        if (!response.ok) {
+            throw new RegistryError(`registry token request failed with ${response.status}`, response.status);
+        }
+        const body = (await response.json());
+        const token = body.token ?? body.access_token;
+        if (!token) {
+            throw new RegistryError('registry token response contained no token');
+        }
+        this.authorization = `Bearer ${token}`;
+        return true;
+    }
+    // Send a request, performing the auth handshake once on a 401. `body` is a
+    // factory because a streamed body cannot be replayed after the challenge.
+    async request(method, urlOrPath, init = {}) {
+        const url = urlOrPath.startsWith('http') ? urlOrPath : this.url(urlOrPath);
+        const send = () => {
+            const body = init.body?.();
+            return fetch(url, {
+                method,
+                headers: { ...(init.headers ?? {}), ...(this.authorization ? { authorization: this.authorization } : {}) },
+                body: body instanceof stream_1.Readable ? stream_1.Readable.toWeb(body) : body,
+                redirect: init.redirect ?? 'follow',
+                ...(body instanceof stream_1.Readable ? { duplex: 'half' } : {}),
+            });
+        };
+        let response = await send();
+        if (response.status === 401 && (await this.authenticate(response.headers.get('www-authenticate')))) {
+            await response.body?.cancel();
+            response = await send();
+        }
+        return response;
+    }
+    async fail(response, what) {
+        const text = await response.text().catch(() => '');
+        const hint = response.status === 401 || response.status === 403 ? ' (check `docker login` for this registry)' : '';
+        throw new RegistryError(`${what} failed with ${response.status} ${text.substring(0, 200)}${hint}`.trim(), response.status);
+    }
+    async manifestExists(tag) {
+        const response = await this.request('HEAD', `manifests/${tag}`, { headers: { accept: MANIFEST_ACCEPT } });
+        if (response.status === 404) {
+            return false;
+        }
+        if (!response.ok) {
+            return this.fail(response, `checking manifest ${tag}`);
+        }
+        return true;
+    }
+    async getManifest(tag) {
+        const response = await this.request('GET', `manifests/${tag}`, { headers: { accept: MANIFEST_ACCEPT } });
+        if (response.status === 404) {
+            return null;
+        }
+        if (!response.ok) {
+            return this.fail(response, `reading manifest ${tag}`);
+        }
+        return (await response.json());
+    }
+    async getBlobJson(digest) {
+        const response = await this.request('GET', `blobs/${digest}`);
+        if (!response.ok) {
+            return this.fail(response, `reading blob ${digest}`);
+        }
+        return (await response.json());
+    }
+    async manifestDigest(tag) {
+        const response = await this.request('HEAD', `manifests/${tag}`, { headers: { accept: MANIFEST_ACCEPT } });
+        if (response.status === 404) {
+            return null;
+        }
+        if (!response.ok) {
+            return this.fail(response, `checking manifest ${tag}`);
+        }
+        return response.headers.get('docker-content-digest');
+    }
+    async putManifest(tag, manifest) {
+        const body = JSON.stringify(manifest);
+        const response = await this.request('PUT', `manifests/${tag}`, {
+            headers: { 'content-type': exports.OCI_MANIFEST },
+            body: () => body,
+        });
+        if (!response.ok) {
+            return this.fail(response, `writing manifest ${tag}`);
+        }
+    }
+    async deleteManifest(digest) {
+        const response = await this.request('DELETE', `manifests/${digest}`);
+        if (response.status === 404) {
+            return;
+        }
+        if (response.status === 405 || response.status === 403 || response.status === 400) {
+            throw new RegistryError(`the registry does not allow deleting cache entries (${response.status}); use its own retention policy instead`, response.status);
+        }
+        if (!response.ok) {
+            return this.fail(response, `deleting manifest ${digest}`);
+        }
+    }
+    async listTags() {
+        const tags = [];
+        let next = 'tags/list?n=1000';
+        while (next) {
+            const response = await this.request('GET', next);
+            if (response.status === 404) {
+                return tags;
+            }
+            if (!response.ok) {
+                return this.fail(response, 'listing tags');
+            }
+            const body = (await response.json());
+            tags.push(...(body.tags ?? []));
+            const link = response.headers.get('link');
+            const match = link ? /<([^>]+)>;\s*rel="next"/.exec(link) : null;
+            next = match ? new URL(match[1], this.url('')).toString() : null;
+        }
+        return tags;
+    }
+    async blobExists(digest) {
+        const response = await this.request('HEAD', `blobs/${digest}`);
+        if (response.status === 404) {
+            return false;
+        }
+        if (!response.ok) {
+            return this.fail(response, `checking blob ${digest}`);
+        }
+        return true;
+    }
+    // Upload a blob from a buffer or a file (monolithic upload: POST to open the
+    // session, PUT the content with its digest to commit it).
+    async uploadBlob(digest, size, content) {
+        if (await this.blobExists(digest)) {
+            return;
+        }
+        const started = await this.request('POST', 'blobs/uploads/', { redirect: 'manual' });
+        if (started.status !== 202) {
+            return this.fail(started, 'starting blob upload');
+        }
+        const location = started.headers.get('location');
+        if (!location) {
+            throw new RegistryError('registry did not return an upload location');
+        }
+        const uploadUrl = new URL(location, this.url(''));
+        uploadUrl.searchParams.set('digest', digest);
+        const response = await this.request('PUT', uploadUrl.toString(), {
+            headers: { 'content-type': 'application/octet-stream', 'content-length': `${size}` },
+            body: () => ('buffer' in content ? content.buffer : (0, fs_1.createReadStream)(content.file)),
+        });
+        if (response.status !== 201) {
+            return this.fail(response, `uploading blob ${digest}`);
+        }
+    }
+    // Download a blob to a file, verifying its digest while streaming.
+    async downloadBlob(descriptor, file) {
+        const response = await this.request('GET', `blobs/${descriptor.digest}`);
+        if (!response.ok || !response.body) {
+            return this.fail(response, `downloading blob ${descriptor.digest}`);
+        }
+        const hash = (0, crypto_1.createHash)('sha256');
+        await (0, promises_2.pipeline)(stream_1.Readable.fromWeb(response.body), new stream_1.Transform({
+            transform(chunk, _encoding, callback) {
+                hash.update(chunk);
+                callback(null, chunk);
+            },
+        }), (0, fs_1.createWriteStream)(file));
+        const actual = `sha256:${hash.digest('hex')}`;
+        if (actual !== descriptor.digest) {
+            throw new RegistryError(`blob digest mismatch: expected ${descriptor.digest}, got ${actual}`);
+        }
+    }
+}
+exports.RegistryClient = RegistryClient;
+async function digestFile(file) {
+    const hash = (0, crypto_1.createHash)('sha256');
+    await (0, promises_2.pipeline)((0, fs_1.createReadStream)(file), hash);
+    return { digest: `sha256:${hash.digest('hex')}`, size: (await (0, promises_1.stat)(file)).size };
+}
+exports.digestFile = digestFile;
+function digestBuffer(buffer) {
+    return { digest: `sha256:${(0, crypto_1.createHash)('sha256').update(buffer).digest('hex')}`, size: buffer.length };
+}
+exports.digestBuffer = digestBuffer;
+//# sourceMappingURL=registry-client.js.map

package/dist/cache/backends/registry/registry-credentials.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveRegistryCredentials = void 0;
+const child_process_1 = require("child_process");
+const os_1 = require("os");
+const path_1 = require("path");
+async function readDockerConfig(environment) {
+    const dir = environment.processEnvs.DOCKER_CONFIG ?? (0, path_1.join)((0, os_1.homedir)(), '.docker');
+    const path = (0, path_1.join)(dir, 'config.json');
+    if (!(await environment.file.exists(path))) {
+        return null;
+    }
+    try {
+        return JSON.parse(await environment.file.read(path));
+    }
+    catch {
+        return null;
+    }
+}
+// `docker-credential-<helper> get` reads the server on stdin and prints
+// {"Username": ..., "Secret": ...}; a missing entry exits non-zero.
+function runCredentialHelper(helper, host) {
+    return new Promise((resolve) => {
+        let stdout = '';
+        let child;
+        try {
+            child = (0, child_process_1.spawn)(`docker-credential-${helper}`, ['get'], { stdio: ['pipe', 'pipe', 'ignore'] });
+        }
+        catch {
+            resolve(null);
+            return;
+        }
+        child.on('error', () => resolve(null));
+        child.stdout.on('data', (chunk) => (stdout += chunk.toString()));
+        child.on('close', (code) => {
+            if (code !== 0) {
+                resolve(null);
+                return;
+            }
+            try {
+                const parsed = JSON.parse(stdout);
+                resolve(parsed.Secret ? { username: parsed.Username ?? '', password: parsed.Secret } : null);
+            }
+            catch {
+                resolve(null);
+            }
+        });
+        child.stdin.end(host);
+    });
+}
+// Resolve credentials for a registry host from the docker config that
+// `docker login` (and CI login actions such as docker/login-action) write, so
+// the cache introduces no credential surface of its own. Order matches docker: credHelpers, then inline auths, then the
+// global credsStore.
+async function resolveRegistryCredentials(credentialHost, environment) {
+    const config = await readDockerConfig(environment);
+    if (!config) {
+        return null;
+    }
+    const helper = config.credHelpers?.[credentialHost];
+    if (helper) {
+        return runCredentialHelper(helper, credentialHost);
+    }
+    const inline = config.auths?.[credentialHost] ?? config.auths?.[`https://${credentialHost}`];
+    if (inline?.auth) {
+        const decoded = Buffer.from(inline.auth, 'base64').toString('utf8');
+        const separator = decoded.indexOf(':');
+        if (separator > 0) {
+            return { username: decoded.substring(0, separator), password: decoded.substring(separator + 1) };
+        }
+    }
+    if (inline?.username && inline.password) {
+        return { username: inline.username, password: inline.password };
+    }
+    if (config.credsStore) {
+        return runCredentialHelper(config.credsStore, credentialHost);
+    }
+    return null;
+}
+exports.resolveRegistryCredentials = resolveRegistryCredentials;
+//# sourceMappingURL=registry-credentials.js.map

package/dist/cache/backends/registry/registry-reference.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseRegistryReference = void 0;
+const DOCKER_HUB_HOST = 'registry-1.docker.io';
+const DOCKER_HUB_CREDENTIAL_HOST = 'https://index.docker.io/v1/';
+function isLocalHost(host) {
+    const name = host.replace(/:\d+$/, '');
+    return name === 'localhost' || name === '127.0.0.1' || name === '[::1]';
+}
+// Parse a repository reference the way docker does: the first path component is
+// a registry host when it contains a `.` or `:` or is `localhost`; otherwise the
+// repository lives on Docker Hub (with the implicit `library/` namespace).
+function parseRegistryReference(value, insecure) {
+    const trimmed = value.replace(/^[a-z]+:\/\//, '').replace(/\/+$/, '');
+    if (trimmed.includes('@') || /:[^/]*$/.test(trimmed.split('/').slice(1).join('/'))) {
+        throw new Error(`registry cache repository "${value}" must not contain a tag or digest`);
+    }
+    const [first, ...rest] = trimmed.split('/');
+    const hasHost = rest.length > 0 && (first.includes('.') || first.includes(':') || first === 'localhost');
+    if (!hasHost) {
+        const repository = rest.length === 0 ? `library/${first}` : trimmed;
+        return { host: DOCKER_HUB_HOST, repository, credentialHost: DOCKER_HUB_CREDENTIAL_HOST, insecure: false };
+    }
+    const host = first === 'docker.io' || first === 'index.docker.io' ? DOCKER_HUB_HOST : first;
+    if (rest.length === 0) {
+        throw new Error(`registry cache repository "${value}" has no repository path`);
+    }
+    return {
+        host,
+        repository: rest.join('/'),
+        credentialHost: host === DOCKER_HUB_HOST ? DOCKER_HUB_CREDENTIAL_HOST : host,
+        insecure: insecure ?? isLocalHost(host),
+    };
+}
+exports.parseRegistryReference = parseRegistryReference;
+//# sourceMappingURL=registry-reference.js.map