hammerkit 1.23.0 → 1.25.0

package/dist/server/definitions.js

@@ -0,0 +1,160 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DefinitionStore = exports.inspectBundle = exports.DefinitionError = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const crypto_1 = require("crypto");
+const bundle_1 = require("../remote/bundle");
+const plan_1 = require("./plan");
+var plan_2 = require("./plan");
+Object.defineProperty(exports, "DefinitionError", { enumerable: true, get: function () { return plan_2.DefinitionError; } });
+Object.defineProperty(exports, "inspectBundle", { enumerable: true, get: function () { return plan_2.inspectBundle; } });
+// The published build files: immutable versions, each named by its hash. They
+// are kept in files, so a restart doesn't forget what runs were started from.
+class DefinitionStore {
+    constructor(directory, 
+    // the environment a build file is read with: on a server, the hosted one
+    environment, options = {}) {
+        this.directory = directory;
+        this.environment = environment;
+        this.options = options;
+        this.versions = new Map();
+        this.now = options.now ?? Date.now;
+    }
+    async load() {
+        let names = [];
+        try {
+            names = await (0, promises_1.readdir)(this.directory);
+        }
+        catch {
+            return; // nothing was published yet
+        }
+        for (const name of names) {
+            for (const file of await (0, promises_1.readdir)((0, path_1.join)(this.directory, name))) {
+                if (!file.endsWith('.json')) {
+                    continue;
+                }
+                const stored = JSON.parse(await (0, promises_1.readFile)((0, path_1.join)(this.directory, name, file), 'utf8'));
+                this.add(stored.version);
+            }
+        }
+    }
+    names() {
+        return [...this.versions.keys()].sort();
+    }
+    // newest first
+    list(name) {
+        return this.has(name) ? [...this.versions.get(name)] : [];
+    }
+    // a hash, or `latest` for the version published last that is not yanked
+    find(name, hash) {
+        const versions = this.list(name);
+        return hash === 'latest' ? versions.find((v) => !v.yanked) : versions.find((v) => v.hash === hash);
+    }
+    /**
+     * Pulls a version back (or, with `undo`, lets it go on). The version is rewritten in
+     * place, its files stay: a run that was started from it has them.
+     */
+    async yank(name, hash, by, options = {}) {
+        const version = this.versions.get(name)?.find((v) => v.hash === hash);
+        if (!version || Boolean(version.yanked) === !options.undo) {
+            return version;
+        }
+        const bundle = await this.files(version);
+        if (!bundle) {
+            return version;
+        }
+        const next = { ...version };
+        if (options.undo) {
+            delete next.yanked;
+        }
+        else {
+            next.yanked = {
+                at: new Date(this.now()).toISOString(),
+                by: { iss: by.iss, sub: by.sub },
+                ...(options.reason ? { reason: options.reason } : {}),
+            };
+        }
+        const file = this.file(name, hash);
+        const temporary = `${file}.${(0, crypto_1.randomBytes)(8).toString('hex')}.tmp`;
+        await (0, promises_1.writeFile)(temporary, JSON.stringify({ version: next, bundle }), { flag: 'wx' });
+        await (0, promises_1.rename)(temporary, file);
+        // the same object, so whoever holds the version sees it
+        Object.keys(version).forEach((key) => delete version[key]);
+        Object.assign(version, next);
+        return version;
+    }
+    async files(version) {
+        try {
+            return JSON.parse(await (0, promises_1.readFile)(this.file(version.name, version.hash), 'utf8')).bundle;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    // Reads a bundle the way a run would and says what it would be as a version;
+    // `existing` is the published version with the same hash, if there is one.
+    async prepare(name, bundle, publisher, 
+    // a proposal is read before anybody trusts it: it may not make the server fetch
+    options = {}) {
+        const hash = (0, bundle_1.hashBundle)(bundle);
+        const existing = this.find(name, hash);
+        if (existing) {
+            return { version: existing, existing };
+        }
+        const { tasks, services, inputs, plan, accounts } = await (0, plan_1.planBundle)(bundle, this.environment, {
+            policy: this.options.policy,
+            noRemote: options.noRemote,
+        });
+        return {
+            version: {
+                name,
+                hash,
+                entry: bundle.entry,
+                publishedAt: new Date(this.now()).toISOString(),
+                publishedBy: { iss: publisher.iss, sub: publisher.sub },
+                tasks,
+                services,
+                inputs,
+                plan,
+                accounts,
+            },
+        };
+    }
+    // Publishing what already is there changes nothing and says so.
+    async publish(name, bundle, publisher, approvedBy) {
+        const prepared = await this.prepare(name, bundle, publisher);
+        if (prepared.existing) {
+            return { version: prepared.existing, created: false };
+        }
+        const version = {
+            ...prepared.version,
+            publishedAt: new Date(this.now()).toISOString(),
+            ...(approvedBy ? { approvedBy: { iss: approvedBy.iss, sub: approvedBy.sub } } : {}),
+        };
+        const file = this.file(name, version.hash);
+        await (0, promises_1.mkdir)((0, path_1.dirname)(file), { recursive: true });
+        const temporary = `${file}.${(0, crypto_1.randomBytes)(8).toString('hex')}.tmp`;
+        await (0, promises_1.writeFile)(temporary, JSON.stringify({ version, bundle }), { flag: 'wx' });
+        await (0, promises_1.rename)(temporary, file);
+        this.add(version);
+        return { version, created: true };
+    }
+    has(name) {
+        return this.versions.has(name);
+    }
+    add(version) {
+        const versions = this.versions.get(version.name) ?? [];
+        versions.unshift(version);
+        versions.sort((a, b) => b.publishedAt.localeCompare(a.publishedAt));
+        this.versions.set(version.name, versions);
+    }
+    file(name, hash) {
+        if (!bundle_1.HASH_PATTERN.test(hash)) {
+            throw new plan_1.DefinitionError('invalid hash');
+        }
+        return (0, path_1.join)(this.directory, name, `${hash.slice('sha256:'.length)}.json`);
+    }
+}
+exports.DefinitionStore = DefinitionStore;
+//# sourceMappingURL=definitions.js.map

package/dist/server/discovery.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DISCOVERY_PATH = exports.SERVER_PROTOCOL = void 0;
+// What a client needs to know about a server before it signs in. Kept apart from
+// the server so the client does not load the http server and token code.
+// Bumped when a client could no longer talk to an older or newer server.
+exports.SERVER_PROTOCOL = 1;
+exports.DISCOVERY_PATH = '/.well-known/hammerkit';
+//# sourceMappingURL=discovery.js.map

package/dist/server/executor.js

@@ -0,0 +1,237 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HostExecutor = exports.defaultWorker = void 0;
+const child_process_1 = require("child_process");
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const readline_1 = require("readline");
+const resolve_git_source_1 = require("../includes/resolve-git-source");
+const git_checkout_1 = require("./git-checkout");
+const run_plan_1 = require("./run-plan");
+// `.git` as a file system may read it: any case, and a name Windows reads without its
+// trailing dots and spaces, or by its short name
+function isGitDirectory(segment) {
+    const name = segment.toLowerCase().replace(/[. ]+$/, '');
+    return name === '.git' || name === 'git~1';
+}
+// A cancelled run gets this long to stop its tasks and remove its containers
+const STOP_GRACE_MS = 15000;
+// what is kept of the error output of a process that died without saying why
+const MAX_STDERR = 2 * 1024;
+/**
+ * The command that starts the process of a run: `hammerkit server-run`, from the
+ * build this module is part of. Only found next to a built `index.js`.
+ */
+function defaultWorker() {
+    const entry = (0, path_1.resolve)(__dirname, '..', 'index.js');
+    return (0, fs_1.existsSync)(entry) ? [process.execPath, entry, 'server-run'] : undefined;
+}
+exports.defaultWorker = defaultWorker;
+// The cache a run shares with others is the one of its source: the runs of a repository
+// share one, those of an upload only the uploader's. Somebody who can only upload can
+// not leave an entry for a run from a reviewed repository to find.
+function cacheNamespace(run) {
+    const key = run.source.type === 'ref' ? `ref\0${run.source.repo}` : `upload\0${run.owner.iss}\0${run.owner.sub}`;
+    return `${run.source.type}-${(0, crypto_1.createHash)('sha256').update(key).digest('hex').slice(0, 24)}`;
+}
+/**
+ * Runs the tasks of a run on this machine with the engine of `hammerkit run`, in a
+ * process of its own.
+ *
+ * The working directory is made for the run and removed afterwards. Its build
+ * file is the published version the run is pinned to, written over the code of the
+ * source, so the commands are the published ones whatever the source says. The process
+ * starts with an environment the server builds: the variables `runs.env` lists, a home
+ * directory that is the run's alone, and a cache directory per source trust. So what
+ * the engine reads of its process (the docker host, the registry logins in the home
+ * directory, the credentials of the operator) is what the operator gave runs and
+ * nothing else. The cache is shared by the runs of the same source, a task is keyed by
+ * its definition and sources and not by where they are.
+ */
+class HostExecutor {
+    constructor(options) {
+        this.options = options;
+    }
+    async execute(run, context) {
+        const { definitions, planning, repos } = this.options;
+        const version = definitions.find(run.definition.name, run.definition.hash);
+        const bundle = version ? await definitions.files(version) : undefined;
+        if (!version || !bundle) {
+            throw new Error(`the build file ${run.definition.name} at ${run.definition.hash} is no longer on this server`);
+        }
+        // pulled back after the run was accepted, while it waited for an approval or in the queue
+        if (version.yanked) {
+            throw new Error(`the build file ${run.definition.name} at ${run.definition.hash} was yanked${version.yanked.reason ? `: ${version.yanked.reason}` : ''}`);
+        }
+        if (run.source.type !== 'ref') {
+            throw new Error('runs from an upload are not supported by this server yet, use a git ref');
+        }
+        const { repo, ref, commit: pinned } = run.source;
+        if (!repos.readable(repo)) {
+            throw new Error(`this server can not read ${repo}, the operator lists it in sources`);
+        }
+        const worker = this.options.worker ?? defaultWorker();
+        if (!worker) {
+            throw new Error('this server has no command to start a run with, it needs to run from a built hammerkit');
+        }
+        // what was approved is what runs: the accounts are read again from the files with
+        // the environment of the run, and the run may use no more than it was admitted for
+        const planned = await (0, run_plan_1.planRun)(bundle, run, planning.environment, planning.policy);
+        const extra = planned.accounts.filter((account) => !run.accounts.includes(account));
+        if (extra.length > 0) {
+            throw new Error(`the run would use ${extra.join(', ')}, which it was not admitted for`);
+        }
+        const abortCtrl = new AbortController();
+        const abort = () => abortCtrl.abort();
+        if (context.signal.aborted) {
+            abort();
+        }
+        context.signal.addEventListener('abort', abort, { once: true });
+        const root = (0, path_1.join)(this.options.directory, run.id);
+        const checkout = (0, path_1.join)(root, 'source');
+        try {
+            const credentials = await repos.credentials(repo);
+            let commit;
+            try {
+                commit = await (0, git_checkout_1.checkoutRef)(repos.url(repo), ref, checkout, credentials.env, abortCtrl.signal, pinned);
+            }
+            catch (e) {
+                throw repos.scrub(e, credentials.values);
+            }
+            context.emit({ type: 'log', task: 'hammerkit', stream: 'stdout', text: `checked out ${repo} at ${commit}` });
+            await this.writeBundle(checkout, bundle.files);
+            const home = (0, path_1.join)(root, 'home');
+            await (0, promises_1.mkdir)(home, { recursive: true });
+            const cache = (0, path_1.join)(this.options.cache, cacheNamespace(run));
+            await (0, promises_1.mkdir)(cache, { recursive: true });
+            const input = {
+                entry: (0, path_1.join)(checkout, version.entry),
+                tasks: run.tasks,
+                options: run.options,
+                accounts: run.accounts,
+                catalog: planning.environment.hosted?.catalog,
+            };
+            await this.runInProcess(worker, input, checkout, this.processEnvironment(home, cache), context, abortCtrl.signal);
+        }
+        finally {
+            context.signal.removeEventListener('abort', abort);
+            await (0, promises_1.rm)(root, { recursive: true, force: true });
+        }
+    }
+    // What the process of a run sees: the variables the operator listed, and a home
+    // directory and cache that are the run's. The server's own are not in it.
+    processEnvironment(home, cache) {
+        const env = {};
+        for (const [key, value] of Object.entries(this.options.planning.environment.processEnvs)) {
+            if (value !== undefined) {
+                env[key] = value;
+            }
+        }
+        env.HOME = home;
+        env.USERPROFILE = home;
+        env.APPDATA = (0, path_1.join)(home, 'AppData', 'Roaming');
+        env.LOCALAPPDATA = (0, path_1.join)(home, 'AppData', 'Local');
+        env.HAMMERKIT_DIR = cache;
+        return env;
+    }
+    // Starts the process, hands it the run on its standard input and relays what it reports. It
+    // is stopped with SIGTERM when the run is cancelled, so the engine removes its containers
+    // and stops its tasks, and killed if it does not stop in time.
+    runInProcess(command, input, cwd, env, context, signal) {
+        return new Promise((resolvePromise, reject) => {
+            const child = (0, child_process_1.spawn)(command[0], command.slice(1), {
+                cwd,
+                env,
+                stdio: ['pipe', 'pipe', 'pipe'],
+                windowsHide: true,
+            });
+            let result;
+            let stderr = '';
+            let killTimer;
+            const stop = () => {
+                child.kill('SIGTERM');
+                killTimer = setTimeout(() => child.kill('SIGKILL'), STOP_GRACE_MS);
+                killTimer.unref();
+            };
+            if (signal.aborted) {
+                stop();
+            }
+            else {
+                signal.addEventListener('abort', stop, { once: true });
+            }
+            (0, readline_1.createInterface)({ input: child.stdout }).on('line', (line) => {
+                try {
+                    const message = JSON.parse(line);
+                    if (message.kind === 'event') {
+                        context.emit(message.event);
+                    }
+                    else if (message.kind === 'result') {
+                        result = message;
+                    }
+                }
+                catch {
+                    // not ours: whatever else the process prints is not an event of the run
+                }
+            });
+            child.stderr.on('data', (chunk) => {
+                stderr = (stderr + chunk.toString('utf8')).slice(-MAX_STDERR);
+            });
+            child.stdin.on('error', () => undefined);
+            child.stdin.end(JSON.stringify(input));
+            child.on('error', (e) => {
+                signal.removeEventListener('abort', stop);
+                clearTimeout(killTimer);
+                reject(new Error(`the process of the run could not start: ${e.message}`));
+            });
+            child.on('close', (code, killed) => {
+                signal.removeEventListener('abort', stop);
+                clearTimeout(killTimer);
+                if (result?.ok) {
+                    resolvePromise();
+                }
+                else if (result) {
+                    reject(new Error(result.error ?? 'the run failed'));
+                }
+                else if (signal.aborted) {
+                    reject(new Error('the run was cancelled'));
+                }
+                else {
+                    const reason = stderr.trim().split('\n').pop() ?? '';
+                    reject(new Error(`the process of the run stopped (${killed ? `signal ${killed}` : `code ${code}`})${reason ? `: ${reason}` : ''}`));
+                }
+            });
+        });
+    }
+    // The published files go over the checkout; nothing may land outside of it or in
+    // its git directory. The checkout is somebody else's: it may hold a symbolic link where
+    // a directory or a file of the bundle goes, which would carry the write out of it, so
+    // none is followed. `.git` is compared the way a case-insensitive or Windows file system
+    // would read it.
+    async writeBundle(directory, files) {
+        for (const [path, content] of Object.entries(files)) {
+            const target = (0, path_1.resolve)(directory, path);
+            const refused = () => new Error(`the published file ${JSON.stringify(path)} can not be written into the source`);
+            if (!(0, resolve_git_source_1.isWithin)(directory, target) || path.split(/[\\/]/).some((part) => isGitDirectory(part))) {
+                throw refused();
+            }
+            const parts = (0, path_1.relative)(directory, target).split(path_1.sep);
+            let current = directory;
+            for (const [index, part] of parts.entries()) {
+                current = (0, path_1.join)(current, part);
+                const stats = await (0, promises_1.lstat)(current).catch(() => undefined);
+                if (!stats) {
+                    break;
+                }
+                if (stats.isSymbolicLink() || (index < parts.length - 1 && !stats.isDirectory())) {
+                    throw refused();
+                }
+            }
+            await (0, promises_1.mkdir)((0, path_1.dirname)(target), { recursive: true });
+            await (0, promises_1.writeFile)(target, content);
+        }
+    }
+}
+exports.HostExecutor = HostExecutor;
+//# sourceMappingURL=executor.js.map

package/dist/server/git-checkout.js

@@ -0,0 +1,115 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveRef = exports.RefError = exports.checkoutRef = exports.isValidRef = void 0;
+const child_process_1 = require("child_process");
+const promises_1 = require("fs/promises");
+const util_1 = require("util");
+const execFileAsync = (0, util_1.promisify)(child_process_1.execFile);
+// Anything but plain network and local transports is refused: `ext::` runs a command.
+const ALLOWED_PROTOCOLS = 'file:git:http:https:ssh';
+// A branch, a tag or a commit. It is passed after `--` as well, but a ref that
+// looks like an option, has a `..` or odd characters is not one of the three.
+const REF_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,199}$/;
+function isValidRef(ref) {
+    return (REF_PATTERN.test(ref) && !ref.includes('..') && !ref.includes('//') && !ref.endsWith('/') && !ref.endsWith('.lock'));
+}
+exports.isValidRef = isValidRef;
+/**
+ * Fetches one commit of a repository into `directory` and checks it out, and
+ * answers which commit that is. The repository is the operator's, the ref is the
+ * caller's. Credentials come from the git configuration of the server, there is no
+ * prompt a run could hang on.
+ */
+async function checkoutRef(repository, ref, directory, env, signal, 
+// the commit the ref was resolved to when the run was submitted: a ref that moved since
+// is not what was approved
+expectedCommit) {
+    if (!isValidRef(ref)) {
+        throw new Error(`${JSON.stringify(ref.slice(0, 80))} is not a branch, tag or commit`);
+    }
+    await (0, promises_1.mkdir)(directory, { recursive: true });
+    const run = (args) => git(args, env, signal, `${repository} at ${ref}`);
+    await run(['init', '--quiet', directory]);
+    await run(['-C', directory, 'fetch', '--quiet', '--depth', '1', '--', repository, ref]);
+    await run(['-C', directory, '-c', 'advice.detachedHead=false', 'checkout', '--quiet', 'FETCH_HEAD']);
+    const commit = (await run(['-C', directory, 'rev-parse', 'HEAD'])).trim();
+    if (expectedCommit && commit !== expectedCommit) {
+        throw new Error(`${ref} is at ${commit} now, the run was submitted for ${expectedCommit}`);
+    }
+    return commit;
+}
+exports.checkoutRef = checkoutRef;
+class RefError extends Error {
+}
+exports.RefError = RefError;
+const COMMIT = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
+/**
+ * The commit a branch or tag of a repository points to now: `refs/heads/<ref>`, else
+ * `refs/tags/<ref>` (the commit of an annotated tag). A name that is both a branch and
+ * a tag is ambiguous. A full commit hash is taken as it is.
+ */
+async function resolveRef(repository, ref, env, signal) {
+    if (!isValidRef(ref)) {
+        throw new RefError(`${JSON.stringify(ref.slice(0, 80))} is not a branch, tag or commit`);
+    }
+    if (COMMIT.test(ref.toLowerCase())) {
+        return ref.toLowerCase();
+    }
+    const output = await git([
+        'ls-remote',
+        '--heads',
+        '--tags',
+        '--',
+        repository,
+        `refs/heads/${ref}`,
+        `refs/tags/${ref}`,
+        `refs/tags/${ref}^{}`,
+    ], env, signal, `the refs of ${repository}`);
+    const found = new Map();
+    for (const line of output.split('\n')) {
+        const [commit, name] = line.split('\t');
+        if (commit && name) {
+            found.set(name, commit);
+        }
+    }
+    const branch = found.get(`refs/heads/${ref}`);
+    const tag = found.get(`refs/tags/${ref}^{}`) ?? found.get(`refs/tags/${ref}`);
+    if (branch && tag) {
+        throw new RefError(`${ref} is both a branch and a tag, name one of them`);
+    }
+    const commit = branch ?? tag;
+    if (!commit) {
+        throw new RefError(`there is no branch or tag ${ref}`);
+    }
+    return commit;
+}
+exports.resolveRef = resolveRef;
+async function git(args, baseEnv, signal, what) {
+    const env = {};
+    for (const [key, value] of Object.entries(baseEnv)) {
+        if (value !== undefined) {
+            env[key] = value;
+        }
+    }
+    env.GIT_TERMINAL_PROMPT = '0';
+    env.GIT_ALLOW_PROTOCOL = ALLOWED_PROTOCOLS;
+    try {
+        return (await execFileAsync('git', args, { env, signal })).stdout;
+    }
+    catch (e) {
+        const error = e;
+        if (error.code === 'ABORT_ERR') {
+            throw new Error(`fetching ${what} was cancelled`);
+        }
+        if (error.code === 'ENOENT') {
+            throw new Error(`unable to fetch ${what}: git is not installed`);
+        }
+        const reason = (error.stderr ?? '')
+            .split('\n')
+            .map((line) => line.trim())
+            .filter((line) => line.length > 0)
+            .pop() ?? error.message;
+        throw new Error(`unable to fetch ${what}: ${reason}`);
+    }
+}
+//# sourceMappingURL=git-checkout.js.map

package/dist/server/http.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readJson = exports.MAX_BODY_BYTES = exports.HttpError = exports.send = void 0;
+function send(res, status, body, headers = {}) {
+    res.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store', ...headers });
+    res.end(JSON.stringify(body));
+}
+exports.send = send;
+// An answer to the caller, thrown from wherever the request turns out to be wrong.
+class HttpError extends Error {
+    constructor(status, message) {
+        super(message);
+        this.status = status;
+    }
+}
+exports.HttpError = HttpError;
+exports.MAX_BODY_BYTES = 64 * 1024;
+async function readJson(req, limit = exports.MAX_BODY_BYTES) {
+    if (!/^application\/json(;|$)/i.test(req.headers['content-type'] ?? '')) {
+        throw new HttpError(415, 'send application/json');
+    }
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        size += chunk.length;
+        if (size > limit) {
+            throw new HttpError(413, `the request is larger than ${limit} bytes`);
+        }
+        chunks.push(chunk);
+    }
+    try {
+        return JSON.parse(Buffer.concat(chunks).toString('utf8'));
+    }
+    catch {
+        throw new HttpError(400, 'the body is not valid JSON');
+    }
+}
+exports.readJson = readJson;
+//# sourceMappingURL=http.js.map