hammerkit 1.23.0 → 1.25.0

package/dist/server/runs.js

@@ -0,0 +1,432 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.unavailableExecutor = exports.RunRegistry = exports.RUN_ID_PATTERN = exports.QuotaError = exports.RunError = exports.isFinal = void 0;
+const crypto_1 = require("crypto");
+const units_1 = require("../utils/units");
+const FINAL_STATES = ['succeeded', 'failed', 'cancelled', 'rejected'];
+function isFinal(state) {
+    return FINAL_STATES.includes(state);
+}
+exports.isFinal = isFinal;
+// The server can not take the run now; `status` is what the caller is told
+class RunError extends Error {
+    constructor() {
+        super(...arguments);
+        this.status = 503;
+    }
+}
+exports.RunError = RunError;
+// A caller is over one of their limits, others may still submit
+class QuotaError extends RunError {
+    constructor() {
+        super(...arguments);
+        this.status = 429;
+    }
+}
+exports.QuotaError = QuotaError;
+exports.RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]{16}$/;
+// Where runs are kept while the server is up. Everything is scoped to the
+// caller that submitted the run: the same id asked for by someone else is
+// "not found", so ids can't be used to find out which runs exist.
+class RunRegistry {
+    constructor(options) {
+        this.options = options;
+        this.entries = new Map();
+        this.queue = [];
+        this.running = 0;
+        this.closed = false;
+        this.maxConcurrent = options.maxConcurrent ?? 4;
+        this.keep = options.keep ?? 200;
+        this.maxEvents = options.maxEvents ?? 5000;
+        this.maxWaiting = options.maxWaiting ?? 100;
+        this.approvalTimeout = options.approvalTimeout ?? 60 * 60 * 1000;
+        this.now = options.now ?? Date.now;
+        this.maxLogBytes = options.maxLogBytes ?? Infinity;
+    }
+    /**
+     * Takes back the runs of a previous start. Those that waited for an approval wait
+     * again, for what is left of their time; those that were queued or running when the
+     * server stopped failed with it, and say so.
+     */
+    async restore() {
+        const store = this.options.store;
+        if (!store) {
+            return;
+        }
+        const { records, skipped } = await store.load();
+        skipped.forEach((name) => this.options.log?.(`skip run record ${name}: it can not be read`));
+        for (const { run, events } of records) {
+            const entry = {
+                run,
+                events,
+                listeners: new Set(),
+                abort: new AbortController(),
+                truncated: events.some((event) => event.type === 'truncated'),
+                bytes: 0,
+                saving: Promise.resolve(),
+            };
+            this.entries.set(run.id, entry);
+            if (run.state === 'awaiting-approval') {
+                const left = Date.parse(run.approvals?.[0]?.expiresAt ?? '') - this.now();
+                if (Number.isFinite(left) && left > 0) {
+                    entry.timer = setTimeout(() => this.expire(entry), left);
+                    entry.timer.unref();
+                }
+                else {
+                    this.finish(entry, 'rejected', 'nobody approved it in time');
+                }
+            }
+            else if (run.state === 'queued' || run.state === 'running') {
+                this.finish(entry, 'failed', `the server stopped while the run was ${run.state}, submit it again`);
+            }
+        }
+        this.evict();
+    }
+    // A run that needs approvals (for the accounts named) waits for them, the others
+    // are queued at once.
+    submit(owner, spec, approvalAccounts = []) {
+        if (this.closed) {
+            throw new RunError('the server is shutting down');
+        }
+        const waiting = approvalAccounts.length > 0;
+        if (waiting && this.waiting().length >= this.maxWaiting) {
+            throw new RunError('too many runs are waiting for an approval');
+        }
+        this.checkLimits(owner, waiting);
+        const run = {
+            ...spec,
+            id: `run_${(0, crypto_1.randomBytes)(12).toString('base64url')}`,
+            owner: { iss: owner.iss, sub: owner.sub },
+            state: waiting ? 'awaiting-approval' : 'queued',
+            createdAt: this.timestamp(),
+        };
+        const entry = {
+            run,
+            events: [],
+            listeners: new Set(),
+            abort: new AbortController(),
+            truncated: false,
+            bytes: 0,
+            saving: Promise.resolve(),
+        };
+        this.entries.set(run.id, entry);
+        if (waiting) {
+            // before the first event, which is written down with the run
+            const expiresAt = new Date(this.now() + this.approvalTimeout).toISOString();
+            run.approvals = approvalAccounts.map((account) => ({ account, expiresAt }));
+        }
+        this.record(entry, { type: 'state', state: run.state });
+        if (waiting) {
+            entry.timer = setTimeout(() => this.expire(entry), this.approvalTimeout);
+            entry.timer.unref();
+        }
+        else {
+            this.queue.push(entry);
+        }
+        this.evict();
+        // what the caller is told is the run as accepted, even when it starts at once
+        const accepted = this.copy(run);
+        this.dispatch();
+        return accepted;
+    }
+    // `canUse` also lets who uses an account the run needed approved read it: they
+    // decided about it, so they may follow it and stop it.
+    get(owner, id, canUse) {
+        const entry = this.find(owner, id, canUse);
+        return entry ? this.copy(entry.run) : undefined;
+    }
+    // newest first
+    list(owner) {
+        return [...this.entries.values()]
+            .filter((entry) => this.owns(owner, entry))
+            .map((entry) => this.copy(entry.run))
+            .reverse();
+    }
+    // A queued run is cancelled at once; a running one when its executor stops.
+    cancel(owner, id, canUse) {
+        const entry = this.find(owner, id, canUse);
+        if (!entry) {
+            return undefined;
+        }
+        if (entry.run.state === 'queued') {
+            this.queue.splice(this.queue.indexOf(entry), 1);
+            this.finish(entry, 'cancelled');
+        }
+        else if (entry.run.state === 'awaiting-approval') {
+            this.finish(entry, 'cancelled');
+        }
+        else if (entry.run.state === 'running') {
+            entry.abort.abort();
+        }
+        return this.copy(entry.run);
+    }
+    // The runs that wait for an approval, of anybody, oldest first. Who may see
+    // them is for the caller to decide.
+    waiting() {
+        return [...this.entries.values()]
+            .filter((entry) => entry.run.state === 'awaiting-approval')
+            .map((entry) => this.copy(entry.run));
+    }
+    // One run that waits (or waited), if `canDecide` says the decider uses one of the
+    // accounts it still needs approved.
+    getForApproval(id, canDecide) {
+        const entry = this.entries.get(id);
+        if (!entry || !this.mayDecide(entry, canDecide)) {
+            return undefined;
+        }
+        return this.copy(entry.run);
+    }
+    // An approval covers the accounts the approver uses; the run is queued when all
+    // of its accounts are covered. A rejection by anybody who covers one of them ends
+    // it. Nobody decides on their own run.
+    decide(decider, id, outcome, canDecide, reason) {
+        const entry = this.entries.get(id);
+        if (!entry || !this.mayDecide(entry, canDecide)) {
+            return { status: 'not-found' };
+        }
+        if (this.owns(decider, entry)) {
+            return { status: 'own-run' };
+        }
+        if (entry.run.state !== 'awaiting-approval' || !entry.run.approvals) {
+            return { status: 'not-waiting', state: entry.run.state };
+        }
+        const by = { iss: decider.iss, sub: decider.sub };
+        const covered = entry.run.approvals.filter((approval) => !approval.approvedBy && canDecide(approval.account));
+        if (outcome === 'approved' && covered.length === 0) {
+            return { status: 'already-approved' };
+        }
+        const accounts = entry.run.approvals
+            .filter((approval) => canDecide(approval.account))
+            .map((approval) => approval.account);
+        this.record(entry, { type: 'approval', outcome, by, accounts, ...(reason ? { reason } : {}) });
+        if (outcome === 'rejected') {
+            this.finish(entry, 'rejected', `rejected by ${by.sub}${reason ? `: ${reason}` : ''}`);
+            return { status: 'ok', run: this.copy(entry.run) };
+        }
+        const approvedAt = this.timestamp();
+        for (const approval of covered) {
+            approval.approvedBy = by;
+            approval.approvedAt = approvedAt;
+        }
+        if (entry.run.approvals.some((approval) => !approval.approvedBy)) {
+            // some accounts still wait for somebody who uses them
+            return { status: 'ok', run: this.copy(entry.run) };
+        }
+        this.clearTimer(entry);
+        entry.run.state = 'queued';
+        this.record(entry, { type: 'state', state: 'queued' });
+        this.queue.push(entry);
+        const approved = this.copy(entry.run);
+        this.dispatch();
+        return { status: 'ok', run: approved };
+    }
+    // The events after `after` (0 for all), and a way to follow new ones. The
+    // listener is not called for a run that is already over.
+    watch(owner, id, after, listener, canUse) {
+        const entry = this.find(owner, id, canUse);
+        if (!entry) {
+            return undefined;
+        }
+        const events = entry.events.filter((event) => event.seq > after);
+        const finished = FINAL_STATES.includes(entry.run.state);
+        if (!finished) {
+            entry.listeners.add(listener);
+        }
+        return { events, finished, stop: () => entry.listeners.delete(listener) };
+    }
+    // Stops accepting runs and aborts the ones in flight.
+    async close() {
+        this.closed = true;
+        for (const entry of this.queue.splice(0)) {
+            this.finish(entry, 'cancelled');
+        }
+        for (const entry of this.entries.values()) {
+            if (entry.run.state === 'awaiting-approval') {
+                if (this.options.store) {
+                    // the record keeps it waiting, the next start takes it back
+                    this.clearTimer(entry);
+                }
+                else {
+                    this.finish(entry, 'cancelled');
+                }
+            }
+        }
+        for (const entry of this.entries.values()) {
+            if (entry.run.state === 'running') {
+                entry.abort.abort();
+            }
+        }
+        while (this.running > 0) {
+            await new Promise((resolve) => setTimeout(resolve, 10));
+        }
+        // the runs that were stopped are written down before the server is gone
+        await Promise.all([...this.entries.values()].map((entry) => entry.saving));
+    }
+    find(owner, id, canUse) {
+        const entry = this.entries.get(id);
+        return entry && (this.owns(owner, entry) || (canUse && this.mayDecide(entry, canUse))) ? entry : undefined;
+    }
+    // Whoever uses an account the run needed approved is to see it and decide.
+    mayDecide(entry, canDecide) {
+        return (entry.run.approvals ?? []).some((approval) => canDecide(approval.account));
+    }
+    expire(entry) {
+        if (entry.run.state === 'awaiting-approval') {
+            this.finish(entry, 'rejected', 'nobody approved it in time');
+        }
+    }
+    clearTimer(entry) {
+        if (entry.timer) {
+            clearTimeout(entry.timer);
+            entry.timer = undefined;
+        }
+    }
+    owns(owner, entry) {
+        return entry.run.owner.iss === owner.iss && entry.run.owner.sub === owner.sub;
+    }
+    copy(run) {
+        return {
+            ...run,
+            tasks: [...run.tasks],
+            definition: { ...run.definition },
+            source: { ...run.source },
+            options: JSON.parse(JSON.stringify(run.options)),
+            owner: { ...run.owner },
+            accounts: [...run.accounts],
+            ...(run.approvals ? { approvals: JSON.parse(JSON.stringify(run.approvals)) } : {}),
+        };
+    }
+    timestamp() {
+        return new Date(this.now()).toISOString();
+    }
+    // What a caller has now, against what they may have.
+    checkLimits(owner, waiting) {
+        const limits = this.options.limits;
+        if (!limits) {
+            return;
+        }
+        const mine = [...this.entries.values()].filter((entry) => this.owns(owner, entry));
+        const count = (state) => mine.filter((entry) => entry.run.state === state).length;
+        if (waiting && count('awaiting-approval') >= limits.waiting) {
+            throw new QuotaError(`you have ${limits.waiting} runs waiting for an approval, wait for a decision or cancel one`);
+        }
+        if (!waiting && count('queued') >= limits.queued) {
+            throw new QuotaError(`you have ${limits.queued} runs in the queue, wait for one to start or cancel one`);
+        }
+    }
+    runningOf(owner) {
+        return [...this.entries.values()].filter((entry) => entry.run.state === 'running' && this.owns(owner, entry)).length;
+    }
+    // The first run in the queue whose caller has a place left: one caller with many runs
+    // does not keep the others from the machine.
+    dispatch() {
+        while (this.running < this.maxConcurrent) {
+            const index = this.queue.findIndex((entry) => !this.options.limits || this.runningOf(entry.run.owner) < this.options.limits.running);
+            if (index < 0) {
+                return;
+            }
+            const [entry] = this.queue.splice(index, 1);
+            this.running++;
+            this.execute(entry).finally(() => {
+                this.running--;
+                this.dispatch();
+            });
+        }
+    }
+    async execute(entry) {
+        entry.run.state = 'running';
+        entry.run.startedAt = this.timestamp();
+        this.record(entry, { type: 'state', state: 'running' });
+        const limit = this.options.maxDuration;
+        const timer = limit
+            ? setTimeout(() => {
+                entry.timedOut = true;
+                entry.abort.abort();
+            }, limit)
+            : undefined;
+        timer?.unref();
+        try {
+            await this.options.executor.execute(this.copy(entry.run), {
+                emit: (event) => this.record(entry, event),
+                signal: entry.abort.signal,
+            });
+            this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
+        }
+        catch (e) {
+            const cancelled = entry.abort.signal.aborted && !entry.timedOut;
+            this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : e instanceof Error ? e.message : String(e));
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    tooLong() {
+        return `the run took longer than ${(0, units_1.formatDuration)(this.options.maxDuration)} and was stopped`;
+    }
+    finish(entry, state, error) {
+        this.clearTimer(entry);
+        entry.run.state = state;
+        entry.run.finishedAt = this.timestamp();
+        if (error) {
+            entry.run.error = error;
+        }
+        this.record(entry, { type: 'state', state, ...(error ? { error } : {}) }, true);
+        entry.listeners.clear();
+        this.evict();
+    }
+    record(entry, event, force = false) {
+        // an executor that keeps talking can't fill the memory of the server or its disk; the
+        // state changes, which are few, are always kept
+        if (!force && event.type !== 'state') {
+            const size = event.type === 'log' ? Buffer.byteLength(event.text) + 64 : 64;
+            if (entry.events.length >= this.maxEvents || entry.bytes + size > this.maxLogBytes) {
+                if (!entry.truncated) {
+                    entry.truncated = true;
+                    this.push(entry, { type: 'truncated' });
+                }
+                return;
+            }
+            entry.bytes += size;
+        }
+        this.push(entry, event);
+    }
+    push(entry, event) {
+        const stored = { ...event, seq: entry.events.length + 1, time: this.timestamp() };
+        entry.events.push(stored);
+        for (const listener of entry.listeners) {
+            listener(stored);
+        }
+        // what a restart must not lose: the runs that wait, and how a run ended. A running
+        // run is not kept up to date, it fails with the server.
+        if (event.type === 'state' || event.type === 'approval') {
+            this.persist(entry);
+        }
+    }
+    persist(entry) {
+        const store = this.options.store;
+        if (!store) {
+            return;
+        }
+        const record = { run: this.copy(entry.run), events: [...entry.events] };
+        entry.saving = entry.saving
+            .then(() => store.save(record))
+            .catch((e) => this.options.log?.(`could not keep run ${record.run.id}: ${e instanceof Error ? e.message : e}`));
+    }
+    // Only finished runs are dropped, oldest first.
+    evict() {
+        const finished = [...this.entries.values()].filter((entry) => FINAL_STATES.includes(entry.run.state));
+        for (const entry of finished.slice(0, Math.max(0, finished.length - this.keep))) {
+            this.entries.delete(entry.run.id);
+            void entry.saving.then(() => this.options.store?.remove(entry.run.id)).catch(() => undefined);
+        }
+    }
+}
+exports.RunRegistry = RunRegistry;
+// Used until a backend is wired in: runs are accepted, recorded and failed with
+// a reason, so the API can be used and tested without pretending to execute.
+exports.unavailableExecutor = {
+    async execute() {
+        throw new Error('this server has no backend that can execute runs yet');
+    },
+};
+//# sourceMappingURL=runs.js.map

package/dist/server/server-command.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerServerCommand = void 0;
+const commander_1 = require("commander");
+const log_1 = require("../log");
+const server_config_1 = require("./server-config");
+const server_1 = require("./server");
+// Runs this machine as a hammerkit server until interrupted. Like `remote`, it
+// only needs its own config file, not a build file.
+function registerServerCommand(program, environment) {
+    program
+        .command('server')
+        .description('serve this machine to remote runs (experimental)')
+        .addOption(new commander_1.Option('-c, --config <file>', 'server config file').env('HAMMERKIT_SERVER_CONFIG'))
+        .action(async (options) => {
+        try {
+            if (!options.config) {
+                throw new Error('pass the server config with --config or HAMMERKIT_SERVER_CONFIG');
+            }
+            const config = await (0, server_config_1.readServerConfig)(environment, options.config);
+            const server = await (0, server_1.startServer)(config, environment, {
+                log: (line) => environment.stdout.write(`${line}\n`),
+            });
+            environment.stdout.write(`hammerkit server listening on ${server.url}\n`);
+            // SIGHUP reads the entitlements again, so who may do what changes without
+            // a restart; the rest of the config needs one
+            const reload = async () => {
+                try {
+                    const result = server.reload(await (0, server_config_1.readServerConfig)(environment, options.config));
+                    if (result.restart.length > 0) {
+                        environment.stdout.write(`changed but not reloaded, restart for: ${result.restart.join(', ')}\n`);
+                    }
+                }
+                catch (e) {
+                    environment.stdout.write(`entitlements were not reloaded: ${(0, log_1.getErrorMessage)(e)}\n`);
+                }
+            };
+            const onReload = () => void reload();
+            if (process.platform !== 'win32') {
+                process.on('SIGHUP', onReload);
+            }
+            await new Promise((resolve) => {
+                if (environment.abortCtrl.signal.aborted) {
+                    resolve();
+                }
+                environment.abortCtrl.signal.addEventListener('abort', () => resolve(), { once: true });
+            });
+            process.off('SIGHUP', onReload);
+            await server.close();
+            environment.stdout.write('hammerkit server stopped\n');
+        }
+        catch (e) {
+            if (e instanceof commander_1.CommanderError) {
+                throw e;
+            }
+            program.error(`Server was not successful: ${(0, log_1.getErrorMessage)(e)}`, { exitCode: 1 });
+        }
+    });
+}
+exports.registerServerCommand = registerServerCommand;
+//# sourceMappingURL=server-command.js.map