hammerkit 1.24.0 → 1.25.0

package/dist/server/run-worker.js

@@ -0,0 +1,179 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.workerMain = exports.runTasks = exports.workerInputSchema = void 0;
+const zod_1 = require("zod");
+const cli_1 = require("../cli");
+const get_file_context_1 = require("../file/get-file-context");
+const log_1 = require("../log");
+const schema_parser_1 = require("../schema/schema-parser");
+const reference_parser_1 = require("../schema/reference-parser");
+const work_scope_parser_1 = require("../schema/work-scope-parser");
+const work_item_status_1 = require("../planner/work-item-status");
+const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
+const redact_1 = require("../utils/redact");
+const empty_writable_1 = require("../utils/empty-writable");
+const units_1 = require("../utils/units");
+const run_request_1 = require("./run-request");
+const run_plan_1 = require("./run-plan");
+// one log line of a task is cut here, the registry caps how many events a run keeps
+const MAX_LOG_TEXT = 8 * 1024;
+/**
+ * What the process that executes a run is told, on its standard input. A run executes
+ * in a process of its own, started with an environment the server built, so what the
+ * engine reads of its process (the home directory, the docker host, the cache directory)
+ * is the run's and not the server's.
+ */
+exports.workerInputSchema = zod_1.z
+    .object({
+    // the build file to start from, in the checkout
+    entry: zod_1.z.string().min(1),
+    tasks: zod_1.z.array(zod_1.z.string()),
+    options: run_request_1.runOptionsSchema,
+    // the accounts the run was admitted for; a secret of any other is refused
+    accounts: zod_1.z.array(zod_1.z.string()),
+    // the operator's providers and accounts, as the server read them from its config
+    catalog: zod_1.z.custom((value) => typeof value === 'object' && value !== null),
+})
+    .strict();
+/**
+ * Executes the tasks of a run with the engine of `hammerkit run`. The caller says which
+ * process this is in: it reports what happens through `emit` and stops when `abortCtrl`
+ * is aborted.
+ */
+async function runTasks(input, cwd, processEnvs, abortCtrl, 
+// whether the run was stopped from outside; the engine aborts too when a task fails
+cancelled, emit) {
+    const secrets = (0, redact_1.createSecretRegistry)();
+    const status = (0, work_item_status_1.statusConsole)((0, empty_writable_1.emptyWritable)(), secrets);
+    // what the engine would print to a terminal is not wanted, its status console
+    // is what is streamed: it is redacted with the secrets of this run
+    const environment = {
+        processEnvs,
+        hosted: { catalog: input.catalog, accounts: input.accounts },
+        abortCtrl,
+        cwd,
+        file: (0, get_file_context_1.getFileContext)(cwd),
+        console: (0, log_1.consoleContext)((0, empty_writable_1.emptyWritable)()),
+        status,
+        secrets,
+        stdout: (0, empty_writable_1.emptyWritable)(),
+        stderr: (0, empty_writable_1.emptyWritable)(),
+        stdoutColumns: 120,
+    };
+    const listener = status.on((message) => {
+        if (message.type === 'console') {
+            emit({
+                type: 'log',
+                task: message.context.name,
+                stream: message.console,
+                text: message.message.slice(0, MAX_LOG_TEXT),
+            });
+        }
+        else if (message.level === 'error' || message.level === 'warn') {
+            emit({
+                type: 'log',
+                task: message.context.name,
+                stream: 'stderr',
+                text: message.message.slice(0, MAX_LOG_TEXT),
+            });
+        }
+    });
+    try {
+        const options = input.options;
+        const execOptions = {
+            logMode: 'live',
+            cacheDefault: options.cache ?? 'checksum',
+            cacheReadOnly: options.cacheReadOnly ?? false,
+            timeout: options.timeout ? (0, units_1.parseDuration)(options.timeout) : null,
+            skipDeps: options.skipDeps !== false,
+        };
+        for (const { name, scope } of (0, run_plan_1.getRunScopes)(input.tasks, options)) {
+            const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(input.entry, environment);
+            const workTree = (0, work_scope_parser_1.getWorkContext)(await (0, reference_parser_1.parseReferences)(ctx, parsed, environment), scope, environment);
+            const cli = (0, cli_1.getCli)(workTree, environment);
+            if (cli.tasks().length === 0) {
+                throw new Error(`no tasks found for ${name}`);
+            }
+            const execution = cli.exec({ ...execOptions, workers: cli.defaultWorkers() });
+            const seen = new Map();
+            const watch = execution.state.on('run-events', () => reportTasks(emit, execution.state.current, seen));
+            try {
+                const result = await execution.start();
+                reportTasks(emit, execution.state.current, seen);
+                if (!result.success) {
+                    throw new Error(cancelled() ? 'the run was cancelled' : `${name} failed`);
+                }
+            }
+            finally {
+                watch.close();
+            }
+        }
+    }
+    finally {
+        listener.close();
+    }
+}
+exports.runTasks = runTasks;
+// A task event when a task has moved on since the last look.
+function reportTasks(emit, workTree, seen) {
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        const state = task.state.current;
+        let event = null;
+        if (state.type === 'running') {
+            event = 'running';
+        }
+        else if (state.type === 'completed') {
+            event = state.skipped ? null : state.cached ? 'cached' : 'succeeded';
+        }
+        else if (state.type === 'error' || state.type === 'crash') {
+            event = 'failed';
+        }
+        if (event && seen.get(task.name) !== event) {
+            seen.set(task.name, event);
+            emit({ type: 'task', task: task.name, state: event });
+        }
+    }
+}
+async function readAll(stream) {
+    const chunks = [];
+    for await (const chunk of stream) {
+        chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    }
+    return Buffer.concat(chunks).toString('utf8');
+}
+/**
+ * The process of one run: reads its input, executes the tasks in the current directory
+ * and reports as lines of JSON. It exits 0 when it could say how the run ended, and the
+ * server reads that from the last line; a process that dies before has crashed.
+ */
+async function workerMain(stdin, stdout, processEnvs, cwd, abortCtrl) {
+    const say = (message) => stdout.write(`${JSON.stringify(message)}\n`);
+    // the server stops a run with a signal
+    let cancelled = false;
+    const stop = () => {
+        cancelled = true;
+        abortCtrl.abort();
+    };
+    const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
+    signals.forEach((signal) => process.on(signal, stop));
+    let input;
+    try {
+        input = exports.workerInputSchema.parse(JSON.parse(await readAll(stdin)));
+    }
+    catch (e) {
+        say({ kind: 'result', ok: false, error: `the run could not be read: ${e instanceof Error ? e.message : e}` });
+        return;
+    }
+    try {
+        await runTasks(input, cwd, processEnvs, abortCtrl, () => cancelled, (event) => say({ kind: 'event', event }));
+        say({ kind: 'result', ok: true });
+    }
+    catch (e) {
+        say({ kind: 'result', ok: false, error: e instanceof Error ? e.message : String(e) });
+    }
+    finally {
+        signals.forEach((signal) => process.off(signal, stop));
+    }
+}
+exports.workerMain = workerMain;
+//# sourceMappingURL=run-worker.js.map

package/dist/server/runs.js

@@ -1,15 +1,29 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.unavailableExecutor = exports.RunRegistry = exports.RUN_ID_PATTERN = exports.RunError = exports.isFinal = void 0;
+exports.unavailableExecutor = exports.RunRegistry = exports.RUN_ID_PATTERN = exports.QuotaError = exports.RunError = exports.isFinal = void 0;
 const crypto_1 = require("crypto");
+const units_1 = require("../utils/units");
 const FINAL_STATES = ['succeeded', 'failed', 'cancelled', 'rejected'];
 function isFinal(state) {
     return FINAL_STATES.includes(state);
 }
 exports.isFinal = isFinal;
+// The server can not take the run now; `status` is what the caller is told
 class RunError extends Error {
+    constructor() {
+        super(...arguments);
+        this.status = 503;
+    }
 }
 exports.RunError = RunError;
+// A caller is over one of their limits, others may still submit
+class QuotaError extends RunError {
+    constructor() {
+        super(...arguments);
+        this.status = 429;
+    }
+}
+exports.QuotaError = QuotaError;
 exports.RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]{16}$/;
 // Where runs are kept while the server is up. Everything is scoped to the
 // caller that submitted the run: the same id asked for by someone else is
@@ -27,16 +41,58 @@ class RunRegistry {
         this.maxWaiting = options.maxWaiting ?? 100;
         this.approvalTimeout = options.approvalTimeout ?? 60 * 60 * 1000;
         this.now = options.now ?? Date.now;
+        this.maxLogBytes = options.maxLogBytes ?? Infinity;
     }
-    // A run that needs an approval waits for it, the others are queued at once.
-    submit(owner, spec, needsApproval = false) {
+    /**
+     * Takes back the runs of a previous start. Those that waited for an approval wait
+     * again, for what is left of their time; those that were queued or running when the
+     * server stopped failed with it, and say so.
+     */
+    async restore() {
+        const store = this.options.store;
+        if (!store) {
+            return;
+        }
+        const { records, skipped } = await store.load();
+        skipped.forEach((name) => this.options.log?.(`skip run record ${name}: it can not be read`));
+        for (const { run, events } of records) {
+            const entry = {
+                run,
+                events,
+                listeners: new Set(),
+                abort: new AbortController(),
+                truncated: events.some((event) => event.type === 'truncated'),
+                bytes: 0,
+                saving: Promise.resolve(),
+            };
+            this.entries.set(run.id, entry);
+            if (run.state === 'awaiting-approval') {
+                const left = Date.parse(run.approvals?.[0]?.expiresAt ?? '') - this.now();
+                if (Number.isFinite(left) && left > 0) {
+                    entry.timer = setTimeout(() => this.expire(entry), left);
+                    entry.timer.unref();
+                }
+                else {
+                    this.finish(entry, 'rejected', 'nobody approved it in time');
+                }
+            }
+            else if (run.state === 'queued' || run.state === 'running') {
+                this.finish(entry, 'failed', `the server stopped while the run was ${run.state}, submit it again`);
+            }
+        }
+        this.evict();
+    }
+    // A run that needs approvals (for the accounts named) waits for them, the others
+    // are queued at once.
+    submit(owner, spec, approvalAccounts = []) {
         if (this.closed) {
             throw new RunError('the server is shutting down');
         }
-        const waiting = needsApproval;
+        const waiting = approvalAccounts.length > 0;
         if (waiting && this.waiting().length >= this.maxWaiting) {
             throw new RunError('too many runs are waiting for an approval');
         }
+        this.checkLimits(owner, waiting);
         const run = {
             ...spec,
             id: `run_${(0, crypto_1.randomBytes)(12).toString('base64url')}`,
@@ -44,13 +100,23 @@ class RunRegistry {
             state: waiting ? 'awaiting-approval' : 'queued',
             createdAt: this.timestamp(),
         };
-        const entry = { run, events: [], listeners: new Set(), abort: new AbortController(), truncated: false };
+        const entry = {
+            run,
+            events: [],
+            listeners: new Set(),
+            abort: new AbortController(),
+            truncated: false,
+            bytes: 0,
+            saving: Promise.resolve(),
+        };
         this.entries.set(run.id, entry);
+        if (waiting) {
+            // before the first event, which is written down with the run
+            const expiresAt = new Date(this.now() + this.approvalTimeout).toISOString();
+            run.approvals = approvalAccounts.map((account) => ({ account, expiresAt }));
+        }
         this.record(entry, { type: 'state', state: run.state });
         if (waiting) {
-            run.approval = {
-                expiresAt: new Date(this.now() + this.approvalTimeout).toISOString(),
-            };
             entry.timer = setTimeout(() => this.expire(entry), this.approvalTimeout);
             entry.timer.unref();
         }
@@ -63,8 +129,10 @@ class RunRegistry {
         this.dispatch();
         return accepted;
     }
-    get(owner, id) {
-        const entry = this.find(owner, id);
+    // `canUse` also lets who uses an account the run needed approved read it: they
+    // decided about it, so they may follow it and stop it.
+    get(owner, id, canUse) {
+        const entry = this.find(owner, id, canUse);
         return entry ? this.copy(entry.run) : undefined;
     }
     // newest first
@@ -75,8 +143,8 @@ class RunRegistry {
             .reverse();
     }
     // A queued run is cancelled at once; a running one when its executor stops.
-    cancel(owner, id) {
-        const entry = this.find(owner, id);
+    cancel(owner, id, canUse) {
+        const entry = this.find(owner, id, canUse);
         if (!entry) {
             return undefined;
         }
@@ -99,8 +167,8 @@ class RunRegistry {
             .filter((entry) => entry.run.state === 'awaiting-approval')
             .map((entry) => this.copy(entry.run));
     }
-    // One run that waits (or waited), if `canDecide` says the decider is admin of
-    // its account.
+    // One run that waits (or waited), if `canDecide` says the decider uses one of the
+    // accounts it still needs approved.
     getForApproval(id, canDecide) {
         const entry = this.entries.get(id);
         if (!entry || !this.mayDecide(entry, canDecide)) {
@@ -108,7 +176,9 @@ class RunRegistry {
         }
         return this.copy(entry.run);
     }
-    // An approval queues the run, a rejection ends it. Nobody decides on their own run.
+    // An approval covers the accounts the approver uses; the run is queued when all
+    // of its accounts are covered. A rejection by anybody who covers one of them ends
+    // it. Nobody decides on their own run.
     decide(decider, id, outcome, canDecide, reason) {
         const entry = this.entries.get(id);
         if (!entry || !this.mayDecide(entry, canDecide)) {
@@ -117,17 +187,31 @@ class RunRegistry {
         if (this.owns(decider, entry)) {
             return { status: 'own-run' };
         }
-        if (entry.run.state !== 'awaiting-approval' || !entry.run.approval) {
+        if (entry.run.state !== 'awaiting-approval' || !entry.run.approvals) {
             return { status: 'not-waiting', state: entry.run.state };
         }
         const by = { iss: decider.iss, sub: decider.sub };
-        this.record(entry, { type: 'approval', outcome, by, ...(reason ? { reason } : {}) });
+        const covered = entry.run.approvals.filter((approval) => !approval.approvedBy && canDecide(approval.account));
+        if (outcome === 'approved' && covered.length === 0) {
+            return { status: 'already-approved' };
+        }
+        const accounts = entry.run.approvals
+            .filter((approval) => canDecide(approval.account))
+            .map((approval) => approval.account);
+        this.record(entry, { type: 'approval', outcome, by, accounts, ...(reason ? { reason } : {}) });
         if (outcome === 'rejected') {
             this.finish(entry, 'rejected', `rejected by ${by.sub}${reason ? `: ${reason}` : ''}`);
             return { status: 'ok', run: this.copy(entry.run) };
         }
-        entry.run.approval.approvedBy = by;
-        entry.run.approval.approvedAt = this.timestamp();
+        const approvedAt = this.timestamp();
+        for (const approval of covered) {
+            approval.approvedBy = by;
+            approval.approvedAt = approvedAt;
+        }
+        if (entry.run.approvals.some((approval) => !approval.approvedBy)) {
+            // some accounts still wait for somebody who uses them
+            return { status: 'ok', run: this.copy(entry.run) };
+        }
         this.clearTimer(entry);
         entry.run.state = 'queued';
         this.record(entry, { type: 'state', state: 'queued' });
@@ -138,8 +222,8 @@ class RunRegistry {
     }
     // The events after `after` (0 for all), and a way to follow new ones. The
     // listener is not called for a run that is already over.
-    watch(owner, id, after, listener) {
-        const entry = this.find(owner, id);
+    watch(owner, id, after, listener, canUse) {
+        const entry = this.find(owner, id, canUse);
         if (!entry) {
             return undefined;
         }
@@ -158,7 +242,13 @@ class RunRegistry {
         }
         for (const entry of this.entries.values()) {
             if (entry.run.state === 'awaiting-approval') {
-                this.finish(entry, 'cancelled');
+                if (this.options.store) {
+                    // the record keeps it waiting, the next start takes it back
+                    this.clearTimer(entry);
+                }
+                else {
+                    this.finish(entry, 'cancelled');
+                }
             }
         }
         for (const entry of this.entries.values()) {
@@ -169,13 +259,16 @@ class RunRegistry {
         while (this.running > 0) {
             await new Promise((resolve) => setTimeout(resolve, 10));
         }
+        // the runs that were stopped are written down before the server is gone
+        await Promise.all([...this.entries.values()].map((entry) => entry.saving));
     }
-    find(owner, id) {
+    find(owner, id, canUse) {
         const entry = this.entries.get(id);
-        return entry && this.owns(owner, entry) ? entry : undefined;
+        return entry && (this.owns(owner, entry) || (canUse && this.mayDecide(entry, canUse))) ? entry : undefined;
     }
+    // Whoever uses an account the run needed approved is to see it and decide.
     mayDecide(entry, canDecide) {
-        return entry.run.approval !== undefined && canDecide(entry.run.account);
+        return (entry.run.approvals ?? []).some((approval) => canDecide(approval.account));
     }
     expire(entry) {
         if (entry.run.state === 'awaiting-approval') {
@@ -199,15 +292,40 @@ class RunRegistry {
             source: { ...run.source },
             options: JSON.parse(JSON.stringify(run.options)),
             owner: { ...run.owner },
-            ...(run.approval ? { approval: JSON.parse(JSON.stringify(run.approval)) } : {}),
+            accounts: [...run.accounts],
+            ...(run.approvals ? { approvals: JSON.parse(JSON.stringify(run.approvals)) } : {}),
         };
     }
     timestamp() {
         return new Date(this.now()).toISOString();
     }
+    // What a caller has now, against what they may have.
+    checkLimits(owner, waiting) {
+        const limits = this.options.limits;
+        if (!limits) {
+            return;
+        }
+        const mine = [...this.entries.values()].filter((entry) => this.owns(owner, entry));
+        const count = (state) => mine.filter((entry) => entry.run.state === state).length;
+        if (waiting && count('awaiting-approval') >= limits.waiting) {
+            throw new QuotaError(`you have ${limits.waiting} runs waiting for an approval, wait for a decision or cancel one`);
+        }
+        if (!waiting && count('queued') >= limits.queued) {
+            throw new QuotaError(`you have ${limits.queued} runs in the queue, wait for one to start or cancel one`);
+        }
+    }
+    runningOf(owner) {
+        return [...this.entries.values()].filter((entry) => entry.run.state === 'running' && this.owns(owner, entry)).length;
+    }
+    // The first run in the queue whose caller has a place left: one caller with many runs
+    // does not keep the others from the machine.
     dispatch() {
-        while (this.running < this.maxConcurrent && this.queue.length > 0) {
-            const entry = this.queue.shift();
+        while (this.running < this.maxConcurrent) {
+            const index = this.queue.findIndex((entry) => !this.options.limits || this.runningOf(entry.run.owner) < this.options.limits.running);
+            if (index < 0) {
+                return;
+            }
+            const [entry] = this.queue.splice(index, 1);
             this.running++;
             this.execute(entry).finally(() => {
                 this.running--;
@@ -219,17 +337,32 @@ class RunRegistry {
         entry.run.state = 'running';
         entry.run.startedAt = this.timestamp();
         this.record(entry, { type: 'state', state: 'running' });
+        const limit = this.options.maxDuration;
+        const timer = limit
+            ? setTimeout(() => {
+                entry.timedOut = true;
+                entry.abort.abort();
+            }, limit)
+            : undefined;
+        timer?.unref();
         try {
             await this.options.executor.execute(this.copy(entry.run), {
                 emit: (event) => this.record(entry, event),
                 signal: entry.abort.signal,
             });
-            this.finish(entry, entry.abort.signal.aborted ? 'cancelled' : 'succeeded');
+            this.finish(entry, entry.timedOut ? 'failed' : entry.abort.signal.aborted ? 'cancelled' : 'succeeded', entry.timedOut ? this.tooLong() : undefined);
         }
         catch (e) {
-            this.finish(entry, entry.abort.signal.aborted ? 'cancelled' : 'failed', entry.abort.signal.aborted ? undefined : e instanceof Error ? e.message : String(e));
+            const cancelled = entry.abort.signal.aborted && !entry.timedOut;
+            this.finish(entry, cancelled ? 'cancelled' : 'failed', entry.timedOut ? this.tooLong() : cancelled ? undefined : e instanceof Error ? e.message : String(e));
+        }
+        finally {
+            clearTimeout(timer);
         }
     }
+    tooLong() {
+        return `the run took longer than ${(0, units_1.formatDuration)(this.options.maxDuration)} and was stopped`;
+    }
     finish(entry, state, error) {
         this.clearTimer(entry);
         entry.run.state = state;
@@ -242,14 +375,18 @@ class RunRegistry {
         this.evict();
     }
     record(entry, event, force = false) {
-        // an executor that keeps talking can't fill the memory of the server; the
+        // an executor that keeps talking can't fill the memory of the server or its disk; the
         // state changes, which are few, are always kept
-        if (!force && event.type !== 'state' && entry.events.length >= this.maxEvents) {
-            if (!entry.truncated) {
-                entry.truncated = true;
-                this.push(entry, { type: 'truncated' });
+        if (!force && event.type !== 'state') {
+            const size = event.type === 'log' ? Buffer.byteLength(event.text) + 64 : 64;
+            if (entry.events.length >= this.maxEvents || entry.bytes + size > this.maxLogBytes) {
+                if (!entry.truncated) {
+                    entry.truncated = true;
+                    this.push(entry, { type: 'truncated' });
+                }
+                return;
             }
-            return;
+            entry.bytes += size;
         }
         this.push(entry, event);
     }
@@ -259,12 +396,28 @@ class RunRegistry {
         for (const listener of entry.listeners) {
             listener(stored);
         }
+        // what a restart must not lose: the runs that wait, and how a run ended. A running
+        // run is not kept up to date, it fails with the server.
+        if (event.type === 'state' || event.type === 'approval') {
+            this.persist(entry);
+        }
+    }
+    persist(entry) {
+        const store = this.options.store;
+        if (!store) {
+            return;
+        }
+        const record = { run: this.copy(entry.run), events: [...entry.events] };
+        entry.saving = entry.saving
+            .then(() => store.save(record))
+            .catch((e) => this.options.log?.(`could not keep run ${record.run.id}: ${e instanceof Error ? e.message : e}`));
     }
     // Only finished runs are dropped, oldest first.
     evict() {
         const finished = [...this.entries.values()].filter((entry) => FINAL_STATES.includes(entry.run.state));
         for (const entry of finished.slice(0, Math.max(0, finished.length - this.keep))) {
             this.entries.delete(entry.run.id);
+            void entry.saving.then(() => this.options.store?.remove(entry.run.id)).catch(() => undefined);
         }
     }
 }

package/dist/server/server-command.js

@@ -22,12 +22,30 @@ function registerServerCommand(program, environment) {
                 log: (line) => environment.stdout.write(`${line}\n`),
             });
             environment.stdout.write(`hammerkit server listening on ${server.url}\n`);
+            // SIGHUP reads the entitlements again, so who may do what changes without
+            // a restart; the rest of the config needs one
+            const reload = async () => {
+                try {
+                    const result = server.reload(await (0, server_config_1.readServerConfig)(environment, options.config));
+                    if (result.restart.length > 0) {
+                        environment.stdout.write(`changed but not reloaded, restart for: ${result.restart.join(', ')}\n`);
+                    }
+                }
+                catch (e) {
+                    environment.stdout.write(`entitlements were not reloaded: ${(0, log_1.getErrorMessage)(e)}\n`);
+                }
+            };
+            const onReload = () => void reload();
+            if (process.platform !== 'win32') {
+                process.on('SIGHUP', onReload);
+            }
             await new Promise((resolve) => {
                 if (environment.abortCtrl.signal.aborted) {
                     resolve();
                 }
                 environment.abortCtrl.signal.addEventListener('abort', () => resolve(), { once: true });
             });
+            process.off('SIGHUP', onReload);
             await server.close();
             environment.stdout.write('hammerkit server stopped\n');
         }