hammerkit 1.27.0-next.9 → 2.0.0-next.2

package/dist/server/kubernetes-sandbox-storage.js

@@ -11,6 +11,7 @@ const tar_2 = require("../utils/tar");
 const apply_1 = require("../kubernetes/apply");
 const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
 const network_policy_1 = require("../kubernetes/network-policy");
+const sandbox_runner_1 = require("../kubernetes/sandbox-runner");
 const volumes_1 = require("../kubernetes/volumes");
 const log_1 = require("../log");
 const sleep_1 = require("../utils/sleep");
@@ -260,6 +261,41 @@ class KubernetesSandboxStorage {
             await (0, promises_1.rm)(repository, { recursive: true, force: true });
         }
     }
+    // The pod writes the tree's files beside the sandbox on the claim and sends them as a tar,
+    // which is extracted here as the files of `attach` are: what a run plans with, up to the
+    // bytes a sandbox may take.
+    async exportTree(key, tree, into, signal) {
+        await (0, promises_1.mkdir)(into, { recursive: true });
+        let bytes = 0;
+        const extract = (0, tar_1.extract)({
+            cwd: into,
+            filter: (_path, entry) => 'type' in entry && ['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type),
+        });
+        const counted = new stream_1.PassThrough();
+        counted.on('data', (chunk) => {
+            bytes += chunk.length;
+            if (bytes > this.limits.bytes) {
+                counted.destroy(new Error(`the files of the sandbox ${key.id} take more than ${(0, units_1.formatSize)(this.limits.bytes)}`));
+            }
+        });
+        const extracted = new Promise((resolve, reject) => {
+            extract.on('finish', resolve);
+            extract.on('error', reject);
+            counted.on('error', reject);
+        });
+        extracted.catch(() => undefined);
+        counted.pipe(extract);
+        const answer = await this.exclusive(key.id, () => this.call(key, { op: 'archive', id: key.id, tree }, signal, true));
+        if ('stream' in answer) {
+            // a pipeline, so the answer is ended with the copy: an error of the pod after the copy
+            // stopped (over the size limit) lands nowhere else
+            await (0, promises_2.pipeline)(answer.stream, counted);
+        }
+        else {
+            counted.end(answer.body);
+        }
+        await extracted;
+    }
     async moveTo(key, commit) {
         await this.request(key, { op: 'move', id: key.id, commit }, AbortSignal.timeout(FETCH_TIMEOUT_MS));
     }
@@ -284,6 +320,8 @@ class KubernetesSandboxStorage {
             // runs on a sandbox go one at a time: a proxy the pod of the files does not own was
             // left by a run before (the server stopped, or its release could not remove it)
             await (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label }, (deployment) => isOwnedBy(deployment, pod.uid)).catch(() => undefined);
+            // and a runner, which the pod did not own when it made way for a run
+            await (0, sandbox_runner_1.removeSandboxRunners)(this.instance, namespace, names.label, (runner) => isOwnedBy(runner, pod.uid)).catch(() => undefined);
             await (0, promises_1.mkdir)(into, { recursive: true });
             await this.copyOut(key, ['tar', 'cf', '-', '-C', `${CLAIM_PATH}/${key.id}/files`, '.'], into, 
             // what the run plans with: files, directories and links, nothing outside of it
@@ -388,7 +426,8 @@ class KubernetesSandboxStorage {
     async removeIn(namespace, id) {
         const names = getSandboxNames(id);
         await this.stopPod({ id, owner: { iss: '', sub: '' } }, namespace);
-        // the cluster removes the proxies the pod owned; those it did not own go here
+        // the cluster removes the proxies and the runner the pod owned; those it did not own go here
+        await (0, sandbox_runner_1.removeSandboxRunners)(this.instance, namespace, names.label);
         await (0, network_policy_1.removeSandboxProxies)(this.instance, namespace, { sandbox: names.label });
         for (const name of [names.claim, names.git]) {
             try {
@@ -502,7 +541,7 @@ class KubernetesSandboxStorage {
         const reader = new AnswerReader(body);
         // the process ends when the server stops waiting for it: closing the connection does
         // not end a command in a pod
-        const seconds = Math.ceil((['create', 'fetch', 'move'].includes(request.op) ? FETCH_TIMEOUT_MS : REQUEST_TIMEOUT_MS) / 1000);
+        const seconds = Math.ceil((['create', 'fetch', 'move', 'archive'].includes(request.op) ? FETCH_TIMEOUT_MS : REQUEST_TIMEOUT_MS) / 1000);
         const done = this.withPod(key, (pod) => this.exec(pod, ['timeout', String(seconds), 'hammerkit', 'server-sandbox', CLAIM_PATH], stream_1.Readable.from([Buffer.from(JSON.stringify(encoded))]), reader, signal));
         done.then(() => reader.complete(), (e) => reader.fail(e));
         const answer = await reader.answer;
@@ -885,6 +924,11 @@ class AnswerReader extends stream_1.Writable {
     }
     _write(chunk, _encoding, callback) {
         if (this.stream) {
+            if (this.stream.destroyed) {
+                // its reader stopped (over a size limit): the rest of the answer is nobody's
+                callback();
+                return;
+            }
             this.stream.write(chunk, callback);
             return;
         }

package/dist/server/overview.js

@@ -0,0 +1,271 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.visibleCapacity = exports.filterRuns = exports.FilterRefused = exports.parseRunFilter = exports.getOverview = exports.queuePlace = exports.visibleReason = exports.toListedRun = exports.mayRead = exports.seesActivityOf = exports.seesActivity = exports.maySee = exports.decides = exports.owns = exports.mayDecide = exports.runActions = exports.getViewer = exports.describeWait = void 0;
+const policy_1 = require("./policy");
+const runs_1 = require("./runs");
+const service_runs_1 = require("./service-runs");
+const repo_1 = require("./repo");
+var wait_reason_1 = require("./wait-reason");
+Object.defineProperty(exports, "describeWait", { enumerable: true, get: function () { return wait_reason_1.describeWait; } });
+function getViewer(config, principal) {
+    return {
+        principal,
+        grant: (0, policy_1.getGrant)(config.entitlements, principal),
+        activity: config.runs.activity === 'everyone',
+    };
+}
+exports.getViewer = getViewer;
+function runActions(viewer, run) {
+    if ((0, runs_1.isFinal)(run.state)) {
+        return undefined;
+    }
+    const mine = owns(viewer, run);
+    const may = {
+        ...(mine || decides(viewer, run) ? { stop: true } : {}),
+        ...(run.up && run.upAt && (mine || (0, service_runs_1.mayOpenEndpoints)(viewer.grant, run)) ? { extend: true } : {}),
+        ...(mayDecide(viewer, run) ? { decide: true } : {}),
+    };
+    return Object.keys(may).length > 0 ? may : undefined;
+}
+exports.runActions = runActions;
+// A run that waits for an approval the caller may give: not their own.
+function mayDecide(viewer, run) {
+    return (run.state === 'awaiting-approval' &&
+        !owns(viewer, run) &&
+        (run.approvals ?? []).some((approval) => !approval.approvedBy && (0, policy_1.canUse)(viewer.grant, approval.account)));
+}
+exports.mayDecide = mayDecide;
+function owns(viewer, run) {
+    return run.owner.iss === viewer.principal.iss && run.owner.sub === viewer.principal.sub;
+}
+exports.owns = owns;
+// Whether the caller decides about the run: they use an account it asked to be approved.
+function decides(viewer, run) {
+    return (run.approvals ?? []).some((approval) => (0, policy_1.canUse)(viewer.grant, approval.account));
+}
+exports.decides = decides;
+/**
+ * Whether the caller may see a run, with the entitlements of the moment: their own; any
+ * with `observe`; one that asked for an approval of an account they use (they decide about
+ * it); a service run whose endpoints they may open; and with `runs.activity: everyone`, one
+ * of a repository they have access to (seesActivityOf).
+ */
+function maySee(viewer, run) {
+    return (owns(viewer, run) ||
+        viewer.grant.observe ||
+        decides(viewer, run) ||
+        (!!run.up && (0, service_runs_1.mayOpenEndpoints)(viewer.grant, run)) ||
+        seesActivityOf(viewer, run));
+}
+exports.maySee = maySee;
+/**
+ * With `runs.activity: everyone`, a caller with a `builds` level sees the runs of others in
+ * the repositories they have access to: the `repos` and the `sandboxes` repositories of their
+ * entitlements. Who may only open endpoints or read checks sees none, and a run of uploaded
+ * files has no repository: it is for its owner, who decides about it and `observe`.
+ */
+function seesActivity(viewer) {
+    return !!viewer.activity && viewer.grant.builds !== null;
+}
+exports.seesActivity = seesActivity;
+function seesActivityOf(viewer, run) {
+    if (!seesActivity(viewer) || run.source.type === 'upload') {
+        return false;
+    }
+    const repo = run.source.repo;
+    return ((0, repo_1.matchesAnyRepo)(viewer.grant.repos, repo) ||
+        viewer.grant.sandboxes.some((sandbox) => (0, repo_1.matchesRepo)(sandbox.repo, repo)));
+}
+exports.seesActivityOf = seesActivityOf;
+/**
+ * Whether the caller may read a run whole: its tasks, why it failed and its log. Its owner,
+ * who decides about it, and `observe`.
+ */
+function mayRead(viewer, run) {
+    return owns(viewer, run) || viewer.grant.observe || decides(viewer, run);
+}
+exports.mayRead = mayRead;
+function toListedRun(viewer, run, queue) {
+    const { owner, results, plan, options, up, error, ...rest } = run;
+    const mine = owns(viewer, run);
+    const may = runActions(viewer, run);
+    const opens = mine || viewer.grant.observe || (!!up && (0, service_runs_1.mayOpenEndpoints)(viewer.grant, run));
+    const timings = Object.fromEntries(Object.entries(results ?? {}).map(([task, result]) => [
+        task,
+        {
+            status: result.status,
+            ...(result.startedAt ? { startedAt: result.startedAt } : {}),
+            ...(result.finishedAt ? { finishedAt: result.finishedAt } : {}),
+            ...(result.durationMs !== undefined ? { durationMs: result.durationMs } : {}),
+            ...(result.cache ? { cached: result.cache.hit } : {}),
+        },
+    ]));
+    return {
+        ...rest,
+        ...(mine ? { plan, options } : { owner: { iss: owner.iss, sub: owner.sub } }),
+        ...(up
+            ? {
+                up: {
+                    ...up,
+                    endpoints: opens ? up.endpoints : up.endpoints.map(({ service, port }) => ({ service, port })),
+                },
+            }
+            : {}),
+        ...(error !== undefined && mayRead(viewer, run) ? { error } : {}),
+        ...(Object.keys(timings).length > 0 ? { timings } : {}),
+        ...(queue ? { queue } : {}),
+        ...(may ? { may } : {}),
+        ...(!mine && mayRead(viewer, run) ? { readable: true } : {}),
+    };
+}
+exports.toListedRun = toListedRun;
+// A reason names a run only the caller may see.
+function visibleReason(reason, visible) {
+    if ((reason.type === 'sandbox' || reason.type === 'held') && !visible(reason.run)) {
+        return { type: reason.type };
+    }
+    return { ...reason };
+}
+exports.visibleReason = visibleReason;
+// Where a queued run stands, and why it waits: the run that holds it is named when the
+// caller may see it.
+function queuePlace(registry, viewer, run) {
+    const queue = run.state === 'queued' ? registry.queued(run.id) : undefined;
+    if (!queue) {
+        return undefined;
+    }
+    const visible = (id) => {
+        const holder = registry.peek(id);
+        return !!holder && maySee(viewer, holder);
+    };
+    return { position: queue.position, waitsFor: visibleReason(queue.waitsFor, visible) };
+}
+exports.queuePlace = queuePlace;
+function getOverview(snapshot, usage, viewer) {
+    const runs = new Map();
+    for (const run of [
+        ...snapshot.running,
+        ...snapshot.queue.map((entry) => entry.run),
+        ...snapshot.awaiting.runs,
+        ...snapshot.services.runs,
+    ]) {
+        runs.set(run.id, run);
+    }
+    const visible = (id) => {
+        const run = runs.get(id);
+        return !!run && maySee(viewer, run);
+    };
+    const queue = snapshot.queue.map((entry, index) => {
+        const place = { position: index + 1, waitsFor: visibleReason(entry.waitsFor, visible) };
+        return {
+            ...place,
+            mine: owns(viewer, entry.run),
+            ...(maySee(viewer, entry.run) ? { run: toListedRun(viewer, entry.run, place) } : {}),
+        };
+    });
+    const running = snapshot.running.filter((run) => maySee(viewer, run));
+    const limits = snapshot.limits;
+    return {
+        you: { iss: viewer.principal.iss, sub: viewer.principal.sub, observe: viewer.grant.observe },
+        places: {
+            max: snapshot.places.max,
+            used: snapshot.places.used,
+            runs: running.map((run) => toListedRun(viewer, run)),
+            others: snapshot.running.length - running.length,
+        },
+        queue,
+        approvals: {
+            waiting: snapshot.awaiting.runs.length,
+            max: snapshot.awaiting.max,
+            runs: snapshot.awaiting.runs.filter((run) => maySee(viewer, run)).map((run) => toListedRun(viewer, run)),
+        },
+        services: {
+            open: snapshot.services.taken,
+            ...(snapshot.services.max !== undefined ? { max: snapshot.services.max } : {}),
+            runs: snapshot.services.runs.filter((run) => maySee(viewer, run)).map((run) => toListedRun(viewer, run)),
+        },
+        limits: {
+            waiting: meter(usage.waiting, limits?.waiting),
+            queued: meter(usage.queued, limits?.queued),
+            running: meter(usage.running, limits?.running),
+            services: meter(usage.services, snapshot.services.perCaller),
+            asleep: meter(usage.asleep, snapshot.services.asleepPerCaller),
+        },
+    };
+}
+exports.getOverview = getOverview;
+function meter(runs, limit) {
+    return { used: runs.length, ...(limit !== undefined ? { limit } : {}), runs: [...runs] };
+}
+const STATES = [
+    'awaiting-approval',
+    'queued',
+    'running',
+    'up',
+    'asleep',
+    'succeeded',
+    'failed',
+    'cancelled',
+    'rejected',
+];
+// `scope`, `state` (a comma separated list) and `kind` of GET /v1/runs.
+function parseRunFilter(params, grant) {
+    const scope = params.get('scope') ?? 'mine';
+    if (scope !== 'mine' && scope !== 'visible' && scope !== 'all') {
+        throw new Error('scope: mine, visible or all');
+    }
+    if (scope === 'all' && !grant.observe) {
+        throw new FilterRefused('scope all needs an entitlement with observe');
+    }
+    const state = params.get('state');
+    const states = state
+        ? state.split(',').map((value) => {
+            if (!STATES.includes(value)) {
+                throw new Error(`state: one or more of ${STATES.join(', ')}`);
+            }
+            return value;
+        })
+        : undefined;
+    const kind = params.get('kind') ?? undefined;
+    if (kind !== undefined && kind !== 'tasks' && kind !== 'services') {
+        throw new Error('kind: tasks or services');
+    }
+    const id = params.get('id') ?? undefined;
+    if (id !== undefined && !/^run_[A-Za-z0-9_-]{16}$/.test(id)) {
+        throw new Error('id: the id of a run, run_...');
+    }
+    return { scope, ...(states ? { states } : {}), ...(kind ? { kind } : {}), ...(id ? { id } : {}) };
+}
+exports.parseRunFilter = parseRunFilter;
+class FilterRefused extends Error {
+}
+exports.FilterRefused = FilterRefused;
+function filterRuns(runs, viewer, filter) {
+    return runs.filter((run) => (filter.scope === 'mine' ? owns(viewer, run) : maySee(viewer, run)) &&
+        (!filter.id || run.id === filter.id) &&
+        (!filter.states || filter.states.includes(run.state)) &&
+        (!filter.kind || (filter.kind === 'services') === !!run.up));
+}
+exports.filterRuns = filterRuns;
+// The machines and what the runs take, with whose the namespaces are where the caller may
+// know: their own, and every one with `observe`.
+function visibleCapacity(capacity, viewer) {
+    const everyone = viewer.grant.observe;
+    return {
+        ...capacity,
+        // what the cluster answered is for operators; others learn that something is missing
+        problems: viewer.grant.observe || capacity.problems.length === 0 ? capacity.problems : ['some of it could not be read'],
+        groups: capacity.groups.map((group) => {
+            const mine = group.owner?.iss === viewer.principal.iss && group.owner?.sub === viewer.principal.sub;
+            if (everyone || mine || !group.owner) {
+                return group;
+            }
+            const { owner, run, ...rest } = group;
+            void owner;
+            void run;
+            return rest;
+        }),
+    };
+}
+exports.visibleCapacity = visibleCapacity;
+//# sourceMappingURL=overview.js.map