hammerkit 1.27.0-next.9 → 2.0.0-next.2

package/dist/server/capacity.js

@@ -0,0 +1,300 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatMemory = exports.formatCpu = exports.HostCapacity = exports.KubernetesCapacity = exports.cachedCapacity = exports.parseQuantity = void 0;
+const os_1 = require("os");
+const log_1 = require("../log");
+const apply_1 = require("../kubernetes/apply");
+const UNITS = {
+    n: 1e-9,
+    u: 1e-6,
+    m: 1e-3,
+    '': 1,
+    k: 1e3,
+    K: 1e3,
+    M: 1e6,
+    G: 1e9,
+    T: 1e12,
+    P: 1e15,
+    E: 1e18,
+    Ki: 1024,
+    Mi: 1024 ** 2,
+    Gi: 1024 ** 3,
+    Ti: 1024 ** 4,
+    Pi: 1024 ** 5,
+    Ei: 1024 ** 6,
+};
+// A quantity of Kubernetes (`250m`, `1.5`, `512Mi`, `12345678n`, `1e3`), as a number.
+function parseQuantity(value) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value === 'number') {
+        return value;
+    }
+    const match = /^([+-]?[0-9.]+(?:[eE][+-]?\d+)?)([a-zA-Z]*)$/.exec(value.trim());
+    if (!match || !(match[2] in UNITS)) {
+        return undefined;
+    }
+    const number = Number(match[1]);
+    return Number.isFinite(number) ? number * UNITS[match[2]] : undefined;
+}
+exports.parseQuantity = parseQuantity;
+function add(into, cpu, memory) {
+    if (cpu !== undefined) {
+        into.cpu = (into.cpu ?? 0) + cpu;
+    }
+    if (memory !== undefined) {
+        into.memory = (into.memory ?? 0) + memory;
+    }
+}
+function addAll(into, from) {
+    if (from) {
+        add(into, from.cpu, from.memory);
+    }
+}
+// Reads at most once in `ms`, and never twice at the same time: the dashboard reloads. A read
+// that takes longer than `wait` is not waited for: a slow cluster API does not hold up a page,
+// the next one shows what it read.
+function cachedCapacity(reader, ms = 15000, now = Date.now, wait = 3000) {
+    let last;
+    const failed = (problem) => ({
+        backend: 'host',
+        at: new Date(now()).toISOString(),
+        runs: { requested: {} },
+        groups: [],
+        problems: [problem],
+    });
+    return {
+        read(runs) {
+            if (!last || now() - last.at >= ms) {
+                const value = reader.read(runs).catch((e) => failed((0, log_1.getErrorMessage)(e)));
+                last = { at: now(), value };
+            }
+            let timer;
+            const late = new Promise((resolve) => {
+                timer = setTimeout(() => resolve(failed('the capacity could not be read in time')), wait);
+                timer.unref?.();
+            });
+            return Promise.race([last.value, late]).finally(() => clearTimeout(timer));
+        },
+    };
+}
+exports.cachedCapacity = cachedCapacity;
+function why(what, e, rule) {
+    const code = (0, apply_1.statusCodeOf)(e);
+    if (code === 403 && rule) {
+        return `${what}: the server may not read it, it needs ${rule}`;
+    }
+    if (code === 404 && what.includes('metrics')) {
+        return `${what}: the cluster serves no metrics API (metrics-server)`;
+    }
+    return `${what}: ${(0, log_1.getErrorMessage)(e)}`;
+}
+const ENDED = ['Succeeded', 'Failed'];
+// On Kubernetes: per namespace of the server, its pods' requests, its quota and its usage.
+class KubernetesCapacity {
+    constructor(instance, options) {
+        this.instance = instance;
+        this.options = options;
+    }
+    async read(runs) {
+        const problems = [];
+        const groups = [];
+        let namespaces = [];
+        try {
+            namespaces = await this.options.namespaces();
+        }
+        catch (e) {
+            problems.push(why('the namespaces of the server', e));
+        }
+        const said = new Set();
+        const once = (text) => {
+            if (!said.has(text)) {
+                said.add(text);
+                problems.push(text);
+            }
+        };
+        await Promise.all(namespaces.map(async ({ name: namespace, kind }) => {
+            const group = {
+                name: namespace,
+                kind,
+                ...this.options.ownerOf(namespace, runs),
+                pods: 0,
+                // a namespace whose pods could be listed and that has none asks for nothing
+                requested: { cpu: 0, memory: 0 },
+            };
+            try {
+                const pods = await this.instance.coreApi.listNamespacedPod({ namespace });
+                for (const pod of pods.items) {
+                    if (ENDED.includes(pod.status?.phase ?? '')) {
+                        continue;
+                    }
+                    group.pods++;
+                    for (const container of pod.spec?.containers ?? []) {
+                        const requests = container.resources?.requests ?? {};
+                        add(group.requested, parseQuantity(requests.cpu) ?? 0, parseQuantity(requests.memory) ?? 0);
+                    }
+                }
+            }
+            catch (e) {
+                once(why('the pods of the namespaces', e, 'list on pods in them (runRole)'));
+            }
+            try {
+                const quota = await this.instance.coreApi.readNamespacedResourceQuota({
+                    name: this.options.quotaName,
+                    namespace,
+                });
+                const hard = quota.spec?.hard ?? {};
+                const cpu = parseQuantity(hard['requests.cpu'] ?? hard.cpu);
+                const memory = parseQuantity(hard['requests.memory'] ?? hard.memory);
+                if (cpu !== undefined || memory !== undefined) {
+                    group.quota = { ...(cpu !== undefined ? { cpu } : {}), ...(memory !== undefined ? { memory } : {}) };
+                }
+            }
+            catch (e) {
+                // a namespace without a quota is no problem
+                if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                    once(why('the quotas of the namespaces', e, 'get on resourcequotas in them (runRole)'));
+                }
+            }
+            if (this.instance.metrics) {
+                try {
+                    const metrics = await this.instance.metrics.getPodMetrics(namespace);
+                    const used = { cpu: 0, memory: 0 };
+                    for (const pod of metrics.items) {
+                        for (const container of pod.containers) {
+                            add(used, parseQuantity(container.usage.cpu), parseQuantity(container.usage.memory));
+                        }
+                    }
+                    group.used = used;
+                }
+                catch (e) {
+                    once(why('what the pods use (metrics)', e, 'list on pods.metrics.k8s.io in them (runRole, see the backend docs)'));
+                }
+            }
+            groups.push(group);
+        }));
+        groups.sort((a, b) => a.name.localeCompare(b.name));
+        const total = { requested: {}, pods: 0 };
+        const used = {};
+        for (const group of groups) {
+            addAll(total.requested, group.requested);
+            total.pods = (total.pods ?? 0) + group.pods;
+            addAll(used, group.used);
+        }
+        if (groups.some((group) => group.used)) {
+            total.used = used;
+        }
+        let machines;
+        try {
+            const nodes = await this.instance.coreApi.listNode();
+            const allocatable = {};
+            for (const node of nodes.items) {
+                const of = node.status?.allocatable ?? {};
+                add(allocatable, parseQuantity(of.cpu), parseQuantity(of.memory));
+            }
+            machines = { count: nodes.items.length, allocatable };
+            if (this.instance.metrics) {
+                try {
+                    const metrics = await this.instance.metrics.getNodeMetrics();
+                    const nodesUsed = {};
+                    for (const node of metrics.items) {
+                        add(nodesUsed, parseQuantity(node.usage.cpu), parseQuantity(node.usage.memory));
+                    }
+                    machines.used = nodesUsed;
+                }
+                catch (e) {
+                    once(why('what the nodes use (metrics)', e, 'list on nodes.metrics.k8s.io (see the backend docs)'));
+                }
+            }
+        }
+        catch (e) {
+            // listing nodes is optional, and many clusters do not let a workload do it
+            if ((0, apply_1.statusCodeOf)(e) === 403) {
+                once('the nodes: not shown, the server may not list nodes (optional, see the backend docs)');
+            }
+            else {
+                once(why('the nodes', e));
+            }
+        }
+        return {
+            backend: 'kubernetes',
+            at: new Date().toISOString(),
+            ...(machines ? { machines } : {}),
+            runs: total,
+            groups,
+            problems,
+        };
+    }
+}
+exports.KubernetesCapacity = KubernetesCapacity;
+// On a host: the machine of the docker daemon, and the containers runs started on it.
+class HostCapacity {
+    constructor(docker, 
+    // whether the docker daemon runs on this machine, so its load is the server's
+    local = true) {
+        this.docker = docker;
+        this.local = local;
+    }
+    async read() {
+        const problems = [];
+        let machines;
+        try {
+            const info = (await this.docker.info());
+            machines = {
+                count: 1,
+                allocatable: { cpu: info.NCPU ?? (0, os_1.cpus)().length, memory: info.MemTotal ?? (0, os_1.totalmem)() },
+                ...(this.local ? { used: { memory: (0, os_1.totalmem)() - (0, os_1.freemem)() }, load: (0, os_1.loadavg)() } : {}),
+            };
+        }
+        catch (e) {
+            problems.push(why('the docker daemon', e));
+            machines = this.local
+                ? {
+                    count: 1,
+                    allocatable: { cpu: (0, os_1.cpus)().length, memory: (0, os_1.totalmem)() },
+                    used: { memory: (0, os_1.totalmem)() - (0, os_1.freemem)() },
+                    load: (0, os_1.loadavg)(),
+                }
+                : undefined;
+        }
+        const runs = { requested: {}, used: {}, containers: 0 };
+        try {
+            const containers = await this.docker.listContainers({ filters: { label: ['app=hammerkit'] } });
+            runs.containers = containers.length;
+            await Promise.all(containers.map(async (container) => {
+                try {
+                    const stats = (await this.docker.getContainer(container.Id).stats({ stream: false }));
+                    const cpu = stats.cpu_stats?.cpu_usage?.total_usage ?? 0;
+                    const before = stats.precpu_stats?.cpu_usage?.total_usage ?? 0;
+                    const system = (stats.cpu_stats?.system_cpu_usage ?? 0) - (stats.precpu_stats?.system_cpu_usage ?? 0);
+                    const online = stats.cpu_stats?.online_cpus ?? 1;
+                    const cores = system > 0 ? ((cpu - before) / system) * online : 0;
+                    const memory = (stats.memory_stats?.usage ?? 0) -
+                        (stats.memory_stats?.stats?.inactive_file ?? stats.memory_stats?.stats?.cache ?? 0);
+                    add(runs.used, Math.max(0, cores), Math.max(0, memory));
+                }
+                catch {
+                    // a container that just ended
+                }
+            }));
+        }
+        catch (e) {
+            problems.push(why('the containers of runs', e));
+            delete runs.used;
+        }
+        return {
+            backend: 'host',
+            at: new Date().toISOString(),
+            ...(machines ? { machines } : {}),
+            runs,
+            groups: [],
+            problems,
+        };
+    }
+}
+exports.HostCapacity = HostCapacity;
+var capacity_format_1 = require("./capacity-format");
+Object.defineProperty(exports, "formatCpu", { enumerable: true, get: function () { return capacity_format_1.formatCpu; } });
+Object.defineProperty(exports, "formatMemory", { enumerable: true, get: function () { return capacity_format_1.formatMemory; } });
+//# sourceMappingURL=capacity.js.map

package/dist/server/checks-api.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleChecks = exports.MAX_LOOKUP_KEYS = void 0;
+const zod_1 = require("zod");
+const http_1 = require("./http");
+const policy_1 = require("./policy");
+const checks_1 = require("./checks");
+const repo_1 = require("./repo");
+// the tasks one lookup may ask about
+exports.MAX_LOOKUP_KEYS = 100;
+const lookupSchema = zod_1.z
+    .object({
+    repo: zod_1.z.string().transform((value, ctx) => {
+        const repo = (0, repo_1.normalizeRepo)(value);
+        if (!repo) {
+            ctx.addIssue({ code: zod_1.z.ZodIssueCode.custom, message: 'expected a repository like github.com/corp/payments' });
+            return zod_1.z.NEVER;
+        }
+        return repo;
+    }),
+    keys: zod_1.z
+        .array(zod_1.z
+        .object({
+        task: zod_1.z.string().min(1).max(200),
+        key: zod_1.z.string().regex(checks_1.CHECK_KEY_PATTERN, 'expected a check key, 64 hex digits'),
+    })
+        .strict())
+        .min(1)
+        .max(exports.MAX_LOOKUP_KEYS),
+})
+    .strict();
+/**
+ * What runs passed, for CI that verifies instead of running (`hammerkit remote verify`):
+ *
+ *   POST /v1/checks/lookup   { repo, keys: [{ task, key }] }  the newest passed run per key
+ *
+ * The caller computed the keys from its own checkout (check-key.ts); a key nothing passed
+ * for, or whose version of the build file was yanked since, is answered with none.
+ */
+async function handleChecks(config, checks, definitions, log, principal, req, res, url) {
+    if (url.pathname !== '/v1/checks/lookup') {
+        return false;
+    }
+    if (req.method !== 'POST') {
+        (0, http_1.send)(res, 405, { error: 'method not allowed' }, { allow: 'POST' });
+        return true;
+    }
+    const parsed = lookupSchema.safeParse(await (0, http_1.readJson)(req));
+    if (!parsed.success) {
+        const issue = parsed.error.issues[0];
+        (0, http_1.send)(res, 400, { error: `${issue.path.join('.') || 'request'}: ${issue.message}` });
+        return true;
+    }
+    const { repo, keys } = parsed.data;
+    const grant = (0, policy_1.getGrant)(config.entitlements, principal);
+    if (!(0, policy_1.mayReadChecks)(grant, repo)) {
+        log(`refuse checks iss=${principal.iss} sub=${principal.sub} repo=${repo}`);
+        (0, http_1.send)(res, 403, { error: `you may not read the checks of ${repo}` });
+        return true;
+    }
+    const answers = await Promise.all(keys.map(async ({ task, key }) => {
+        const record = await checks.find(repo, key);
+        const yanked = record ? definitions.find(record.definition)?.yanked : undefined;
+        if (!record || yanked) {
+            return {
+                task,
+                key,
+                passed: null,
+                ...(yanked ? { reason: `it passed with ${record?.definition}, which was yanked since` } : {}),
+            };
+        }
+        const { repo: _repo, key: _key, accounts: _accounts, ...view } = record;
+        void _repo;
+        void _key;
+        void _accounts;
+        return { task, key, passed: view };
+    }));
+    log(`checks iss=${principal.iss} sub=${principal.sub} repo=${repo} passed=${answers.filter((a) => a.passed).length}/${answers.length}`);
+    (0, http_1.send)(res, 200, { checks: answers });
+    return true;
+}
+exports.handleChecks = handleChecks;
+//# sourceMappingURL=checks-api.js.map

package/dist/server/checks.js

@@ -0,0 +1,116 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CheckStore = exports.getCheckRecords = exports.isCheckRun = exports.CHECK_KEY_PATTERN = void 0;
+const crypto_1 = require("crypto");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+exports.CHECK_KEY_PATTERN = /^[0-9a-f]{64}$/;
+// The runs whose results are checks: those of a commit the server checked out, and those
+// on a copy of the files of a sandbox, which hold nothing a commit of them would not. A run
+// with arguments ran something other than the task, and is no check of it.
+function isCheckRun(run) {
+    // a run of services keeps them up, it passes nothing
+    if ((run.options.args ?? []).length > 0 || run.up) {
+        return false;
+    }
+    return run.source.type === 'ref' || (run.source.type === 'sandbox' && run.source.check === true);
+}
+exports.isCheckRun = isCheckRun;
+// What a run that ended passed, with the keys its executor reported before the tasks ran.
+function getCheckRecords(run, keys) {
+    if (!isCheckRun(run) || run.source.type === 'upload') {
+        return [];
+    }
+    const source = run.source.type === 'ref'
+        ? { type: 'ref', ref: run.source.ref, commit: run.source.commit }
+        : { type: 'sandbox', sandbox: run.source.id, commit: run.source.commit, tree: run.source.tree };
+    const records = [];
+    for (const [task, result] of Object.entries(run.results ?? {})) {
+        const key = keys[task];
+        if (!key || !exports.CHECK_KEY_PATTERN.test(key) || (result.status !== 'succeeded' && result.status !== 'cached')) {
+            continue;
+        }
+        records.push({
+            repo: run.source.repo,
+            key,
+            task,
+            status: result.status,
+            run: run.id,
+            by: { iss: run.owner.iss, sub: run.owner.sub },
+            source,
+            definition: run.definition,
+            accounts: [...run.accounts],
+            finishedAt: result.finishedAt ?? run.finishedAt ?? new Date().toISOString(),
+            ...(result.durationMs !== undefined ? { durationMs: result.durationMs } : {}),
+        });
+    }
+    return records;
+}
+exports.getCheckRecords = getCheckRecords;
+/**
+ * One file per repository and key, the newest run's: `<directory>/<repo hash>/<key>.json`,
+ * written to a temporary file and renamed. A record older than `keep` is not found and is
+ * removed by `sweep`.
+ */
+class CheckStore {
+    constructor(directory, options) {
+        this.directory = directory;
+        this.options = options;
+        this.now = options.now ?? Date.now;
+    }
+    async save(records) {
+        for (const record of records) {
+            const folder = (0, path_1.join)(this.directory, repoFolder(record.repo));
+            await (0, promises_1.mkdir)(folder, { recursive: true, mode: 0o700 });
+            const file = (0, path_1.join)(folder, `${record.key}.json`);
+            const temporary = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
+            await (0, promises_1.writeFile)(temporary, JSON.stringify(record), { mode: 0o600 });
+            await (0, promises_1.rename)(temporary, file);
+        }
+    }
+    async find(repo, key) {
+        if (!exports.CHECK_KEY_PATTERN.test(key)) {
+            return undefined;
+        }
+        let record;
+        try {
+            record = JSON.parse(await (0, promises_1.readFile)((0, path_1.join)(this.directory, repoFolder(repo), `${key}.json`), 'utf8'));
+        }
+        catch {
+            return undefined;
+        }
+        if (record.repo !== repo || record.key !== key || this.expired(record)) {
+            return undefined;
+        }
+        return record;
+    }
+    // Removes the records that are older than `keep`.
+    async sweep() {
+        let folders;
+        try {
+            folders = await (0, promises_1.readdir)(this.directory);
+        }
+        catch {
+            return;
+        }
+        for (const folder of folders) {
+            const path = (0, path_1.join)(this.directory, folder);
+            for (const name of await (0, promises_1.readdir)(path).catch(() => [])) {
+                const file = (0, path_1.join)(path, name);
+                const stats = await (0, promises_1.stat)(file).catch(() => undefined);
+                if (stats && this.now() - stats.mtimeMs > this.options.keep) {
+                    await (0, promises_1.rm)(file, { force: true });
+                }
+            }
+        }
+    }
+    expired(record) {
+        const at = Date.parse(record.finishedAt);
+        return !Number.isFinite(at) || this.now() - at > this.options.keep;
+    }
+}
+exports.CheckStore = CheckStore;
+function repoFolder(repo) {
+    return (0, crypto_1.createHash)('sha256').update(repo).digest('hex').slice(0, 32);
+}
+//# sourceMappingURL=checks.js.map