@yunuseon/kayra 0.7.1 → 0.8.1

package/cli/dist/deployer/builder.js

@@ -0,0 +1,196 @@
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { createServer } from "node:http";
+import { join } from "node:path";
+import { err, ok } from "../result.js";
+import { field, registered } from "./app.js";
+import { owner, runId, sha } from "./brand.js";
+import { body, json, send } from "./http.js";
+import { page, runPage } from "./page.js";
+import { spawnLines } from "./process.js";
+import { pushedRef, signed } from "./webhook.js";
+const fromFile = (path) => () => (existsSync(path) ? readFileSync(path, "utf8").trim() : "");
+const settings = (env) => {
+    const platformDir = env["PLATFORM_DIR"] ?? "/opt/platform";
+    return {
+        platformDir,
+        work: env["WORK_DIR"] ?? "/work",
+        deployer: env["DEPLOYER_URL"] ?? "http://deployer:8090",
+        resolveImages: env["RESOLVE_IMAGES"] ?? "/opt/kayra/scripts/platform/resolve-images.sh",
+        webhookSecret: () => env["WEBHOOK_SECRET"] ?? "",
+        gitToken: fromFile("/run/platform-secrets/GIT_TOKEN"),
+    };
+};
+/** git with the platform's token as a header: neither in the clone's config nor in the process list. */
+const gitEnv = (token) => ({
+    GIT_TERMINAL_PROMPT: "0",
+    GIT_CONFIG_COUNT: "1",
+    GIT_CONFIG_KEY_0: "http.extraHeader",
+    GIT_CONFIG_VALUE_0: `Authorization: Basic ${Buffer.from(`x-access-token:${token}`).toString("base64")}`,
+});
+/** A full commit id as it is; otherwise the branch or tag it names (a tag's commit, not the tag object). */
+export const resolveRef = async (app, ref, token) => {
+    const given = sha(ref);
+    if (given.ok)
+        return given;
+    const exit = await spawnLines(["git", "ls-remote", app.repo, `refs/heads/${ref}`, `refs/tags/${ref}`, `refs/tags/${ref}^{}`], { env: gitEnv(token), capture: true, timeoutMs: 60_000 }, () => undefined);
+    if (exit.code !== 0)
+        return err(`git ls-remote ${app.repo} failed (exit ${String(exit.code)})`);
+    const refs = new Map(exit.stdout.split("\n").flatMap((line) => {
+        const [commit, name] = line.split("\t");
+        return commit === undefined || name === undefined ? [] : [[name, commit]];
+    }));
+    const found = refs.get(`refs/tags/${ref}^{}`) ?? refs.get(`refs/heads/${ref}`) ?? refs.get(`refs/tags/${ref}`);
+    return found === undefined ? err(`${ref} is no branch or tag of ${app.repo}`) : sha(found);
+};
+const run = async (argv, options, line) => {
+    const exit = await spawnLines(argv, options, line);
+    return exit.code === 0 ? ok(exit.stdout) : err(`${argv.slice(0, 3).join(" ")} failed (exit ${String(exit.code)})`);
+};
+const sequence = async (steps) => steps.reduce(async (previous, next) => {
+    const so = await previous;
+    return so.ok ? next() : so;
+}, Promise.resolve(ok("")));
+/**
+ * One app's checkout at a commit, its images built where the registry lacks
+ * them (resolve-images.sh, through buildkitd), and the compose files app.yml
+ * lists at that commit. A clone per app, kept between runs; the deployer never
+ * runs two for the same app at once, and neither does this.
+ */
+export const prepare = async (app, commit, s, line) => {
+    const dir = join(s.work, app.name);
+    const env = gitEnv(s.gitToken());
+    const git = (...args) => run(["git", "-C", dir, ...args], { env, timeoutMs: 10 * 60_000 }, line);
+    mkdirSync(dir, { recursive: true });
+    const fresh = !existsSync(join(dir, ".git"));
+    const checkedOut = await sequence([
+        () => (fresh ? run(["git", "init", "-q", dir], {}, line) : Promise.resolve(ok(""))),
+        () => (fresh ? git("remote", "add", "origin", app.repo) : git("remote", "set-url", "origin", app.repo)),
+        () => git("fetch", "--quiet", "--prune", "origin", "+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*"),
+        async () => {
+            const there = await git("checkout", "--quiet", "--force", "--detach", commit);
+            return there.ok ? there : sequence([() => git("fetch", "--quiet", "origin", commit), () => git("checkout", "--quiet", "--force", "--detach", commit)]);
+        },
+        () => git("clean", "-ffdxq"),
+    ]);
+    if (!checkedOut.ok)
+        return checkedOut;
+    const root = join(dir, app.path);
+    const buildEnv = { REGISTRY: "localhost:5000" };
+    const planned = await run(["bash", s.resolveImages, app.name, root, "--plan"], { env: buildEnv, capture: true }, line);
+    if (!planned.ok)
+        return planned;
+    const artifacts = planned.value
+        .split("\n")
+        .filter((row) => row !== "")
+        .map((row) => row.split("\t"));
+    const built = await sequence(artifacts.map(([name]) => () => {
+        line(`── Image ${name ?? "?"}`);
+        return run(["bash", s.resolveImages, app.name, root, "--ensure", name ?? ""], { env: buildEnv }, line);
+    }));
+    if (!built.ok)
+        return built;
+    const listed = await run(["yq", "-o=json", ".compose // []", join(root, "app.yml")], { capture: true }, line);
+    if (!listed.ok)
+        return listed;
+    const files = JSON.parse(listed.value);
+    if (!Array.isArray(files) || !files.every((file) => typeof file === "string"))
+        return err("app.yml: compose is not a list of paths");
+    const outside = files.find((file) => file.startsWith("/") || file.split("/").includes(".."));
+    if (outside !== undefined)
+        return err(`app.yml: compose file ${outside} is outside the app`);
+    const missing = files.find((file) => !existsSync(join(root, file)));
+    if (missing !== undefined)
+        return err(`app.yml: compose file ${missing} does not exist at ${commit}`);
+    return ok({
+        sha: commit,
+        images: artifacts.map(([, variable, ref]) => ({ variable, ref })),
+        compose: files.map((file) => ({ path: file, content: readFileSync(join(root, file), "utf8") })),
+    });
+};
+const hook = async (s, name, request, response) => {
+    const raw = await body(request);
+    if (raw === undefined || !signed(s.webhookSecret(), raw, request.headers)) {
+        send(response, 401, "bad signature\n");
+        return;
+    }
+    const app = registered(s.platformDir, name);
+    if (!app.ok) {
+        send(response, 404, `${app.error}\n`);
+        return;
+    }
+    const ref = pushedRef(raw);
+    if (ref !== `refs/heads/${app.value.ref}`) {
+        send(response, 202, `ignored: ${ref ?? "not a push"} (deploys follow ${app.value.ref})\n`);
+        return;
+    }
+    const forwarded = await fetch(`${s.deployer}/push/${app.value.name}`, { method: "POST" }).catch(() => undefined);
+    send(response, forwarded?.ok === true ? 202 : 502, forwarded?.ok === true ? "queued\n" : "the deployer did not answer\n");
+};
+const pages = (s, path, response) => {
+    const [, runs, who, id] = path.split("/");
+    if (path === "/") {
+        send(response, 200, page(join(s.platformDir, "runs")), "text/html; charset=utf-8");
+        return;
+    }
+    const minted = owner(who ?? "");
+    const run = runId(id ?? "");
+    if (runs === "runs" && minted.ok && run.ok) {
+        send(response, 200, runPage(join(s.platformDir, "runs"), minted.value, run.value), "text/html; charset=utf-8");
+        return;
+    }
+    send(response, 404, "not found\n");
+};
+const internal = async (s, path, request, response) => {
+    const input = await json(request);
+    const name = typeof input === "object" && input !== null ? field(input, "app") : undefined;
+    const app = registered(s.platformDir, typeof name === "string" ? name : "");
+    if (!app.ok || typeof input !== "object" || input === null) {
+        send(response, 404, `${app.ok ? "bad request" : app.error}\n`);
+        return;
+    }
+    if (path === "/resolve") {
+        const ref = field(input, "ref");
+        const resolved = await resolveRef(app.value, typeof ref === "string" ? ref : app.value.ref, s.gitToken());
+        send(response, resolved.ok ? 200 : 422, JSON.stringify(resolved.ok ? { sha: resolved.value } : { error: resolved.error }), "application/json");
+        return;
+    }
+    const commit = sha(String(field(input, "sha")));
+    if (path !== "/prepare" || !commit.ok) {
+        send(response, 404, "not found\n");
+        return;
+    }
+    // One JSON object per line: {log} as the build goes, then {bundle} or {error}.
+    response.writeHead(200, { "Content-Type": "application/x-ndjson" });
+    const prepared = await prepare(app.value, commit.value, s, (text) => response.write(`${JSON.stringify({ log: text })}\n`));
+    response.end(`${JSON.stringify(prepared.ok ? { bundle: prepared.value } : { error: prepared.error })}\n`);
+};
+const failing = (handler) => (request, response) => {
+    Promise.resolve(handler(request, response)).catch((error) => {
+        console.error(error);
+        if (!response.headersSent)
+            send(response, 500, "internal error\n");
+        else
+            response.end();
+    });
+};
+export const serveBuilder = (env) => {
+    const s = settings(env);
+    createServer(failing(async (request, response) => {
+        const path = new URL(request.url ?? "/", "http://builder").pathname;
+        const hooked = /^\/hook\/([^/]+)\/?$/.exec(path);
+        if (request.method === "POST" && hooked?.[1] !== undefined)
+            await hook(s, hooked[1], request, response);
+        else if (request.method === "GET")
+            pages(s, path, response);
+        else
+            send(response, 405, "method not allowed\n");
+    })).listen(8080);
+    createServer(failing(async (request, response) => {
+        const path = new URL(request.url ?? "/", "http://builder").pathname;
+        if (request.method === "POST")
+            await internal(s, path, request, response);
+        else
+            send(response, 405, "method not allowed\n");
+    })).listen(8081);
+    console.log("builder: :8080 hooks and pages, :8081 resolve and prepare");
+};

package/cli/dist/deployer/bundle.js

@@ -0,0 +1,49 @@
+import { err, ok } from "../result.js";
+import { field } from "./app.js";
+import { sha } from "./brand.js";
+const relative = (path) => path !== "" && !path.startsWith("/") && !path.split("/").some((segment) => segment === ".." || segment === "");
+export const parseBundle = (app, expected, json) => {
+    if (typeof json !== "object" || json === null)
+        return err("bundle: not an object");
+    const commit = field(json, "sha");
+    const images = field(json, "images");
+    const compose = field(json, "compose");
+    const minted = sha(typeof commit === "string" ? commit : "");
+    if (!minted.ok || minted.value !== expected)
+        return err(`bundle: for ${String(commit)}, not ${expected}`);
+    if (!Array.isArray(images) || !Array.isArray(compose) || compose.length === 0)
+        return err("bundle: images or compose missing");
+    const own = new RegExp(`^localhost:5000/${app}/[a-z0-9._/-]+:[0-9a-f]{12}$`);
+    const parsedImages = images.map((image) => {
+        if (typeof image !== "object" || image === null)
+            return "an image is not an object";
+        const variable = field(image, "variable");
+        const ref = field(image, "ref");
+        if (typeof variable !== "string" || !/^[A-Z][A-Z0-9_]*_IMAGE$/.test(variable))
+            return `image variable ${String(variable)}`;
+        if (typeof ref !== "string" || !own.test(ref))
+            return `image ${String(ref)} is outside ${app}/`;
+        return { variable, ref };
+    });
+    const parsedCompose = compose.map((file) => {
+        if (typeof file !== "object" || file === null)
+            return "a compose file is not an object";
+        const path = field(file, "path");
+        const content = field(file, "content");
+        if (typeof path !== "string" || !relative(path))
+            return `compose file ${String(path)}`;
+        if (typeof content !== "string")
+            return `compose file ${path} has no content`;
+        return { path, content };
+    });
+    const problem = [...parsedImages, ...parsedCompose].find((item) => typeof item === "string");
+    if (problem !== undefined)
+        return err(`bundle: ${problem}`);
+    return ok({
+        sha: minted.value,
+        images: parsedImages.filter((item) => typeof item !== "string"),
+        compose: parsedCompose.filter((item) => typeof item !== "string"),
+    });
+};
+/** images.env, as compose reads it next to the app's own .env. */
+export const imagesEnv = (bundle) => bundle.images.map((image) => `${image.variable}=${image.ref}\n`).join("");

package/cli/dist/deployer/compose.js

@@ -0,0 +1,26 @@
+/**
+ * The app's compose project, run in the run's directory: its .env from the
+ * platform (operator config), the image refs from this run, the compose files
+ * from the commit being deployed. Never COMPOSE_PROJECT_NAME from the
+ * environment: -p names the project.
+ */
+const project = (app, appsDir, bundle) => {
+    const dir = `${appsDir}/${app.name}`;
+    return [
+        "compose", "-p", app.name, "--project-directory", dir, "--env-file", `${dir}/.env`, "--env-file", "images.env",
+        ...bundle.compose.flatMap((file) => ["-f", file.path]),
+    ];
+};
+/**
+ * What admission checks. --no-env-resolution: otherwise compose reads every
+ * env_file here, from any path (another app's secrets included), and bakes it
+ * into the config; unresolved, the allow-list sees env_file and refuses it.
+ */
+export const config = (app, appsDir, bundle) => [
+    "docker", ...project(app, appsDir, bundle), "config", "--no-env-resolution", "--format", "json",
+];
+export const policy = (app, script) => ["bash", script, app.name, ...app.hosts];
+/** --remove-orphans: a service the app dropped from its compose files must not keep running. */
+export const up = (app, appsDir, bundle) => [
+    "docker", ...project(app, appsDir, bundle), "up", "-d", "--wait", "--wait-timeout", "180", "--remove-orphans",
+];

package/cli/dist/deployer/cron.js

@@ -0,0 +1,44 @@
+import { all, err, map, ok } from "../result.js";
+const range = (from, to, step) => Array.from({ length: Math.floor((to - from) / step) + 1 }, (_, i) => from + i * step);
+const part = (text, min, max) => {
+    const match = /^(\*|(\d+)(?:-(\d+))?)(?:\/(\d+))?$/.exec(text);
+    if (match === null)
+        return err(`'${text}'`);
+    const [, whole, low, high, every] = match;
+    const step = every === undefined ? 1 : Number(every);
+    const from = whole === "*" ? min : Number(low);
+    const to = whole === "*" ? max : high === undefined ? (every === undefined ? from : max) : Number(high);
+    return from < min || to > max || from > to || step < 1 ? err(`'${text}' (${String(min)}-${String(max)})`) : ok(range(from, to, step));
+};
+const values = (text, min, max) => map(all(text.split(",").map((piece) => part(piece, min, max))), (parts) => new Set(parts.flat()));
+export const parseCron = (spec) => {
+    const fields = spec.trim().split(/\s+/);
+    const [minute, hour, day, month, weekday] = fields;
+    if (fields.length !== 5 || minute === undefined || hour === undefined || day === undefined || month === undefined || weekday === undefined) {
+        return err(`'${spec}' is not a five-field cron spec`);
+    }
+    const parsed = all([values(minute, 0, 59), values(hour, 0, 23), values(day, 1, 31), values(month, 1, 12), values(weekday, 0, 7)]);
+    if (!parsed.ok)
+        return err(`'${spec}': ${parsed.error}`);
+    const [minutes, hours, days, months, weekdays] = parsed.value;
+    if (minutes === undefined || hours === undefined || days === undefined || months === undefined || weekdays === undefined) {
+        return err(`'${spec}' is not a five-field cron spec`);
+    }
+    return ok({
+        minute: minutes,
+        hour: hours,
+        day: days,
+        month: months,
+        // 7 is Sunday too.
+        weekday: new Set([...weekdays].map((d) => d % 7)),
+        eitherDay: day !== "*" && weekday !== "*",
+    });
+};
+export const matches = (cron, at) => {
+    const day = cron.day.has(at.getDate());
+    const weekday = cron.weekday.has(at.getDay());
+    return (cron.minute.has(at.getMinutes()) &&
+        cron.hour.has(at.getHours()) &&
+        cron.month.has(at.getMonth() + 1) &&
+        (cron.eitherDay ? day || weekday : day && weekday));
+};

package/cli/dist/deployer/ctl.js

@@ -0,0 +1,77 @@
+import { existsSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { appOwner, appName, owner as mintOwner, runId } from "./brand.js";
+import { releases } from "./records.js";
+import { log, runsOf } from "./store.js";
+/**
+ * The operator's side, run inside the deployer container by `kayra app …` and
+ * `kayra job …` over SSH. Reads runs/ and releases/ directly; anything that
+ * starts a run goes through the daemon's control port, so the queue stays the
+ * only place runs begin.
+ */
+const platformDir = process.env["PLATFORM_DIR"] ?? "/opt/platform";
+const runs = join(platformDir, "runs");
+const fail = (message) => {
+    console.error(message);
+    return 1;
+};
+const post = async (path, payload) => {
+    const response = await fetch(`http://127.0.0.1:8091${path}`, { method: "POST", body: JSON.stringify(payload) }).catch(() => undefined);
+    if (response === undefined)
+        return fail("the deployer is not running");
+    process.stdout.write(await response.text());
+    return response.ok ? 0 : 1;
+};
+const releasesOf = (app) => {
+    const dir = join(platformDir, "releases", app);
+    return releases(existsSync(dir) ? readdirSync(dir) : []);
+};
+const commands = {
+    runs: ([app]) => {
+        const name = appName(app ?? "");
+        if (!name.ok)
+            return fail(name.error);
+        runsOf(runs, appOwner(name.value)).slice(0, 20).forEach((run) => {
+            console.log([run.id, run.state.padEnd(8), (run.sha ?? "").slice(0, 12).padEnd(12), run.why.padEnd(8), run.what].join("  "));
+        });
+        return 0;
+    },
+    log: ([who, id]) => {
+        const minted = mintOwner(who === "gc" || who === "renovate" ? `_${who}` : (who ?? ""));
+        if (!minted.ok)
+            return fail(minted.error);
+        const given = id === undefined ? undefined : runId(id);
+        const run = given === undefined ? runsOf(runs, minted.value)[0]?.id : given.ok ? given.value : undefined;
+        const text = run === undefined ? undefined : log(runs, minted.value, run);
+        if (text === undefined)
+            return fail(`no such run of ${minted.value}`);
+        process.stdout.write(text);
+        return 0;
+    },
+    deploy: ([app, ref]) => post("/deploy", { app, ref: ref ?? "", why: "deploy" }),
+    rollback: ([app, commit]) => {
+        const name = appName(app ?? "");
+        if (!name.ok)
+            return fail(name.error);
+        const [live, ...earlier] = releasesOf(name.value);
+        if (live === undefined)
+            return fail(`${name.value} has not been deployed yet`);
+        if (commit === undefined) {
+            console.log(`${name.value} runs ${live.sha} (since ${live.time}). Earlier releases:`);
+            earlier.forEach((release) => {
+                console.log(`  ${release.sha}  ${release.time}`);
+            });
+            return earlier.length === 0 ? fail("nothing to roll back to") : 0;
+        }
+        const target = earlier.find((release) => release.sha.startsWith(commit));
+        if (target === undefined || commit.length < 7)
+            return fail(`${commit} is not an earlier release of ${name.value}`);
+        return post("/deploy", { app: name.value, ref: target.sha, why: "rollback" });
+    },
+    job: ([job, flag]) => post("/job", { job, dryRun: flag === "--dry-run" }),
+};
+export const ctl = async (args) => {
+    const [name, ...rest] = args;
+    const command = name === undefined ? undefined : commands[name];
+    return command === undefined ? fail(`usage: ctl ${Object.keys(commands).join("|")} …`) : command(rest);
+};