@yunuseon/kayra 0.9.1 → 0.11.0

package/ansible/roles/platform/defaults/main.yml

@@ -27,6 +27,10 @@ acme_dns_token: ""
 git_token: ""
 # The secret forges sign webhooks with (deploy.<domain>/hook/<app>).
 webhook_secret: ""
+# Where the deployer raises alerts: {} for nowhere, or {kind: github-issue,
+# repository: owner/name, app_id: …} with notifier_github_app_key in secrets.yml.
+notifier: {}
+notifier_github_app_key: ""
 
 # owner/repo of the repos this server's Renovate job keeps up to date
 # (weekly version-bump PRs); the job runs only where the list is not empty.
@@ -39,8 +43,9 @@ renovate_token: ""
 # people's releases; a server whose repos carry the operator's own images
 # runs it more often, so a new build reaches a PR soon after it is pushed.
 renovate_schedule: "0 4 * * 1"
-# Where those repos live: github, or forgejo with renovate_endpoint, its API
-# on platform-internal (http://forgejo:3000/api/v1). With a Forgejo, also
+# Where those repos live: github, gitlab (gitlab.com), or forgejo with
+# renovate_endpoint, its API on platform-internal
+# (http://forgejo:3000/api/v1). With a Forgejo, also
 # renovate_git_url: endpoint, so Renovate clones over that address and not the
 # public one the API reports, and a renovate_host_rules entry allowing it as
 # an internal host.

package/ansible/roles/platform/tasks/main.yml

@@ -14,6 +14,13 @@
     fail_msg: renovate_repositories is set, so renovate_token must be set in servers/{{ server }}/secrets.yml
   when: renovate_repositories | length > 0
 
+- name: Check the notifier has its key
+  ansible.builtin.assert:
+    that: notifier_github_app_key | length > 0
+    fail_msg: notifier is a github-issue notifier, so notifier_github_app_key must be set in servers/{{ server }}/secrets.yml
+    quiet: true
+  when: notifier.kind | default('') == 'github-issue'
+
 - name: Create platform directories
   ansible.builtin.file:
     path: "{{ item }}"
@@ -50,7 +57,7 @@
     owner: "1000"
     group: "1000"
     mode: "0755"
-  loop: [runs, releases]
+  loop: [runs, releases, alerts]
   become: true
   register: deployer_dirs
 
@@ -276,6 +283,27 @@
   become: true
   register: platform_tokens
 
+# Only the deployer mounts this: the builder faces the internet.
+- name: Create the deployer's secrets directory
+  ansible.builtin.file:
+    path: "{{ platform_dir }}/deployer-secrets"
+    state: directory
+    owner: "1000"
+    group: "1000"
+    mode: "0500"
+  become: true
+
+- name: Render the notifier's key
+  ansible.builtin.copy:
+    content: "{{ notifier_github_app_key }}"
+    dest: "{{ platform_dir }}/deployer-secrets/NOTIFIER_GITHUB_APP_KEY"
+    owner: "1000"
+    group: "1000"
+    mode: "0400"
+  when: notifier_github_app_key | length > 0
+  no_log: true
+  become: true
+
 - name: Render platform .env
   ansible.builtin.template:
     src: platform.env.j2

package/ansible/roles/platform/templates/platform.env.j2

@@ -23,6 +23,8 @@ RENOVATE_HOST_RULES={{ renovate_host_rules | to_json if renovate_host_rules else
 RENOVATE_REPOSITORIES={{ renovate_repositories | to_json }}
 
 WEBHOOK_SECRET={{ webhook_secret }}
+SERVER={{ server }}
+NOTIFIER={{ notifier | to_json if notifier else '' }}
 
 # Authelia SMTP. Empty address falls the notifier back to a file — passkey
 # enrolment then needs the code read off the server. The password is NOT here:

package/cli/dist/cli.js

@@ -14,7 +14,7 @@ export const parser = (fleet) => {
         dryRun: option("--dry-run", { description: message `Print the plan instead of running it.` }),
         command: or(
         // On one server
-        or(onServer("up", message `Provision, set up everything, point /etc/hosts at it.`), onServer("provision", message `Create or update the machine (target home: build the image to import).`), onServer("deploy", message `Host setup and every app, no provisioning.`), onServer("setup", message `The host layer only: the playbook for the server's kind.`), onServer("plan", message `What provision and setup would change, changing nothing (OpenTofu plan, Ansible check mode).`), onServer("apps", message `Make the server's apps match server.yml: register each, remove the rest (data kept).`), onServer("hosts", message `Point the domain at a multipass VM in /etc/hosts (sudo).`), command("ssh", object({ name: constant("ssh"), server, remote: multiple(argument(string({ metavar: "COMMAND" }))) }), { brief: message `SSH into the server, or run a command there (after --).` }), onServer("down", message `Destroy the machine.`), onServer("adopt", message `Once, for a server we did not create: add the deploy user as root.`), command("backup", command("test", object({ name: constant("backup test"), server })), {
+        or(onServer("up", message `Provision, set up everything, point /etc/hosts at it.`), onServer("provision", message `Create or update the machine (target home: build the image to import).`), onServer("deploy", message `Host setup and every app, no provisioning.`), onServer("setup", message `The host layer only: the playbook for the server's kind.`), onServer("plan", message `What provision and setup would change, changing nothing (OpenTofu plan, Ansible check mode).`), onServer("verify", message `Whether a platform server works as its users see it: logins, certificates, webhooks, apps, runs, alerts.`), onServer("apps", message `Make the server's apps match server.yml: register each, remove the rest (data kept).`), onServer("hosts", message `Point the domain at a multipass VM in /etc/hosts (sudo).`), command("ssh", object({ name: constant("ssh"), server, remote: multiple(argument(string({ metavar: "COMMAND" }))) }), { brief: message `SSH into the server, or run a command there (after --).` }), onServer("down", message `Destroy the machine.`), onServer("adopt", message `Once, for a server we did not create: add the deploy user as root.`), command("backup", command("test", object({ name: constant("backup test"), server })), {
             brief: message `Restore the latest backup to a scratch dir, check it, delete it.`,
         }), command("app", or(command("check", object({ name: constant("app check"), dir: optional(argument(string({ metavar: "DIR" }))) }), {
             brief: message `In an app's repository: what the platform checks at deploy (app.yml, images, compose policy). No fleet needed.`,

package/cli/dist/commands/server.js

@@ -148,7 +148,13 @@ export const hosts = (engine, server) => ok(server.target === "multipass"
     ]
     : [{ kind: "note", text: `hosts: ${server.target} uses real DNS, nothing to do.` }]);
 const sequence = (plans) => map(all(plans), (each) => each.flat());
-export const deploy = (engine, server) => sequence([setup(engine, server), apps(engine, server)]);
+/** Whether a platform server works as its users see it (scripts/local/verify.sh); other kinds have nothing to check yet. */
+const verifyStep = (engine, server) => run(["bash", script(engine, "verify.sh"), server.name], fleetEnv(engine));
+export const verify = (engine, server) => server.kind === "platform"
+    ? ok([verifyStep(engine, server)])
+    : err(`'${server.name}' is of kind ${server.kind}: kayra verify checks platform servers only.`);
+/** Host setup, then the apps; a platform server is verified at the end, so a deploy that leaves it broken fails. */
+export const deploy = (engine, server) => map(sequence([setup(engine, server), apps(engine, server)]), (plan) => server.kind === "platform" ? [...plan, verifyStep(engine, server)] : plan);
 export const up = (engine, server) => sequence([provision(engine, server), deploy(engine, server), hosts(engine, server)]);
 export const down = (engine, server) => server.terraform
     ? ok([

package/cli/dist/deployer/builder.js

@@ -7,7 +7,8 @@ import { owner, runId, sha } from "./brand.js";
 import { body, json, send } from "./http.js";
 import { page, runPage } from "./page.js";
 import { spawnLines } from "./process.js";
-import { pushedRef, signed } from "./webhook.js";
+import { forgeRepo, signed } from "./forge.js";
+import { pushedRef } from "./webhook.js";
 const fromFile = (path) => () => (existsSync(path) ? readFileSync(path, "utf8").trim() : "");
 const settings = (env) => {
     const platformDir = env["PLATFORM_DIR"] ?? "/opt/platform";
@@ -109,15 +110,16 @@ export const prepare = async (app, commit, s, line) => {
 };
 const hook = async (s, name, request, response) => {
     const raw = await body(request);
-    if (raw === undefined || !signed(s.webhookSecret(), raw, request.headers)) {
-        send(response, 401, "bad signature\n");
-        return;
-    }
     const app = registered(s.platformDir, name);
     if (!app.ok) {
         send(response, 404, `${app.error}\n`);
         return;
     }
+    const repo = forgeRepo(app.value.repo);
+    if (raw === undefined || repo === undefined || !signed(repo, s.webhookSecret(), raw, request.headers)) {
+        send(response, 401, "bad signature\n");
+        return;
+    }
     const ref = pushedRef(raw);
     if (ref !== `refs/heads/${app.value.ref}`) {
         send(response, 202, `ignored: ${ref ?? "not a push"} (deploys follow ${app.value.ref})\n`);

package/cli/dist/deployer/deployer.js

@@ -9,11 +9,13 @@ import { all, err, ok } from "../result.js";
 import { field, registered } from "./app.js";
 import { appName, appOwner, jobOwner, newRunId, sha } from "./brand.js";
 import { matches, parseCron } from "./cron.js";
-import { statusRequest } from "./forge.js";
+import { forge, forgeRepo } from "./forge.js";
 import { json, send } from "./http.js";
 import { spawnLines } from "./process.js";
 import { empty, enqueue, finish } from "./queue.js";
 import { expired, moved, releaseName, releases } from "./records.js";
+import { githubIssues } from "./github-issues.js";
+import { decide, parseNotify, silent } from "./notify.js";
 import { deploy, renovate } from "./run.js";
 import { runsOf } from "./store.js";
 const token = (name) => {
@@ -40,8 +42,48 @@ const settings = (env) => {
         gc: must(parseCron("0 3 * * *")),
         poll: must(parseCron("17 * * * *")),
         env: strings,
+        server: env["SERVER"] ?? env["DOMAIN"] ?? "this server",
+        notifier: notifierFor(must(parseNotify(env["NOTIFIER"])), env["PLATFORM_DIR"] ?? "/opt/platform"),
     };
 };
+/** The configured notifier; its open alerts are kept in <platformDir>/alerts/<key>. */
+const notifierFor = (config, platformDir) => {
+    if (config.kind === "none")
+        return silent;
+    const alerts = join(platformDir, "alerts");
+    const keyFile = "/run/deployer-secrets/NOTIFIER_GITHUB_APP_KEY";
+    return githubIssues(config.repository, config.appId, {
+        fetch: async (url, init) => {
+            const response = await fetch(url, init);
+            return { status: response.status, json: await response.json().catch(() => undefined) };
+        },
+        privateKey: () => (existsSync(keyFile) ? ok(readFileSync(keyFile, "utf8")) : err(`${keyFile} is missing: notifier_github_app_key in secrets.yml`)),
+        now: () => Date.now(),
+        openIssue: (key) => {
+            const path = join(alerts, key);
+            const number = existsSync(path) ? Number(readFileSync(path, "utf8")) : Number.NaN;
+            return Number.isInteger(number) ? number : undefined;
+        },
+        remember: (key, issue) => {
+            mkdirSync(alerts, { recursive: true });
+            if (issue === undefined)
+                rmSync(join(alerts, key), { force: true });
+            else
+                writeFileSync(join(alerts, key), String(issue));
+        },
+    });
+};
+/** Raises or resolves what the finished run means; a notifier's failure is logged, never the run's. */
+const notify = async (s, end, log) => {
+    const decision = decide(end);
+    const done = decision.kind === "raise"
+        ? await s.notifier.raise(decision.alert)
+        : decision.kind === "resolve"
+            ? await s.notifier.resolve(decision.key, decision.note)
+            : ok(undefined);
+    if (!done.ok)
+        log(`(alert not sent: ${done.error})`);
+};
 const runsDir = (s) => join(s.platformDir, "runs");
 const keepRuns = 50;
 const registeredApps = (s) => {
@@ -119,14 +161,14 @@ const effects = (s, who, id, log) => ({
         return Promise.resolve();
     },
     status: async (app, commit, context, state, description) => {
-        const request = statusRequest(app.repo, commit, { context, state, description, target: target(s, who, id) }, token("GIT_TOKEN"));
-        if (request === undefined) {
+        const repo = forgeRepo(app.repo);
+        if (repo === undefined) {
             log(`(no commit status: ${app.repo} is on no forge kayra knows)`);
             return;
         }
-        const response = await fetch(request.url, { method: "POST", headers: request.headers, body: request.body }).catch(() => undefined);
-        if (response?.ok !== true)
-            log(`(commit status ${context} not posted: ${response === undefined ? "no answer" : `HTTP ${String(response.status)}`})`);
+        const posted = await forge(repo, token("GIT_TOKEN"), fetch).postStatus(commit, { context, state, description, target: target(s, who, id) });
+        if (!posted.ok)
+            log(`(commit status ${context} not posted: ${posted.error})`);
     },
 });
 const describe = (request) => request.work.kind === "deploy"
@@ -180,6 +222,7 @@ const start = async (s, request) => {
         finished: new Date().toISOString(),
     };
     writeFileSync(join(dir, "meta.json"), JSON.stringify(ended));
+    await notify(s, { server: s.server, request, ok: outcome.ok, ...(outcome.sha === undefined ? {} : { sha: outcome.sha }), page: target(s, request.owner, id) }, log);
     expired(readdirSync(join(runsDir(s), request.owner)), keepRuns).forEach((old) => {
         rmSync(join(runsDir(s), request.owner, old), { recursive: true, force: true });
     });

package/cli/dist/deployer/forge.js

@@ -1,25 +1,17 @@
-/**
- * A commit status on the app's forge, linking to the run: GitHub's REST API,
- * or for any other host Forgejo's (Gitea's statuses endpoint). Undefined for a
- * repo URL neither understands.
- */
-export const statusRequest = (repo, commit, status, token) => {
-    const match = /^(https?:\/\/[^/]+)\/([^/]+)\/([^/]+?)(\.git)?\/?$/.exec(repo);
+import { forgejo } from "./forgejo.js";
+import { github } from "./github.js";
+import { gitlab } from "./gitlab.js";
+const adapters = { github, forgejo, gitlab };
+const kinds = { "https://github.com": "github", "https://gitlab.com": "gitlab" };
+/** GitHub for https://github.com, GitLab for https://gitlab.com, a Forgejo for any other host; undefined for a URL none understands. */
+export const forgeRepo = (repo) => {
+    const match = /^(https?:\/\/[^/]+)\/(.+)\/([^/]+?)(\.git)?\/?$/.exec(repo);
     if (match === null)
         return undefined;
     const [, origin, owner, name] = match;
     if (origin === undefined || owner === undefined || name === undefined)
         return undefined;
-    const body = JSON.stringify({ state: status.state, context: status.context, description: status.description, target_url: status.target });
-    return origin === "https://github.com"
-        ? {
-            url: `https://api.github.com/repos/${owner}/${name}/statuses/${commit}`,
-            headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "Content-Type": "application/json" },
-            body,
-        }
-        : {
-            url: `${origin}/api/v1/repos/${owner}/${name}/statuses/${commit}`,
-            headers: { Authorization: `token ${token}`, "Content-Type": "application/json" },
-            body,
-        };
+    return { kind: kinds[origin] ?? "forgejo", origin, owner, name };
 };
+export const forge = (repo, token, fetch) => adapters[repo.kind].api(repo, token, fetch);
+export const signed = (repo, secret, body, headers) => adapters[repo.kind].signed(secret, body, headers);

package/cli/dist/deployer/forgejo.js

@@ -0,0 +1,22 @@
+import { err, ok } from "../result.js";
+import { header, hmacMatches } from "./webhook.js";
+/** A Forgejo's API (Gitea's statuses endpoint); webhooks signed `X-Forgejo-Signature: <hex>` (or Gitea's, or GitHub's, which it sends too). */
+export const forgejo = {
+    api: (repo, token, fetch) => ({
+        postStatus: async (commit, status) => {
+            const response = await fetch(`${repo.origin}/api/v1/repos/${repo.owner}/${repo.name}/statuses/${commit}`, {
+                method: "POST",
+                headers: { Authorization: `token ${token}`, "Content-Type": "application/json" },
+                body: JSON.stringify({ state: status.state, context: status.context, description: status.description, target_url: status.target }),
+            }).catch(() => undefined);
+            if (response === undefined)
+                return err("no answer");
+            return response.ok ? ok(undefined) : err(`HTTP ${String(response.status)}`);
+        },
+    }),
+    signed: (secret, body, headers) => {
+        const hub = header(headers, "x-hub-signature-256");
+        const own = header(headers, "x-forgejo-signature") ?? header(headers, "x-gitea-signature");
+        return hmacMatches(secret, body, own ?? (hub?.startsWith("sha256=") === true ? hub.slice(7) : undefined));
+    },
+};

package/cli/dist/deployer/github-issues.js

@@ -0,0 +1,71 @@
+import { createSign } from "node:crypto";
+import { err, ok } from "../result.js";
+import { field } from "./app.js";
+const base64url = (data) => Buffer.from(data).toString("base64url");
+/** The app's own token: a JWT signed with its private key, valid nine minutes. */
+export const appJwt = (appId, privateKey, nowSeconds) => {
+    const header = base64url(JSON.stringify({ alg: "RS256", typ: "JWT" }));
+    const payload = base64url(JSON.stringify({ iat: nowSeconds - 60, exp: nowSeconds + 540, iss: appId }));
+    const signature = createSign("RSA-SHA256").update(`${header}.${payload}`).sign(privateKey);
+    return `${header}.${payload}.${base64url(signature)}`;
+};
+const api = "https://api.github.com";
+const headers = (token) => ({
+    Authorization: `Bearer ${token}`,
+    Accept: "application/vnd.github+json",
+    "Content-Type": "application/json",
+});
+export const githubIssues = (repository, appId, world) => {
+    const call = async (token, method, path, body) => {
+        const response = await world
+            .fetch(`${api}${path}`, { method, headers: headers(token), ...(body === undefined ? {} : { body: JSON.stringify(body) }) })
+            .catch((error) => ({ status: 0, json: String(error) }));
+        return response.status >= 200 && response.status < 300 ? ok(response.json) : err(`GitHub ${method} ${path}: ${String(response.status)}`);
+    };
+    /** The installation's token for the repository, through the app's JWT. */
+    const token = async () => {
+        const key = world.privateKey();
+        if (!key.ok)
+            return key;
+        const jwt = appJwt(appId, key.value, Math.floor(world.now() / 1000));
+        const installation = await call(jwt, "GET", `/repos/${repository}/installation`);
+        const id = installation.ok && typeof installation.value === "object" && installation.value !== null ? field(installation.value, "id") : undefined;
+        if (typeof id !== "number")
+            return err(installation.ok ? "GitHub: the app is not installed on the repository" : installation.error);
+        const access = await call(jwt, "POST", `/app/installations/${String(id)}/access_tokens`);
+        const value = access.ok && typeof access.value === "object" && access.value !== null ? field(access.value, "token") : undefined;
+        return typeof value === "string" ? ok(value) : err(access.ok ? "GitHub: no installation token" : access.error);
+    };
+    const raise = async (alert) => {
+        if (world.openIssue(alert.key) !== undefined)
+            return ok(undefined);
+        const minted = await token();
+        if (!minted.ok)
+            return minted;
+        const created = await call(minted.value, "POST", `/repos/${repository}/issues`, { title: alert.title, body: alert.body });
+        if (!created.ok)
+            return created;
+        const number = typeof created.value === "object" && created.value !== null ? field(created.value, "number") : undefined;
+        if (typeof number !== "number")
+            return err("GitHub: the new issue has no number");
+        world.remember(alert.key, number);
+        return ok(undefined);
+    };
+    const resolve = async (key, note) => {
+        const issue = world.openIssue(key);
+        if (issue === undefined)
+            return ok(undefined);
+        const minted = await token();
+        if (!minted.ok)
+            return minted;
+        const commented = await call(minted.value, "POST", `/repos/${repository}/issues/${String(issue)}/comments`, { body: note });
+        if (!commented.ok)
+            return commented;
+        const closed = await call(minted.value, "PATCH", `/repos/${repository}/issues/${String(issue)}`, { state: "closed", state_reason: "completed" });
+        if (!closed.ok)
+            return closed;
+        world.remember(key, undefined);
+        return ok(undefined);
+    };
+    return { raise, resolve };
+};

package/cli/dist/deployer/github.js

@@ -0,0 +1,21 @@
+import { err, ok } from "../result.js";
+import { header, hmacMatches } from "./webhook.js";
+/** GitHub's REST API; webhooks signed `X-Hub-Signature-256: sha256=<hex>`. */
+export const github = {
+    api: (repo, token, fetch) => ({
+        postStatus: async (commit, status) => {
+            const response = await fetch(`https://api.github.com/repos/${repo.owner}/${repo.name}/statuses/${commit}`, {
+                method: "POST",
+                headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "Content-Type": "application/json" },
+                body: JSON.stringify({ state: status.state, context: status.context, description: status.description, target_url: status.target }),
+            }).catch(() => undefined);
+            if (response === undefined)
+                return err("no answer");
+            return response.ok ? ok(undefined) : err(`HTTP ${String(response.status)}`);
+        },
+    }),
+    signed: (secret, body, headers) => {
+        const hub = header(headers, "x-hub-signature-256");
+        return hub?.startsWith("sha256=") === true && hmacMatches(secret, body, hub.slice(7));
+    },
+};

package/cli/dist/deployer/gitlab.js

@@ -0,0 +1,25 @@
+import { err, ok } from "../result.js";
+import { header, secretMatches } from "./webhook.js";
+/** GitLab knows no "failure" or "error", only "failed". */
+const states = { pending: "pending", success: "success", failure: "failed", error: "failed" };
+/**
+ * GitLab's REST API (v4), the project addressed by its URL-encoded path; the
+ * context is a status's name. Webhooks carry the secret itself as
+ * `X-Gitlab-Token`: GitLab signs nothing.
+ */
+export const gitlab = {
+    api: (repo, token, fetch) => ({
+        postStatus: async (commit, status) => {
+            const project = encodeURIComponent(`${repo.owner}/${repo.name}`);
+            const response = await fetch(`${repo.origin}/api/v4/projects/${project}/statuses/${commit}`, {
+                method: "POST",
+                headers: { "PRIVATE-TOKEN": token, "Content-Type": "application/json" },
+                body: JSON.stringify({ state: states[status.state], name: status.context, description: status.description, target_url: status.target }),
+            }).catch(() => undefined);
+            if (response === undefined)
+                return err("no answer");
+            return response.ok ? ok(undefined) : err(`HTTP ${String(response.status)}`);
+        },
+    }),
+    signed: (secret, _body, headers) => secretMatches(secret, header(headers, "x-gitlab-token")),
+};

package/cli/dist/deployer/notify.js

@@ -0,0 +1,61 @@
+import { err, ok } from "../result.js";
+import { field } from "./app.js";
+export const silent = {
+    raise: () => Promise.resolve(ok(undefined)),
+    resolve: () => Promise.resolve(ok(undefined)),
+};
+export const parseNotify = (raw) => {
+    if (raw === undefined || raw.trim() === "")
+        return ok({ kind: "none" });
+    try {
+        const json = JSON.parse(raw);
+        if (typeof json !== "object" || json === null)
+            return err("notifier: not an object");
+        const kind = field(json, "kind");
+        const repository = field(json, "repository");
+        const appId = field(json, "app_id");
+        if (kind !== "github-issue")
+            return err(`notifier: unknown kind ${String(kind)}`);
+        if (typeof repository !== "string" || !/^[\w.-]+\/[\w.-]+$/.test(repository))
+            return err(`notifier: repository ${String(repository)}`);
+        if (typeof appId !== "string" && typeof appId !== "number")
+            return err("notifier: app_id missing");
+        return ok({ kind, repository, appId: String(appId) });
+    }
+    catch {
+        return err("notifier: not JSON");
+    }
+};
+/**
+ * What a finished run means for alerts. A deploy of the app's own ref that
+ * failed (a push or the poll) raises; a deploy someone started by hand
+ * (kayra app deploy | rollback) only resolves, since whoever started it is
+ * watching. A platform job raises when it fails. Any success resolves.
+ */
+export const decide = (end) => {
+    const { request, server } = end;
+    const key = request.owner;
+    const commit = end.sha === undefined ? "" : ` at ${end.sha.slice(0, 12)}`;
+    if (end.ok)
+        return { kind: "resolve", key, note: `Recovered: ${end.page}` };
+    if (request.work.kind === "deploy") {
+        if (request.work.why !== "push" && request.work.why !== "poll")
+            return { kind: "none" };
+        return {
+            kind: "raise",
+            alert: {
+                key,
+                title: `${server}: ${request.work.app} did not deploy${commit}`,
+                body: `${request.work.app}'s ${request.work.ref} failed to build or deploy on ${server}${commit}, after a ${request.work.why}.\n\nThe run: ${end.page}\n\nIt stays open until a deploy of ${request.work.app} on ${server} succeeds.`,
+            },
+        };
+    }
+    return {
+        kind: "raise",
+        alert: {
+            key,
+            title: `${server}: the ${request.work.job} job failed`,
+            body: `The platform's ${request.work.job} job failed on ${server}.\n\nThe run: ${end.page}\n\nIt stays open until the job succeeds on ${server}.`,
+        },
+    };
+};

package/cli/dist/deployer/webhook.js

@@ -1,22 +1,25 @@
-import { createHmac, timingSafeEqual } from "node:crypto";
+import { createHash, createHmac, timingSafeEqual } from "node:crypto";
 import { field } from "./app.js";
-/**
- * Whether a webhook was signed with the platform's secret: GitHub's
- * `X-Hub-Signature-256: sha256=<hex>` (Forgejo sends it too), or Forgejo's own
- * `X-Forgejo-Signature: <hex>`. HMAC-SHA256 over the raw body.
- */
-export const signed = (secret, body, headers) => {
-    if (secret === "")
-        return false;
-    const hub = headers["x-hub-signature-256"];
-    const forgejo = headers["x-forgejo-signature"] ?? headers["x-gitea-signature"];
-    const given = typeof hub === "string" && hub.startsWith("sha256=") ? hub.slice(7) : typeof forgejo === "string" ? forgejo : undefined;
-    if (given === undefined || !/^[0-9a-f]{64}$/.test(given))
+/** A header's single value; undefined when it is missing or repeated. */
+export const header = (headers, name) => {
+    const value = headers[name];
+    return typeof value === "string" ? value : undefined;
+};
+/** Whether `given` is the hex HMAC-SHA256 of the raw body under the platform's secret. */
+export const hmacMatches = (secret, body, given) => {
+    if (secret === "" || given === undefined || !/^[0-9a-f]{64}$/.test(given))
         return false;
     const expected = createHmac("sha256", secret).update(body).digest();
     return timingSafeEqual(expected, Buffer.from(given, "hex"));
 };
-/** The ref a push event moved (`refs/heads/main`); undefined for any other event, e.g. GitHub's ping. */
+/** Whether `given` is the platform's secret itself, compared in constant time. */
+export const secretMatches = (secret, given) => {
+    if (secret === "" || given === undefined)
+        return false;
+    const digest = (text) => createHash("sha256").update(text).digest();
+    return timingSafeEqual(digest(secret), digest(given));
+};
+/** The ref a push event moved (`refs/heads/main`); undefined for any other event, e.g. GitHub's ping. All three forges send it as `ref`. */
 export const pushedRef = (body) => {
     try {
         const event = JSON.parse(body.toString("utf8"));

package/cli/dist/fleet.js

@@ -39,8 +39,12 @@ const vault = (descriptor) => {
         ? map(absolutePath(file), (path) => ({ kind: "file", file: vaultPasswordFile(path) }))
         : err("vault_password_file must be a path");
 };
-/** Apps live in their own repo: the builder clones it over HTTPS, and the app root stays inside it. */
-const app = (value) => {
+/**
+ * Apps live in their own repo: the builder clones it over HTTPS, and the app
+ * root stays inside it. A multipass server, a test machine by definition, may
+ * clone over plain HTTP too: the upgrade lab serves its fixture app that way.
+ */
+const app = (target) => (value) => {
     const entry = mapping(value);
     if (entry === undefined)
         return err("an app must be a mapping");
@@ -48,8 +52,9 @@ const app = (value) => {
     if (!name.ok)
         return name;
     const repo = entry["repo"];
-    if (typeof repo !== "string" || !repo.startsWith("https://"))
-        return err(`app ${name.value}: repo must be an https:// URL`);
+    const schemes = target === "multipass" ? ["https://", "http://"] : ["https://"];
+    if (typeof repo !== "string" || !schemes.some((scheme) => repo.startsWith(scheme)))
+        return err(`app ${name.value}: repo must be an https:// URL${target === "multipass" ? " (or http:// on multipass)" : ""}`);
     const path = entry["path"] ?? ".";
     if (typeof path !== "string" || `/${path}/`.includes("/../"))
         return err(`app ${name.value}: path must stay inside its repo`);
@@ -98,7 +103,7 @@ export const parseServer = (files) => {
         return err("apps must be a list");
     if (kind.value !== "platform" && declaredApps.length > 0)
         return err("apps run on platform servers only");
-    const apps = all(declaredApps.map(app));
+    const apps = all(declaredApps.map(app(target.value)));
     if (!apps.ok)
         return apps;
     const shape = checkShape("server.yml", descriptor.value, serverSchema);

package/cli/dist/main.js

@@ -38,6 +38,8 @@ const onServer = (on, target, command) => {
             return server.setup(on, target);
         case "plan":
             return server.preview(on, target);
+        case "verify":
+            return server.verify(on, target);
         case "apps":
             return server.apps(on, target);
         case "hosts":

package/cli/dist/settings.js

@@ -22,7 +22,7 @@ export const settings = [
     setting("acme_dns", platform, z.enum(["", "hetzner"]), "Certificates by DNS-01 instead of HTTP-01, for a server the internet never reaches: the DNS provider (hetzner)."),
     setting("renovate_repositories", platform, texts, "owner/repo of the repos this server's Renovate job keeps up to date; the job runs only where the list is not empty."),
     setting("renovate_schedule", platform, text, "When the Renovate job runs, as a five-field cron spec in the server's time (default weekly: 0 4 * * 1)."),
-    setting("renovate_platform", platform, z.enum(["github", "forgejo"]), "Where the repos live: github (default) or forgejo."),
+    setting("renovate_platform", platform, z.enum(["github", "forgejo", "gitlab"]), "Where the repos live: github (default), forgejo or gitlab."),
     setting("renovate_endpoint", platform, text, "With a Forgejo: its API on platform-internal, e.g. http://forgejo:3000/api/v1."),
     setting("renovate_git_url", platform, z.enum(["default", "ssh", "endpoint"]), "How Renovate clones; endpoint with a Forgejo, so it uses the internal address."),
     setting("renovate_host_rules", platform, z.array(record({ matchHost: text })), "Renovate hostRules: allowInternal for a Forgejo on platform-internal, credentials for a private registry."),
@@ -64,6 +64,9 @@ export const settings = [
     // edge
     setting("wireguard", edge, record({ address: text, port: z.number(), peer_public_key: text, peer_allowed_ips: texts, keepalive: z.number().optional() }), "The tunnel into the private network: the edge's address, port, and the one peer (its key, what it may reach)."),
     setting("edge_routes", edge, z.array(record({ subdomains: texts, upstream: text, forwarded_headers: z.boolean().optional() })), "Public names the edge serves: subdomains of the domain and the upstream inside the tunnel they go to."),
+    setting("notifier", platform, z.discriminatedUnion("kind", [
+        z.strictObject({ kind: z.literal("github-issue"), repository: text, app_id: z.union([text, z.number()]) }),
+    ]), "Where the deployer raises alerts (a failed deploy of an app's ref, a failed job) and resolves them: kind github-issue, an issue in repository (owner/name) opened by the GitHub App app_id; the key in notifier_github_app_key."),
     setting("backup_check", edge, record({
         stale_after_hours: z.number(),
         github_repository: text,
@@ -71,6 +74,7 @@ export const settings = [
         repositories: z.array(record({ name: text, url: text, user: text, restore_test_max_days: z.number().optional() })),
     }), "The hourly check of the private network's backups: when a snapshot is stale, where issues go, which repositories."),
     // Secrets
+    secret("notifier_github_app_key", platform, text, "With notifier kind github-issue: the GitHub App's private key (PEM) the deployer opens issues with."),
     secret("git_token", platform, text, "Token the builder clones app repos with and the deployer posts commit statuses with."),
     secret("webhook_secret", platform, text, "HMAC secret the apps' push webhooks (deploy.<domain>/hook/<app>) are signed with."),
     secret("renovate_token", platform, text, "Token the Renovate job opens PRs with."),

package/compose/docker-compose.ci.yml

@@ -128,12 +128,18 @@ services:
       - RENOVATE_GITHUB_COM_TOKEN=${RENOVATE_GITHUB_COM_TOKEN:-}
       - RENOVATE_HOST_RULES=${RENOVATE_HOST_RULES:-}
       - RENOVATE_REPOSITORIES=${RENOVATE_REPOSITORIES:-}
+      - SERVER=${SERVER:-}
+      - NOTIFIER=${NOTIFIER:-}
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ${APPS_DIR:?}:${APPS_DIR:?}:ro
       - ./apps:/opt/platform/apps:ro
       - ./runs:/opt/platform/runs
       - ./releases:/opt/platform/releases
+      # The alerts open now: one file per key, holding where it is open.
+      - ./alerts:/opt/platform/alerts
+      # The notifier's key; mounted nowhere else.
+      - ./deployer-secrets:/run/deployer-secrets:ro
       # One directory per app, one file per secret. Written by bin/app-apply.
       - ./app-secrets:/run/app-secrets:ro
       # GIT_TOKEN (commit statuses) and RENOVATE_TOKEN.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yunuseon/kayra",
-  "version": "0.9.1",
+  "version": "0.11.0",
   "description": "A self-hosted app platform and the CLI that runs it: Traefik, Authelia, a registry, rootless BuildKit and a small deployer per server, set up with OpenTofu and Ansible.",
   "license": "MIT",
   "author": "Yunus Öztürk",

package/schema/defaults.json

@@ -53,9 +53,10 @@
       "type": "string",
       "enum": [
         "github",
-        "forgejo"
+        "forgejo",
+        "gitlab"
       ],
-      "description": "Where the repos live: github (default) or forgejo."
+      "description": "Where the repos live: github (default), forgejo or gitlab."
     },
     "renovate_endpoint": {
       "type": "string",
@@ -468,6 +469,35 @@
       },
       "description": "Public names the edge serves: subdomains of the domain and the upstream inside the tunnel they go to."
     },
+    "notifier": {
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
+            "kind": {
+              "type": "string",
+              "const": "github-issue"
+            },
+            "repository": {
+              "type": "string"
+            },
+            "app_id": {
+              "type": [
+                "string",
+                "number"
+              ]
+            }
+          },
+          "required": [
+            "kind",
+            "repository",
+            "app_id"
+          ],
+          "additionalProperties": false
+        }
+      ],
+      "description": "Where the deployer raises alerts (a failed deploy of an app's ref, a failed job) and resolves them: kind github-issue, an issue in repository (owner/name) opened by the GitHub App app_id; the key in notifier_github_app_key."
+    },
     "backup_check": {
       "type": "object",
       "properties": {

package/schema/vars-platform.json

@@ -53,9 +53,10 @@
       "type": "string",
       "enum": [
         "github",
-        "forgejo"
+        "forgejo",
+        "gitlab"
       ],
-      "description": "Where the repos live: github (default) or forgejo."
+      "description": "Where the repos live: github (default), forgejo or gitlab."
     },
     "renovate_endpoint": {
       "type": "string",
@@ -314,6 +315,35 @@
       ],
       "additionalProperties": {},
       "description": "The server's own compose stacks: their directory on the server and the names under stacks/ to run."
+    },
+    "notifier": {
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
+            "kind": {
+              "type": "string",
+              "const": "github-issue"
+            },
+            "repository": {
+              "type": "string"
+            },
+            "app_id": {
+              "type": [
+                "string",
+                "number"
+              ]
+            }
+          },
+          "required": [
+            "kind",
+            "repository",
+            "app_id"
+          ],
+          "additionalProperties": false
+        }
+      ],
+      "description": "Where the deployer raises alerts (a failed deploy of an app's ref, a failed job) and resolves them: kind github-issue, an issue in repository (owner/name) opened by the GitHub App app_id; the key in notifier_github_app_key."
     }
   },
   "additionalProperties": false,

package/scripts/local/verify.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# =============================================================================
+# verify.sh <server> — is a platform server working, as its users see it?
+# Run by `kayra verify` and at the end of `kayra deploy` / `kayra up`.
+#
+# From here, the way the world reaches it (each name pinned to the server's
+# address, so neither DNS nor /etc/hosts has to be in place yet):
+#   - auth.<domain> answers; the platform's own pages (hosts.conf) send a
+#     browser to the login, not a 403 from rules Authelia never loaded
+#   - every certificate is a real one (Let's Encrypt or mkcert), never
+#     Traefik's default
+#   - an unsigned webhook is refused; every app hostname answers, not 5xx
+# On the server (one SSH call):
+#   - every app in server.yml is registered, and nothing else
+#   - every app's containers run, none unhealthy; its last run did not fail
+#   - every platform container runs, none unhealthy; no alert is open
+#
+# Certificates and routes can lag a fresh deploy (ACME, DNS-01's minute), so
+# the probes retry for VERIFY_WAIT seconds (default 150) before failing.
+# Written for macOS's bash 3.2 too.
+# =============================================================================
+set -euo pipefail
+
+KAYRA_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+FLEET_ROOT="${FLEET_ROOT:?the directory of the fleet (servers/), set by kayra}"
+name="${1:?Usage: verify.sh <server>}"
+dir="${FLEET_ROOT}/servers/${name}"
+wait="${VERIFY_WAIT:-150}"
+failed=0
+
+fail() { echo "✗ $*"; failed=1; }
+pass() { echo "✓ $*"; }
+
+domain="$(yq -r '.domain' "${dir}/server.yml")"
+inventory="${dir}/inventory.ini"
+[ -f "$inventory" ] || { fail "no inventory for ${name}: provision it first"; exit 1; }
+field() { awk -v key="$1" '{for (i = 1; i <= NF; i++) if (index($i, key "=") == 1) {print substr($i, length(key) + 2); exit}}' "$inventory"; }
+address="$(field ansible_host)"
+port="$(field ansible_port)"
+user="$(field ansible_user)"
+key="$(field ansible_ssh_private_key_file)"
+registered_wanted="$(yq -r '.apps // [] | .[].name' "${dir}/server.yml")"
+
+# ── From outside ────────────────────────────────────────────────────────────
+# "<status> <redirect> <verify result> <issuer CN>" for a host, pinned to the server.
+probe() {
+    local host="$1" method="${2:-GET}" result issuer
+    result="$(curl -s -o /dev/null -m 10 -X "$method" -H 'Accept: text/html' \
+        --resolve "${host}:443:${address}" -w '%{http_code} %{redirect_url} %{ssl_verify_result}' \
+        "https://${host}/" 2>/dev/null || echo "000 - -")"
+    issuer="$(openssl s_client -connect "${address}:443" -servername "$host" </dev/null 2>/dev/null \
+        | openssl x509 -noout -issuer 2>/dev/null | sed -n 's/.*CN *= *//p')"
+    echo "${result} ${issuer:-none}"
+}
+
+# Probes until `test` accepts the answer or VERIFY_WAIT runs out; prints the last answer.
+until_ok() {
+    local host="$1" method="$2" test="$3" deadline answer
+    deadline=$(( $(date +%s) + wait ))
+    while :; do
+        answer="$(probe "$host" "$method")"
+        if "$test" "$answer"; then echo "$answer"; return 0; fi
+        [ "$(date +%s)" -lt "$deadline" ] || { echo "$answer"; return 1; }
+        sleep 10
+    done
+}
+
+real_cert() { case "$1" in *"TRAEFIK DEFAULT CERT"|*" none") return 1 ;; esac; [ "$(cut -d' ' -f3 <<< "$1")" = 0 ]; }
+answers() { real_cert "$1" && [ "$(cut -d' ' -f1 <<< "$1")" = 200 ]; }
+to_login() { real_cert "$1" && [ "$(cut -d' ' -f1 <<< "$1")" = 302 ] && case "$(cut -d' ' -f2 <<< "$1")" in "https://auth.${domain}/"*) return 0 ;; esac; return 1; }
+not_5xx() { real_cert "$1" && case "$(cut -d' ' -f1 <<< "$1")" in 000|5*) return 1 ;; esac; }
+refused() { real_cert "$1" && [ "$(cut -d' ' -f1 <<< "$1")" = 401 ]; }
+
+describe() { echo "HTTP $(cut -d' ' -f1 <<< "$1"), certificate by $(cut -d' ' -f4- <<< "$1") (verify $(cut -d' ' -f3 <<< "$1"))"; }
+
+check() {
+    local label="$1" host="$2" method="$3" test="$4" answer
+    if answer="$(until_ok "$host" "$method" "$test")"; then pass "$label"; else fail "${label}: $(describe "$answer")"; fi
+}
+
+check "auth.${domain} answers" "auth.${domain}" GET answers
+while IFS= read -r sub; do
+    [ "$sub" = auth ] || check "${sub}.${domain} sends a browser to the login" "${sub}.${domain}" GET to_login
+done < <(awk '!/^[[:space:]]*(#|$)/ {print $1}' "${KAYRA_ROOT}/hosts.conf")
+
+first_app="$(head -1 <<< "$registered_wanted")"
+if [ -n "$first_app" ]; then
+    hook() {
+        curl -s -o /dev/null -m 10 -X POST --resolve "deploy.${domain}:443:${address}" \
+            -w '%{http_code}' "https://deploy.${domain}/hook/${first_app}" 2>/dev/null || echo 000
+    }
+    code="$(hook)"
+    [ "$code" = 401 ] && pass "an unsigned webhook is refused" || fail "an unsigned webhook to /hook/${first_app}: HTTP ${code}, not 401"
+    while IFS= read -r sub; do
+        [ -z "$sub" ] || check "${sub}.${domain} answers" "${sub}.${domain}" GET not_5xx
+    done < <(bash "${KAYRA_ROOT}/scripts/local/app-manifests.sh" --subdomains "$name")
+fi
+
+# ── On the server ───────────────────────────────────────────────────────────
+facts="$(ssh -p "${port:-22}" -i "$key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+    "${user}@${address}" 'set -e
+cd /opt/platform
+echo "registered $(ls apps 2>/dev/null | sed -n "s/\.json$//p" | sort | tr "\n" " ")"
+echo "alerts $(ls alerts 2>/dev/null | tr "\n" " ")"
+docker compose ps -a --format "{{.Service}}|{{.State}}|{{.Health}}" | sed "s/^/platform /"
+for app in $(ls apps 2>/dev/null | sed -n "s/\.json$//p"); do
+    docker ps -a --filter "label=com.docker.compose.project=$app" --format "{{.Names}}|{{.State}}|{{.Status}}" | sed "s/^/container $app /"
+    last=$(ls runs/$app 2>/dev/null | sort | tail -1)
+    [ -z "$last" ] || echo "run $app $(sed -n "s/.*\"state\":\"\([a-z]*\)\".*/\1/p" runs/$app/$last/meta.json) $last"
+done' 2>&1)" || { fail "the server answers over SSH: ${facts}"; exit 1; }
+
+wanted="$(sort <<< "$registered_wanted" | tr '\n' ' ')"
+have="$(sed -n 's/^registered //p' <<< "$facts")"
+wanted="$(xargs <<< "$wanted")"
+have="$(xargs <<< "$have")"
+[ "$wanted" = "$have" ] && pass "registered: ${have:-no apps}" \
+    || fail "registered: ${have:-nothing}; server.yml says ${wanted:-no apps} (kayra apps ${name})"
+
+alerts="$(sed -n 's/^alerts //p' <<< "$facts" | xargs)"
+[ -z "$alerts" ] && pass "no alert is open" || fail "open alerts: ${alerts}"
+
+bad_platform="$(sed -n 's/^platform //p' <<< "$facts" | awk -F'|' '$2 != "running" || $3 == "unhealthy" || $3 == "starting" {print $1 " " $2 " " $3}')"
+[ -z "$bad_platform" ] && pass "every platform container runs" || fail "platform containers: $(paste -sd',' - <<< "$bad_platform")"
+
+while IFS= read -r app; do
+    [ -n "$app" ] || continue
+    containers="$(sed -n "s/^container ${app} //p" <<< "$facts")"
+    run="$(sed -n "s/^run ${app} //p" <<< "$facts")"
+    # One-shot containers (an init, a migration) exit 0 and stay exited.
+    bad="$(awk -F'|' '$2 != "running" && $3 !~ /^Exited \(0\)/ || $3 ~ /unhealthy|health: starting/ {print $1 " " $3}' <<< "$containers")"
+    # Registered and never run: waiting for its first push, not broken.
+    if [ -z "$containers" ] && [ -z "$run" ]; then pass "${app}: not deployed yet"; continue
+    elif [ -z "$containers" ]; then fail "${app}: no containers"
+    elif [ -n "$bad" ]; then fail "${app}: $(paste -sd',' - <<< "$bad")"
+    else pass "${app}: $(grep -c . <<< "$containers") container(s) up"; fi
+    case "$run" in
+        failure*) fail "${app}: its last run failed ($(cut -d' ' -f2 <<< "$run"): kayra app logs ${name} ${app})" ;;
+        "") pass "${app}: no runs yet" ;;
+        *) pass "${app}: last run $(cut -d' ' -f1 <<< "$run")" ;;
+    esac
+done <<< "$registered_wanted"
+
+exit "$failed"