@yunuseon/kayra 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/ansible/playbooks/apps-prune.yml +6 -6
- package/ansible/playbooks/platform.yml +5 -5
- package/ansible/roles/app/tasks/main.yml +26 -14
- package/ansible/roles/app/templates/app.env.j2 +1 -1
- package/ansible/roles/common/tasks/main.yml +1 -1
- package/ansible/roles/forgejo-config/tasks/main.yml +4 -9
- package/ansible/roles/platform/defaults/main.yml +8 -15
- package/ansible/roles/platform/tasks/main.yml +96 -63
- package/ansible/roles/platform/templates/platform.env.j2 +3 -11
- package/cli/dist/brand.js +0 -3
- package/cli/dist/cli.js +10 -1
- package/cli/dist/commands/server.js +16 -2
- package/cli/dist/deployer/app.js +47 -0
- package/cli/dist/deployer/brand.js +8 -0
- package/cli/dist/deployer/builder.js +196 -0
- package/cli/dist/deployer/bundle.js +49 -0
- package/cli/dist/deployer/compose.js +26 -0
- package/cli/dist/deployer/cron.js +44 -0
- package/cli/dist/deployer/ctl.js +77 -0
- package/cli/dist/deployer/deployer.js +284 -0
- package/cli/dist/deployer/forge.js +25 -0
- package/cli/dist/deployer/http.js +22 -0
- package/cli/dist/deployer/main.js +19 -0
- package/cli/dist/deployer/page.js +32 -0
- package/cli/dist/deployer/process.js +31 -0
- package/cli/dist/deployer/queue.js +28 -0
- package/cli/dist/deployer/records.js +48 -0
- package/cli/dist/deployer/run.js +76 -0
- package/cli/dist/deployer/store.js +31 -0
- package/cli/dist/deployer/webhook.js +31 -0
- package/cli/dist/fleet.js +1 -1
- package/cli/dist/main.js +11 -7
- package/cli/dist/settings.js +6 -10
- package/compose/authelia/configuration.yml +3 -3
- package/compose/deployer/Dockerfile.builder +37 -0
- package/compose/deployer/Dockerfile.deployer +40 -0
- package/compose/deployer/kayra-deployer +6 -0
- package/compose/docker-compose.ci.yml +71 -79
- package/hosts.conf +2 -2
- package/package.json +2 -3
- package/schema/defaults.json +2 -14
- package/schema/server.json +1 -1
- package/schema/vars-platform.json +2 -14
- package/scripts/local/require-main.sh +5 -1
- package/scripts/platform/compose-policy.sh +84 -0
- package/scripts/platform/registry-gc.sh +99 -0
- package/scripts/platform/resolve-images.sh +107 -0
- package/scripts/server/{jenkins-app-apply → app-apply} +9 -10
- package/scripts/server/app-purge +1 -1
- package/scripts/server/app-remove +8 -9
- package/ansible/roles/app/templates/casc-app.yaml.j2 +0 -68
- package/compose/jenkins/Dockerfile +0 -40
- package/compose/jenkins/casc/00-platform.yaml +0 -158
- package/compose/jenkins/plugins.txt +0 -97
- package/compose/jenkins-agent/Dockerfile +0 -24
- package/scripts/server/jenkins-api +0 -28
- package/scripts/server/jenkins-reload +0 -12
package/cli/dist/main.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { run } from "@optique/run";
|
|
3
|
-
import { readFileSync } from "node:fs";
|
|
4
3
|
import { createInterface } from "node:readline/promises";
|
|
5
|
-
import { absolutePath, kayraRoot,
|
|
4
|
+
import { absolutePath, kayraRoot, within } from "./brand.js";
|
|
6
5
|
import { parser } from "./cli.js";
|
|
7
6
|
import { doctor } from "./commands/doctor.js";
|
|
8
7
|
import * as server from "./commands/server.js";
|
|
@@ -19,10 +18,11 @@ const engine = absolutePath(import.meta.dirname);
|
|
|
19
18
|
if (!here.ok || !engine.ok)
|
|
20
19
|
throw new Error("node reported a relative path");
|
|
21
20
|
const kayra = kayraRoot(within(engine.value, "..", ".."));
|
|
22
|
-
const
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
const appKinds = { "app runs": "runs", "app logs": "logs", "app deploy": "deploy", "app rollback": "rollback" };
|
|
22
|
+
const isDeployer = (command) => command.name === "job" || command.name in appKinds;
|
|
23
|
+
const deployerCommand = (command) => command.name === "job"
|
|
24
|
+
? { kind: "job", job: command.job, noPrs: command.noPrs }
|
|
25
|
+
: { kind: appKinds[command.name], app: command.app, more: "more" in command ? command.more : undefined };
|
|
26
26
|
const onServer = (on, target, command) => {
|
|
27
27
|
switch (command.name) {
|
|
28
28
|
case "up":
|
|
@@ -50,13 +50,17 @@ const onServer = (on, target, command) => {
|
|
|
50
50
|
}
|
|
51
51
|
};
|
|
52
52
|
const plan = (fleet, command) => {
|
|
53
|
-
const on = { kayra,
|
|
53
|
+
const on = { kayra, fleet: fleet.root };
|
|
54
54
|
if (command.name === "doctor")
|
|
55
55
|
return ok(doctor(fleet, kayra));
|
|
56
56
|
if (command.name === "certs")
|
|
57
57
|
return server.certsFor(on, command.domain, command.subdomains);
|
|
58
58
|
if (command.name === "app purge")
|
|
59
59
|
return flatMap(serverNamed(fleet, command.server), (target) => server.appPurge(on, target, command.app));
|
|
60
|
+
if (isDeployer(command)) {
|
|
61
|
+
const deployerArgs = deployerCommand(command);
|
|
62
|
+
return flatMap(serverNamed(fleet, command.server), (target) => server.deployer(target, deployerArgs));
|
|
63
|
+
}
|
|
60
64
|
return flatMap(serverNamed(fleet, command.server), (target) => onServer(on, target, command));
|
|
61
65
|
};
|
|
62
66
|
const io = {
|
package/cli/dist/settings.js
CHANGED
|
@@ -15,16 +15,13 @@ export const settings = [
|
|
|
15
15
|
setting("acme_email", ["platform", "edge"], text, "The Let's Encrypt account contact: expiry warnings land here."),
|
|
16
16
|
setting("docker_version", ["platform", "edge"], text, "The Docker CE major the docker role pins and holds, as an apt version glob (default 5:29*)."),
|
|
17
17
|
// platform: the platform role
|
|
18
|
-
setting("git_remote_url", platform, text, "kayra's repository: Jenkins loads the pipeline library and platform jobs from it."),
|
|
19
|
-
setting("platform_ref", platform, text, "The ref of git_remote_url Jenkins loads from; defaults to the tag of the kayra version that ran (v<version>). Only a test server sets a branch."),
|
|
20
|
-
setting("jenkins_admin_user", platform, text, "Jenkins' local admin user (default admin)."),
|
|
21
18
|
setting("authelia_policy", platform, z.enum(["one_factor", "two_factor"]), "What Authelia asks of staff: two_factor (default) or one_factor."),
|
|
22
19
|
setting("authelia_smtp_address", platform, text, "SMTP server for Authelia's mails (passkey enrolment); empty: mails go to a file."),
|
|
23
20
|
setting("authelia_smtp_username", platform, text, "SMTP user for Authelia."),
|
|
24
21
|
setting("authelia_smtp_sender", platform, text, "Sender address of Authelia's mails."),
|
|
25
22
|
setting("acme_dns", platform, z.enum(["", "hetzner"]), "Certificates by DNS-01 instead of HTTP-01, for a server the internet never reaches: the DNS provider (hetzner)."),
|
|
26
|
-
setting("renovate_repositories", platform, texts, "owner/repo of the repos this server's
|
|
27
|
-
setting("renovate_schedule", platform, text, "When
|
|
23
|
+
setting("renovate_repositories", platform, texts, "owner/repo of the repos this server's Renovate job keeps up to date; the job runs only where the list is not empty."),
|
|
24
|
+
setting("renovate_schedule", platform, text, "When the Renovate job runs, as a five-field cron spec in the server's time (default weekly: 0 4 * * 1)."),
|
|
28
25
|
setting("renovate_platform", platform, z.enum(["github", "forgejo"]), "Where the repos live: github (default) or forgejo."),
|
|
29
26
|
setting("renovate_endpoint", platform, text, "With a Forgejo: its API on platform-internal, e.g. http://forgejo:3000/api/v1."),
|
|
30
27
|
setting("renovate_git_url", platform, z.enum(["default", "ssh", "endpoint"]), "How Renovate clones; endpoint with a Forgejo, so it uses the internal address."),
|
|
@@ -74,10 +71,9 @@ export const settings = [
|
|
|
74
71
|
repositories: z.array(record({ name: text, url: text, user: text, restore_test_max_days: z.number().optional() })),
|
|
75
72
|
}), "The hourly check of the private network's backups: when a snapshot is stale, where issues go, which repositories."),
|
|
76
73
|
// Secrets
|
|
77
|
-
secret("
|
|
78
|
-
secret("
|
|
79
|
-
secret("
|
|
80
|
-
secret("renovate_token", platform, text, "Token platform-renovate opens PRs with."),
|
|
74
|
+
secret("git_token", platform, text, "Token the builder clones app repos with and the deployer posts commit statuses with."),
|
|
75
|
+
secret("webhook_secret", platform, text, "HMAC secret the apps' push webhooks (deploy.<domain>/hook/<app>) are signed with."),
|
|
76
|
+
secret("renovate_token", platform, text, "Token the Renovate job opens PRs with."),
|
|
81
77
|
secret("renovate_github_com_token", platform, text, "Public-read GitHub token for Renovate's lookups off GitHub."),
|
|
82
78
|
secret("authelia_session_secret", platform, text, "Authelia's session secret."),
|
|
83
79
|
secret("authelia_storage_encryption_key", platform, text, "Authelia's storage encryption key."),
|
|
@@ -110,7 +106,7 @@ export const defaultsSchema = z.strictObject(shape(settings.filter((each) => !ea
|
|
|
110
106
|
export const secretsSchema = (kind) => z.looseObject(shape(settingsFor(kind, true)));
|
|
111
107
|
/** server.yml: topology only. */
|
|
112
108
|
export const serverSchema = z.strictObject({
|
|
113
|
-
kind: z.enum(serverKinds).optional().describe("Which playbook: platform (default; Traefik, Authelia,
|
|
109
|
+
kind: z.enum(serverKinds).optional().describe("Which playbook: platform (default; Traefik, Authelia, the deployer, apps), edge or nas."),
|
|
114
110
|
target: z.enum(["hetzner", "multipass", "home"]).describe("Where it runs: hetzner, multipass (a local VM) or home (a machine nothing creates)."),
|
|
115
111
|
domain: z.string().describe("The server's domain; platform subdomains and the apps' hang off it."),
|
|
116
112
|
vault: z.boolean().optional().describe("secrets.yml is vault-encrypted (required outside multipass)."),
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# =============================================================================
|
|
2
|
-
# Authelia — login for the platform's own pages (
|
|
2
|
+
# Authelia — login for the platform's own pages (the runs pages, Traefik's
|
|
3
3
|
# dashboard). Apps bring their own login; their users never live here.
|
|
4
4
|
#
|
|
5
5
|
# Authelia does NOT expand ${VAR} the way docker compose does. Interpolation here
|
|
@@ -86,7 +86,7 @@ authentication_backend:
|
|
|
86
86
|
access_control:
|
|
87
87
|
default_policy: 'deny'
|
|
88
88
|
rules:
|
|
89
|
-
- domain: '
|
|
89
|
+
- domain: 'deploy.{{ env "DOMAIN" }}'
|
|
90
90
|
policy: '{{ env "AUTH_POLICY" }}'
|
|
91
91
|
subject:
|
|
92
92
|
- 'group:admin'
|
|
@@ -104,7 +104,7 @@ session:
|
|
|
104
104
|
expiration: '1 day'
|
|
105
105
|
remember_me: '1 week'
|
|
106
106
|
cookies:
|
|
107
|
-
# The apex, not auth.<DOMAIN> — the cookie has to reach
|
|
107
|
+
# The apex, not auth.<DOMAIN> — the cookie has to reach deploy.<DOMAIN>
|
|
108
108
|
# and traefik.<DOMAIN> too. Authelia requires this to be a suffix of (or equal to) authelia_url.
|
|
109
109
|
- domain: '{{ env "DOMAIN" }}'
|
|
110
110
|
authelia_url: 'https://auth.{{ env "DOMAIN" }}'
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# The builder — git, buildctl, jq, yq and node: it clones app repositories,
|
|
3
|
+
# builds their images through buildkitd and answers forge webhooks.
|
|
4
|
+
#
|
|
5
|
+
# Deliberately has NO docker CLI and no docker socket: it handles code from
|
|
6
|
+
# arbitrary commits, and the host's socket would make that host root. Images
|
|
7
|
+
# are built by buildkitd over TCP. No language toolchain either: apps build and
|
|
8
|
+
# test inside their Dockerfiles, so a new app never changes this image.
|
|
9
|
+
#
|
|
10
|
+
# Context: <platform_dir>/kayra, copied by the platform role (the deployer's
|
|
11
|
+
# code and the platform scripts).
|
|
12
|
+
# =============================================================================
|
|
13
|
+
|
|
14
|
+
FROM node:24.21.0-trixie-slim@sha256:173f125896c3b47ddf056734c7ea789d04595a6a08769a8f78e0df642781fb66
|
|
15
|
+
|
|
16
|
+
# renovate: datasource=github-releases depName=moby/buildkit
|
|
17
|
+
ARG BUILDKIT_VERSION=v0.34.0
|
|
18
|
+
# renovate: datasource=github-releases depName=mikefarah/yq
|
|
19
|
+
ARG YQ_VERSION=v4.54.1
|
|
20
|
+
|
|
21
|
+
RUN apt-get update && \
|
|
22
|
+
apt-get install -y --no-install-recommends ca-certificates curl git jq && \
|
|
23
|
+
arch="$(dpkg --print-architecture)" && \
|
|
24
|
+
curl -fsSL "https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-${arch}.tar.gz" \
|
|
25
|
+
| tar -xz -C /usr/local bin/buildctl && \
|
|
26
|
+
curl -fsSL -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${arch}" && \
|
|
27
|
+
chmod +x /usr/local/bin/yq && \
|
|
28
|
+
rm -rf /var/lib/apt/lists/* && \
|
|
29
|
+
install -d -o node -g node /work
|
|
30
|
+
|
|
31
|
+
COPY --chmod=0755 kayra-deployer /usr/local/bin/kayra-deployer
|
|
32
|
+
COPY app /opt/kayra
|
|
33
|
+
# The code is ES modules, from either a checkout (.ts) or the package (.js).
|
|
34
|
+
RUN echo '{"type": "module"}' > /opt/kayra/package.json
|
|
35
|
+
|
|
36
|
+
USER node
|
|
37
|
+
ENTRYPOINT ["kayra-deployer"]
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# The deployer — node, the Docker CLI with Compose, buildctl and jq: it admits
|
|
3
|
+
# and starts apps, records releases and runs the platform's jobs (registry GC,
|
|
4
|
+
# Renovate). The only container of the platform's CI with the docker socket,
|
|
5
|
+
# and Traefik never routes to it.
|
|
6
|
+
#
|
|
7
|
+
# Context: <platform_dir>/kayra, copied by the platform role.
|
|
8
|
+
# =============================================================================
|
|
9
|
+
|
|
10
|
+
FROM node:24.21.0-trixie-slim@sha256:173f125896c3b47ddf056734c7ea789d04595a6a08769a8f78e0df642781fb66
|
|
11
|
+
|
|
12
|
+
# renovate: datasource=github-releases depName=moby/buildkit
|
|
13
|
+
ARG BUILDKIT_VERSION=v0.34.0
|
|
14
|
+
# What the Renovate job runs, pinned here so Renovate keeps it current itself.
|
|
15
|
+
# renovate: datasource=docker depName=renovate/renovate
|
|
16
|
+
ARG RENOVATE_VERSION=44.115.10
|
|
17
|
+
ENV RENOVATE_IMAGE=renovate/renovate:${RENOVATE_VERSION}
|
|
18
|
+
|
|
19
|
+
RUN apt-get update && \
|
|
20
|
+
apt-get install -y --no-install-recommends ca-certificates curl gnupg jq && \
|
|
21
|
+
install -m 0755 -d /etc/apt/keyrings && \
|
|
22
|
+
curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
|
|
23
|
+
chmod a+r /etc/apt/keyrings/docker.gpg && \
|
|
24
|
+
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
|
|
25
|
+
https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \
|
|
26
|
+
> /etc/apt/sources.list.d/docker.list && \
|
|
27
|
+
apt-get update && \
|
|
28
|
+
apt-get install -y --no-install-recommends docker-ce-cli docker-compose-plugin && \
|
|
29
|
+
arch="$(dpkg --print-architecture)" && \
|
|
30
|
+
curl -fsSL "https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-${arch}.tar.gz" \
|
|
31
|
+
| tar -xz -C /usr/local bin/buildctl && \
|
|
32
|
+
rm -rf /var/lib/apt/lists/*
|
|
33
|
+
|
|
34
|
+
COPY --chmod=0755 kayra-deployer /usr/local/bin/kayra-deployer
|
|
35
|
+
COPY app /opt/kayra
|
|
36
|
+
# The code is ES modules, from either a checkout (.ts) or the package (.js).
|
|
37
|
+
RUN echo '{"type": "module"}' > /opt/kayra/package.json
|
|
38
|
+
|
|
39
|
+
USER node
|
|
40
|
+
ENTRYPOINT ["kayra-deployer"]
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# kayra-deployer builder | deployer | ctl … — the deployer's one program, as
|
|
3
|
+
# TypeScript from a kayra checkout or JavaScript from the published package.
|
|
4
|
+
main=/opt/kayra/deployer/main.ts
|
|
5
|
+
[ -e "$main" ] || main=/opt/kayra/deployer/main.js
|
|
6
|
+
exec node "$main" "$@"
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# =============================================================================
|
|
2
|
-
# Platform CI overlay — Docker Registry
|
|
2
|
+
# Platform CI overlay — Docker Registry, BuildKit, the builder and the deployer
|
|
3
3
|
# App-agnostic: provides CI/CD infrastructure for any application.
|
|
4
4
|
# Usage: docker compose -f docker-compose.yml -f docker-compose.ci.yml up
|
|
5
5
|
# =============================================================================
|
|
@@ -53,104 +53,96 @@ services:
|
|
|
53
53
|
networks:
|
|
54
54
|
- internal
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
# The builder: clones app repos and builds their images through buildkitd,
|
|
58
|
+
# answers forge webhooks, shows the runs. Faces app code and the internet,
|
|
59
|
+
# so it gets no docker socket, no docker CLI and no app secrets.
|
|
60
|
+
# ---------------------------------------------------------------------------
|
|
61
|
+
builder:
|
|
57
62
|
build:
|
|
58
|
-
context: ./
|
|
59
|
-
dockerfile: Dockerfile
|
|
60
|
-
|
|
61
|
-
|
|
63
|
+
context: ./deployer
|
|
64
|
+
dockerfile: Dockerfile.builder
|
|
65
|
+
restart: unless-stopped
|
|
66
|
+
command: ["builder"]
|
|
67
|
+
environment:
|
|
68
|
+
- WEBHOOK_SECRET=${WEBHOOK_SECRET:-}
|
|
69
|
+
- BUILDKIT_HOST=tcp://buildkitd:1234
|
|
70
|
+
- REGISTRY_PUSH_HOST=registry:5000
|
|
71
|
+
volumes:
|
|
72
|
+
- builder-work:/work
|
|
73
|
+
- ./apps:/opt/platform/apps:ro
|
|
74
|
+
- ./runs:/opt/platform/runs:ro
|
|
75
|
+
# GIT_TOKEN, to clone. Files, so a new one needs no restart.
|
|
76
|
+
- ./platform-secrets:/run/platform-secrets:ro
|
|
77
|
+
mem_limit: 512m
|
|
78
|
+
labels:
|
|
79
|
+
- "traefik.enable=true"
|
|
80
|
+
- "traefik.docker.network=platform-internal"
|
|
81
|
+
- "traefik.http.routers.deploy.rule=Host(`deploy.${DOMAIN:-localhost}`)"
|
|
82
|
+
- "traefik.http.routers.deploy.entrypoints=websecure"
|
|
83
|
+
- "traefik.http.routers.deploy.tls.certresolver=le"
|
|
84
|
+
- "traefik.http.routers.deploy.middlewares=authelia@file"
|
|
85
|
+
- "traefik.http.routers.deploy.service=deploy"
|
|
86
|
+
# Forges cannot log in; the builder checks the payload's HMAC instead.
|
|
87
|
+
- "traefik.http.routers.deploy-hook.rule=Host(`deploy.${DOMAIN:-localhost}`) && PathPrefix(`/hook/`)"
|
|
88
|
+
- "traefik.http.routers.deploy-hook.entrypoints=websecure"
|
|
89
|
+
- "traefik.http.routers.deploy-hook.tls.certresolver=le"
|
|
90
|
+
- "traefik.http.routers.deploy-hook.priority=100"
|
|
91
|
+
- "traefik.http.routers.deploy-hook.service=deploy"
|
|
92
|
+
- "traefik.http.services.deploy.loadbalancer.server.port=8080"
|
|
93
|
+
networks:
|
|
94
|
+
- internal
|
|
95
|
+
depends_on:
|
|
96
|
+
buildkitd:
|
|
97
|
+
condition: service_started
|
|
98
|
+
|
|
99
|
+
# ---------------------------------------------------------------------------
|
|
100
|
+
# The deployer: the queue, admission, `docker compose up`, release records,
|
|
101
|
+
# commit statuses, and the platform's jobs (registry GC, Renovate, the
|
|
102
|
+
# hourly poll). The one container here with the docker socket and the app
|
|
103
|
+
# secrets; Traefik never routes to it. Everything it gets from the builder
|
|
104
|
+
# is checked again, and compose files still go through the policy.
|
|
105
|
+
#
|
|
106
|
+
# No COMPOSE_PROJECT_NAME here on purpose: every deploy runs `docker
|
|
107
|
+
# compose -p <app>`, and an exported project name would beat it. The
|
|
108
|
+
# platform's directories are mounted at their own paths.
|
|
109
|
+
# ---------------------------------------------------------------------------
|
|
110
|
+
deployer:
|
|
111
|
+
build:
|
|
112
|
+
context: ./deployer
|
|
113
|
+
dockerfile: Dockerfile.deployer
|
|
114
|
+
restart: unless-stopped
|
|
115
|
+
command: ["deployer"]
|
|
116
|
+
# The socket's group, whose GID differs per host.
|
|
62
117
|
group_add:
|
|
63
118
|
- ${DOCKER_GID:?}
|
|
64
|
-
restart: unless-stopped
|
|
65
|
-
# No COMPOSE_PROJECT_NAME here on purpose: every app's deploy runs
|
|
66
|
-
# `docker compose` from this container, and an exported project name would
|
|
67
|
-
# beat the one the deploy passes. APPS_DIR is where app .env files live.
|
|
68
119
|
environment:
|
|
69
|
-
- JENKINS_ADMIN_USER=${JENKINS_ADMIN_USER:-admin}
|
|
70
|
-
- JENKINS_ADMIN_PASSWORD=${JENKINS_ADMIN_PASSWORD:-admin}
|
|
71
|
-
- JENKINS_GIT_REPO=${GIT_REMOTE_URL:?}
|
|
72
|
-
- GITHUB_WEBHOOK_SECRET=${GITHUB_WEBHOOK_SECRET:-}
|
|
73
120
|
- DOMAIN=${DOMAIN:-localhost}
|
|
74
121
|
- APPS_DIR=${APPS_DIR:?}
|
|
75
|
-
-
|
|
122
|
+
- PLATFORM_DIR=/opt/platform
|
|
76
123
|
- PLATFORM_RENOVATE=${PLATFORM_RENOVATE:-false}
|
|
77
|
-
- RENOVATE_SCHEDULE=${RENOVATE_SCHEDULE:-
|
|
124
|
+
- RENOVATE_SCHEDULE=${RENOVATE_SCHEDULE:-0 4 * * 1}
|
|
78
125
|
- RENOVATE_PLATFORM=${RENOVATE_PLATFORM:-github}
|
|
79
126
|
- RENOVATE_ENDPOINT=${RENOVATE_ENDPOINT:-}
|
|
80
127
|
- RENOVATE_GIT_URL=${RENOVATE_GIT_URL:-}
|
|
81
128
|
- RENOVATE_GITHUB_COM_TOKEN=${RENOVATE_GITHUB_COM_TOKEN:-}
|
|
82
129
|
- RENOVATE_HOST_RULES=${RENOVATE_HOST_RULES:-}
|
|
83
130
|
- RENOVATE_REPOSITORIES=${RENOVATE_REPOSITORIES:-}
|
|
84
|
-
# Tell JCasC to merge every YAML in this directory at startup.
|
|
85
|
-
- CASC_JENKINS_CONFIG=/var/jenkins_home/casc
|
|
86
131
|
volumes:
|
|
87
|
-
- jenkins-home:/var/jenkins_home
|
|
88
|
-
# The controller is the only node with the docker socket, and it runs
|
|
89
|
-
# the deploy stage, so it needs the app's compose directory too.
|
|
90
132
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
91
133
|
- ${APPS_DIR:?}:${APPS_DIR:?}:ro
|
|
92
|
-
- ./
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
#
|
|
134
|
+
- ./apps:/opt/platform/apps:ro
|
|
135
|
+
- ./runs:/opt/platform/runs
|
|
136
|
+
- ./releases:/opt/platform/releases
|
|
137
|
+
# One directory per app, one file per secret. Written by bin/app-apply.
|
|
138
|
+
- ./app-secrets:/run/app-secrets:ro
|
|
139
|
+
# GIT_TOKEN (commit statuses) and RENOVATE_TOKEN.
|
|
98
140
|
- ./platform-secrets:/run/platform-secrets:ro
|
|
99
|
-
|
|
100
|
-
- "traefik.enable=true"
|
|
101
|
-
- "traefik.docker.network=platform-internal"
|
|
102
|
-
- "traefik.http.routers.jenkins.rule=Host(`jenkins.${DOMAIN:-localhost}`)"
|
|
103
|
-
- "traefik.http.routers.jenkins.entrypoints=websecure"
|
|
104
|
-
- "traefik.http.routers.jenkins.tls.certresolver=le"
|
|
105
|
-
- "traefik.http.routers.jenkins.middlewares=authelia@file"
|
|
106
|
-
- "traefik.http.routers.jenkins.service=jenkins"
|
|
107
|
-
# GitHub cannot log in; the plugin checks the payload's HMAC instead.
|
|
108
|
-
- "traefik.http.routers.jenkins-webhook.rule=Host(`jenkins.${DOMAIN:-localhost}`) && Path(`/github-webhook/`)"
|
|
109
|
-
- "traefik.http.routers.jenkins-webhook.entrypoints=websecure"
|
|
110
|
-
- "traefik.http.routers.jenkins-webhook.tls.certresolver=le"
|
|
111
|
-
- "traefik.http.routers.jenkins-webhook.priority=100"
|
|
112
|
-
- "traefik.http.routers.jenkins-webhook.service=jenkins"
|
|
113
|
-
- "traefik.http.services.jenkins.loadbalancer.server.port=8080"
|
|
141
|
+
mem_limit: 512m
|
|
114
142
|
networks:
|
|
115
143
|
- internal
|
|
116
|
-
healthcheck:
|
|
117
|
-
test: ["CMD", "curl", "-sf", "http://127.0.0.1:8080/login"]
|
|
118
|
-
interval: 15s
|
|
119
|
-
timeout: 5s
|
|
120
|
-
retries: 10
|
|
121
|
-
start_period: 60s
|
|
122
|
-
|
|
123
|
-
jenkins-agent:
|
|
124
|
-
build:
|
|
125
|
-
context: ./jenkins-agent
|
|
126
|
-
dockerfile: Dockerfile
|
|
127
|
-
restart: unless-stopped
|
|
128
|
-
# Written by the platform role once Jenkins is up; absent on first start,
|
|
129
|
-
# when the agent crash-loops until the role recreates it with the secret.
|
|
130
|
-
env_file:
|
|
131
|
-
- path: ./jenkins-agent.env
|
|
132
|
-
required: false
|
|
133
|
-
environment:
|
|
134
|
-
- JENKINS_URL=http://jenkins-controller:8080
|
|
135
|
-
- JENKINS_AGENT_NAME=builder
|
|
136
|
-
- JENKINS_AGENT_WORKDIR=/home/jenkins/agent
|
|
137
|
-
- DOMAIN=${DOMAIN:-localhost}
|
|
138
|
-
# Builds go to buildkitd, not to a docker daemon. This agent runs pipeline
|
|
139
|
-
# code from untrusted branches and deliberately has no docker socket.
|
|
140
|
-
- BUILDKIT_HOST=tcp://buildkitd:1234
|
|
141
|
-
- REGISTRY_PUSH_HOST=registry:5000
|
|
142
|
-
volumes:
|
|
143
|
-
- jenkins-agent-workdir:/home/jenkins/agent
|
|
144
|
-
networks:
|
|
145
|
-
- internal
|
|
146
|
-
depends_on:
|
|
147
|
-
jenkins-controller:
|
|
148
|
-
condition: service_healthy
|
|
149
|
-
buildkitd:
|
|
150
|
-
condition: service_started
|
|
151
144
|
|
|
152
145
|
volumes:
|
|
153
146
|
registry-data:
|
|
154
147
|
buildkit-cache:
|
|
155
|
-
|
|
156
|
-
jenkins-agent-workdir:
|
|
148
|
+
builder-work:
|
package/hosts.conf
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Platform subdomains to add to /etc/hosts and to the mkcert SAN list.
|
|
2
2
|
# App subdomains live in each app's app.yml; apps may not claim these.
|
|
3
|
-
# One subdomain per line. Read by
|
|
4
|
-
|
|
3
|
+
# One subdomain per line. Read by scripts/local/setup-hosts.sh and scripts/local/certs.sh.
|
|
4
|
+
deploy
|
|
5
5
|
auth
|
|
6
6
|
traefik
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yunuseon/kayra",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "A self-hosted app platform and the CLI that runs it: Traefik, Authelia,
|
|
3
|
+
"version": "0.8.1",
|
|
4
|
+
"description": "A self-hosted app platform and the CLI that runs it: Traefik, Authelia, a registry, rootless BuildKit and a small deployer per server, set up with OpenTofu and Ansible.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Yunus Öztürk",
|
|
7
7
|
"repository": {
|
|
@@ -69,7 +69,6 @@
|
|
|
69
69
|
"self-hosted",
|
|
70
70
|
"platform",
|
|
71
71
|
"traefik",
|
|
72
|
-
"jenkins",
|
|
73
72
|
"ansible",
|
|
74
73
|
"opentofu",
|
|
75
74
|
"hetzner",
|
package/schema/defaults.json
CHANGED
|
@@ -10,18 +10,6 @@
|
|
|
10
10
|
"type": "string",
|
|
11
11
|
"description": "The Docker CE major the docker role pins and holds, as an apt version glob (default 5:29*)."
|
|
12
12
|
},
|
|
13
|
-
"git_remote_url": {
|
|
14
|
-
"type": "string",
|
|
15
|
-
"description": "kayra's repository: Jenkins loads the pipeline library and platform jobs from it."
|
|
16
|
-
},
|
|
17
|
-
"platform_ref": {
|
|
18
|
-
"type": "string",
|
|
19
|
-
"description": "The ref of git_remote_url Jenkins loads from; defaults to the tag of the kayra version that ran (v<version>). Only a test server sets a branch."
|
|
20
|
-
},
|
|
21
|
-
"jenkins_admin_user": {
|
|
22
|
-
"type": "string",
|
|
23
|
-
"description": "Jenkins' local admin user (default admin)."
|
|
24
|
-
},
|
|
25
13
|
"authelia_policy": {
|
|
26
14
|
"type": "string",
|
|
27
15
|
"enum": [
|
|
@@ -55,11 +43,11 @@
|
|
|
55
43
|
"items": {
|
|
56
44
|
"type": "string"
|
|
57
45
|
},
|
|
58
|
-
"description": "owner/repo of the repos this server's
|
|
46
|
+
"description": "owner/repo of the repos this server's Renovate job keeps up to date; the job runs only where the list is not empty."
|
|
59
47
|
},
|
|
60
48
|
"renovate_schedule": {
|
|
61
49
|
"type": "string",
|
|
62
|
-
"description": "When
|
|
50
|
+
"description": "When the Renovate job runs, as a five-field cron spec in the server's time (default weekly: 0 4 * * 1)."
|
|
63
51
|
},
|
|
64
52
|
"renovate_platform": {
|
|
65
53
|
"type": "string",
|
package/schema/server.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"type": "object",
|
|
4
4
|
"properties": {
|
|
5
5
|
"kind": {
|
|
6
|
-
"description": "Which playbook: platform (default; Traefik, Authelia,
|
|
6
|
+
"description": "Which playbook: platform (default; Traefik, Authelia, the deployer, apps), edge or nas.",
|
|
7
7
|
"type": "string",
|
|
8
8
|
"enum": [
|
|
9
9
|
"platform",
|
|
@@ -10,18 +10,6 @@
|
|
|
10
10
|
"type": "string",
|
|
11
11
|
"description": "The Docker CE major the docker role pins and holds, as an apt version glob (default 5:29*)."
|
|
12
12
|
},
|
|
13
|
-
"git_remote_url": {
|
|
14
|
-
"type": "string",
|
|
15
|
-
"description": "kayra's repository: Jenkins loads the pipeline library and platform jobs from it."
|
|
16
|
-
},
|
|
17
|
-
"platform_ref": {
|
|
18
|
-
"type": "string",
|
|
19
|
-
"description": "The ref of git_remote_url Jenkins loads from; defaults to the tag of the kayra version that ran (v<version>). Only a test server sets a branch."
|
|
20
|
-
},
|
|
21
|
-
"jenkins_admin_user": {
|
|
22
|
-
"type": "string",
|
|
23
|
-
"description": "Jenkins' local admin user (default admin)."
|
|
24
|
-
},
|
|
25
13
|
"authelia_policy": {
|
|
26
14
|
"type": "string",
|
|
27
15
|
"enum": [
|
|
@@ -55,11 +43,11 @@
|
|
|
55
43
|
"items": {
|
|
56
44
|
"type": "string"
|
|
57
45
|
},
|
|
58
|
-
"description": "owner/repo of the repos this server's
|
|
46
|
+
"description": "owner/repo of the repos this server's Renovate job keeps up to date; the job runs only where the list is not empty."
|
|
59
47
|
},
|
|
60
48
|
"renovate_schedule": {
|
|
61
49
|
"type": "string",
|
|
62
|
-
"description": "When
|
|
50
|
+
"description": "When the Renovate job runs, as a five-field cron spec in the server's time (default weekly: 0 4 * * 1)."
|
|
63
51
|
},
|
|
64
52
|
"renovate_platform": {
|
|
65
53
|
"type": "string",
|
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
# every other server is only ever changed from a clean main that matches
|
|
7
7
|
# origin/main, so what runs there is exactly what was reviewed and merged.
|
|
8
8
|
# Uncommitted changes in other servers' folders do not count: a test server's
|
|
9
|
-
# branch pointers must not block a prod deploy.
|
|
9
|
+
# branch pointers must not block a prod deploy. The kayra that runs must be the
|
|
10
|
+
# release the fleet pins, not a linked checkout: everything the server runs,
|
|
11
|
+
# its deploy pipeline included, is copied from it.
|
|
10
12
|
#
|
|
11
13
|
# Usage: FLEET_ROOT=<fleet> bash scripts/local/require-main.sh <server-name>
|
|
12
14
|
set -euo pipefail
|
|
@@ -23,6 +25,8 @@ fail() {
|
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
cd "$FLEET_ROOT"
|
|
28
|
+
[ ! -L node_modules/@yunuseon/kayra ] \
|
|
29
|
+
|| fail "node_modules/@yunuseon/kayra is a linked checkout. Release it, bump package.json, then: npm ci"
|
|
26
30
|
branch="$(git branch --show-current)"
|
|
27
31
|
[ "$branch" = "main" ] || fail "The checkout is on '${branch:-a detached HEAD}'. Merge it, then: git switch main && git pull"
|
|
28
32
|
git fetch -q origin main || fail "Could not fetch origin/main to compare against."
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# =============================================================================
|
|
3
|
+
# compose-policy.sh <app> <allowed-host>... < config.json
|
|
4
|
+
#
|
|
5
|
+
# The platform's admission check, run by the deployer before every
|
|
6
|
+
# `docker compose up`. Input is `docker compose config --no-env-resolution
|
|
7
|
+
# --format json`: the fully resolved config, so variables, overlays and
|
|
8
|
+
# defaults cannot hide anything. Env files stay unresolved on purpose: the
|
|
9
|
+
# deployer would read them from its own filesystem, platform secrets
|
|
10
|
+
# included. Prints one line per violation and exits 1 if there is any.
|
|
11
|
+
#
|
|
12
|
+
# An allow-list: a service key the platform has not thought about is refused,
|
|
13
|
+
# rather than a new Compose feature quietly opening a hole. Modelled on
|
|
14
|
+
# Kubernetes' Pod Security "restricted" profile, adapted to what matters on a
|
|
15
|
+
# shared Docker host: an app must not reach the host, other apps, or the
|
|
16
|
+
# platform, must not claim anyone else's hostnames or Traefik names, and must
|
|
17
|
+
# declare how much memory it may take. Running as root inside the container is
|
|
18
|
+
# allowed (unlike restricted): without capabilities or host access it gets
|
|
19
|
+
# nothing a non-root user would not.
|
|
20
|
+
# =============================================================================
|
|
21
|
+
set -euo pipefail
|
|
22
|
+
|
|
23
|
+
app="${1:?Usage: compose-policy.sh <app> <allowed-host>... < config.json}"
|
|
24
|
+
shift
|
|
25
|
+
hosts=$(printf '%s\n' "$@" | jq -R . | jq -s .)
|
|
26
|
+
|
|
27
|
+
jq -r --arg app "$app" --argjson hosts "$hosts" '
|
|
28
|
+
def own($name): $name | startswith($app + "_");
|
|
29
|
+
def registry_ref: test("^(localhost|registry):5000/");
|
|
30
|
+
def allowed: ["image", "command", "entrypoint", "environment", "labels",
|
|
31
|
+
"networks", "volumes", "tmpfs", "healthcheck", "depends_on", "restart",
|
|
32
|
+
"user", "working_dir", "init", "read_only", "expose", "stop_grace_period",
|
|
33
|
+
"stop_signal", "mem_limit", "mem_reservation", "cpus", "deploy"];
|
|
34
|
+
|
|
35
|
+
. as $root |
|
|
36
|
+
(.services // {} | to_entries[] | .key as $svc | .value as $s |
|
|
37
|
+
(
|
|
38
|
+
($s | keys - allowed | .[] | "\(.) is not allowed"),
|
|
39
|
+
($s.deploy // {} | keys - ["resources", "placement", "replicas"] | .[] | "deploy.\(.) is not allowed"),
|
|
40
|
+
(if ($s.deploy.resources.reservations.devices // []) | length > 0
|
|
41
|
+
then "deploy.resources.reservations.devices is not allowed" else empty end),
|
|
42
|
+
(if ($s.mem_limit // $s.deploy.resources.limits.memory) == null
|
|
43
|
+
then "no memory limit (set mem_limit)" else empty end),
|
|
44
|
+
($s.networks // {} | to_entries[] | .key as $net | .value // {} |
|
|
45
|
+
(keys - ["aliases"] | .[] | "networks.\($net).\(.) is not allowed"),
|
|
46
|
+
(select($root.networks[$net].name == "platform-public") | .aliases // [] | .[]
|
|
47
|
+
| select(startswith($app + "-") | not)
|
|
48
|
+
| "alias \(.) on platform-public must start with \($app)-")),
|
|
49
|
+
($s.volumes // [] | .[] | select(.type != "volume" and .type != "tmpfs")
|
|
50
|
+
| "\(.type) mount \(.source // "") -> \(.target) (named volumes only)"),
|
|
51
|
+
(if ($s.image // "") | registry_ref and (test("^[^/]+/" + $app + "/") | not)
|
|
52
|
+
then "image \($s.image) is outside the \($app)/ namespace" else empty end),
|
|
53
|
+
($s.labels // {} | to_entries[] |
|
|
54
|
+
(select(.key | test("^traefik\\.(tcp|udp)\\.")) | "\(.key) (HTTP routers only)"),
|
|
55
|
+
(select(.key | test("^traefik\\.http\\.(routers|services|middlewares)\\."))
|
|
56
|
+
| (.key | split(".")[3]) as $name
|
|
57
|
+
| select($name | startswith($app + "-") | not)
|
|
58
|
+
| "\(.key): Traefik names must start with \($app)-"),
|
|
59
|
+
(select(.value | test("@internal")) | "\(.key) references an @internal service"),
|
|
60
|
+
(select(.key | test("^traefik\\.http\\.routers\\.[^.]+\\.middlewares$"))
|
|
61
|
+
| select(.value | split(",") | map(gsub("\\s"; "")) | index("authelia@file"))
|
|
62
|
+
| "\(.key): authelia@file is the platform'"'"'s login; apps bring their own"),
|
|
63
|
+
(select(.key | test("^traefik\\.http\\.routers\\.[^.]+\\.rule$")) | .key as $k | .value as $rule |
|
|
64
|
+
([$rule | scan("Host\\(`([^`]*)`\\)") | .[0]]) as $claimed |
|
|
65
|
+
if ($rule | test("HostRegexp|HostSNI")) then "\($k): only Host() rules are allowed"
|
|
66
|
+
elif ($claimed | length) == 0 then "\($k): rule must name a Host()"
|
|
67
|
+
else ($claimed[] | select(. as $h | $hosts | index($h) | not)
|
|
68
|
+
| "\($k): host \(.) is not one of this app'"'"'s")
|
|
69
|
+
end)
|
|
70
|
+
)
|
|
71
|
+
) | "service \($svc): \(.)"
|
|
72
|
+
),
|
|
73
|
+
(.networks // {} | to_entries[] | .value as $n |
|
|
74
|
+
(select(($n.name // "") != "platform-public" and (own($n.name // "") | not))
|
|
75
|
+
| "network \(.key): \($n.name // "?") is neither the app'"'"'s own nor platform-public"),
|
|
76
|
+
(select($n.name == "platform-public" and $n.external != true)
|
|
77
|
+
| "network \(.key): platform-public is the platform'"'"'s, declare it external: true")),
|
|
78
|
+
(.volumes // {} | to_entries[] | .value as $v |
|
|
79
|
+
(select($v.external) | "volume \(.key): external volumes are not allowed"),
|
|
80
|
+
(select(($v.driver_opts // {}) | length > 0) | "volume \(.key): driver_opts are not allowed"),
|
|
81
|
+
(select(own($v.name // "") | not) | "volume \(.key): \($v.name) belongs to another project")),
|
|
82
|
+
(.secrets // {}, .configs // {} | to_entries[] | select(.value.file)
|
|
83
|
+
| "\(.key): file-backed secrets/configs read the host filesystem")
|
|
84
|
+
' | { violations=$(cat); [ -z "$violations" ] || { echo "$violations"; exit 1; }; }
|