@yunuseon/kayra 0.0.0-stage → 0.5.0

package/ansible/roles/backup-check/templates/backup-check.sh.j2

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Generated by Ansible (backup-check role). Hourly, from backup-check.timer.
+#
+# Opens a GitHub issue when a repository's newest snapshot is older than
+# {{ backup_check.stale_after_hours }} hours, or (with restore_test_max_days) when its last restore
+# test failed or is overdue, and closes it once that has passed. GitHub is
+# only told about changes, which state/ records. The issue comes from a GitHub
+# App, not a personal token: GitHub notifies no one about their own actions.
+set -euo pipefail
+
+state=/opt/backup-check/state
+repo={{ backup_check.github_repository | quote }}
+stale_after=$(( {{ backup_check.stale_after_hours | int }} * 3600 ))
+
+b64url() { openssl base64 -A | tr '+/' '-_' | tr -d '='; }
+
+github_token() {
+    local now header payload jwt installation
+    now=$(date +%s)
+    header=$(printf '{"alg":"RS256","typ":"JWT"}' | b64url)
+    payload=$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' $((now - 60)) $((now + 540)) {{ backup_check.github_app_id | quote }} | b64url)
+    jwt="${header}.${payload}.$(printf '%s.%s' "$header" "$payload" \
+        | openssl dgst -sha256 -sign /opt/backup-check/github-app.pem | b64url)"
+    installation=$(github "$jwt" "https://api.github.com/repos/${repo}/installation" | jq -r .id)
+    github "$jwt" -X POST "https://api.github.com/app/installations/${installation}/access_tokens" | jq -r .token
+}
+
+github() {
+    local token="$1"; shift
+    curl -fsS -H "Authorization: Bearer ${token}" -H "Accept: application/vnd.github+json" "$@"
+}
+
+# A file's Last-Modified as epoch seconds; nothing if the file is missing.
+modified() {
+    local header status
+    header=$(curl -sSI --netrc-file /opt/backup-check/netrc "$1")
+    status=$(printf '%s\n' "$header" | head -1 | cut -d' ' -f2)
+    case "$status" in
+        200) date -d "$(printf '%s\n' "$header" | sed -n 's/^last-modified: *//Ip' | tr -d '\r')" +%s ;;
+        404) ;;
+        *) echo "$1: HTTP ${status}" >&2; return 1 ;;
+    esac
+}
+
+# The newest snapshot's upload time; nothing without one. A read-only Storage
+# Box sub-account answers no PROPFIND, only Apache's directory listing: sorted
+# newest first, then that file's Last-Modified.
+newest_snapshot() {
+    local url="${1%/}/snapshots/" name
+    name=$(curl -fsS --netrc-file /opt/backup-check/netrc "${url}?C=M;O=D" \
+        | grep -oE 'href="[0-9a-f]{64}"' | head -1 | cut -d'"' -f2)
+    [ -z "$name" ] || modified "${url}${name}"
+}
+
+when() { date -d "@$1" '+%Y-%m-%d %H:%M %Z'; }
+
+# One issue per key while its problem lasts; state/<key> holds its number.
+open_issue() {
+    local key="$1" title="$2" body="$3"
+    [ ! -f "${state}/${key}" ] || return 0
+    github "$(github_token)" -X POST "https://api.github.com/repos/${repo}/issues" \
+        -d "$(jq -n --arg title "$title" --arg body "${body}
+
+cc @${repo%%/*}" '{title: $title, body: $body}')" \
+        | jq -r .number > "${state}/${key}"
+}
+
+close_issue() {
+    local key="$1" comment="$2" token issue
+    [ -f "${state}/${key}" ] || return 0
+    token=$(github_token)
+    issue="https://api.github.com/repos/${repo}/issues/$(cat "${state}/${key}")"
+    github "$token" -X POST "${issue}/comments" -d "$(jq -n --arg body "$comment" '{body: $body}')" >/dev/null
+    github "$token" -X PATCH "$issue" -d '{"state": "closed", "state_reason": "completed"}' >/dev/null
+    rm "${state}/${key}"
+}
+
+check_snapshots() {
+    local name="$1" url="$2" newest last
+    newest=$(newest_snapshot "$url")
+    if [ $(( $(date +%s) - ${newest:-0} )) -gt "$stale_after" ]; then
+        if [ -n "$newest" ]; then last="The newest snapshot is from $(when "$newest")."; else last="There is no snapshot yet."; fi
+        open_issue "$name" "Backup stale: ${name}" "No backup of \`${name}\` has reached its repository for more than {{ backup_check.stale_after_hours }} hours. ${last}
+
+Look at the backup container's log on the source. This issue closes itself when the next snapshot arrives."
+    else
+        close_issue "$name" "A new snapshot arrived."
+    fi
+}
+
+# The source uploads restore-test.passed or .failed next to the repository
+# after each restore test (restic-backup role).
+check_restore_test() {
+    local name="$1" url="${2%/}" max_days="$3" passed failed problem last
+    passed=$(modified "${url}/restore-test.passed")
+    failed=$(modified "${url}/restore-test.failed")
+    if [ "${failed:-0}" -gt "${passed:-0}" ]; then
+        problem="The last restore test of \`${name}\` failed, at $(when "$failed")."
+    elif [ $(( $(date +%s) - ${passed:-0} )) -gt $(( max_days * 86400 )) ]; then
+        if [ -n "$passed" ]; then last="The last one passed at $(when "$passed")."; else last="None has passed yet."; fi
+        problem="No restore test of \`${name}\` has passed for more than ${max_days} days. ${last}"
+    fi
+    if [ -n "${problem:-}" ]; then
+        open_issue "${name}-restore-test" "Restore test: ${name}" "${problem}
+
+Its log is log/restore-test.log in the backup directory on the source; \`kayra backup test ${name}\` runs one by hand and follows it. This issue closes itself when the next one passes."
+    else
+        close_issue "${name}-restore-test" "A restore test passed."
+    fi
+}
+
+{% for repository in backup_check.repositories %}
+check_snapshots {{ repository.name | quote }} {{ repository.url | quote }}
+{% if repository.restore_test_max_days is defined %}
+check_restore_test {{ repository.name | quote }} {{ repository.url | quote }} {{ repository.restore_test_max_days | int }}
+{% endif %}
+{% endfor %}

package/ansible/roles/common/handlers/main.yml

@@ -0,0 +1,16 @@
+---
+# =============================================================================
+# Common role — handlers
+# =============================================================================
+
+- name: Reload sysctl
+  ansible.builtin.command:
+    cmd: sysctl --system
+  become: true
+  changed_when: true
+
+- name: Restart ssh
+  ansible.builtin.systemd:
+    name: ssh
+    state: restarted
+  become: true

package/ansible/roles/common/tasks/main.yml

@@ -0,0 +1,100 @@
+---
+# =============================================================================
+# Common role — Base system setup
+# =============================================================================
+
+# A refreshed cache changes nothing on the server: not worth a changed.
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: true
+    cache_valid_time: 3600
+  become: true
+  changed_when: false
+
+- name: Install essential packages
+  ansible.builtin.apt:
+    name:
+      - ca-certificates
+      - curl
+      - python3
+      # deb822_repository in the docker role needs it on the target.
+      - python3-debian
+    state: present
+  become: true
+
+- name: Set hostname
+  ansible.builtin.hostname:
+    name: "{{ inventory_hostname }}"
+  become: true
+
+- name: Read timezone
+  ansible.builtin.command: timedatectl show --property=Timezone --value
+  register: current_timezone
+  changed_when: false
+
+- name: Set timezone
+  ansible.builtin.command: timedatectl set-timezone {{ timezone | default('Europe/Berlin') }}
+  when: current_timezone.stdout != timezone | default('Europe/Berlin')
+  changed_when: true
+  become: true
+
+- name: Harden SSH
+  ansible.builtin.copy:
+    content: |
+      PermitRootLogin no
+      PasswordAuthentication no
+    # 10- sorts before cloud-init's 50-cloud-init.conf, and sshd keeps the
+    # first value it reads for each keyword.
+    dest: /etc/ssh/sshd_config.d/10-hardening.conf
+    mode: "0644"
+  become: true
+  notify: Restart ssh
+
+# The box runs two JVMs and builds images while serving production. Swap is not
+# a substitute for RAM — it stops the OOM killer picking a random victim
+# (postgres, say) during a Jenkins build.
+- name: Create swapfile
+  ansible.builtin.shell: fallocate -l 2G /swapfile && chmod 600 /swapfile && mkswap /swapfile
+  args:
+    creates: /swapfile
+  become: true
+
+- name: Persist swapfile in fstab
+  ansible.builtin.lineinfile:
+    path: /etc/fstab
+    line: "/swapfile none swap sw 0 0"
+    regexp: '^/swapfile\s'
+  become: true
+
+- name: Enable swap
+  ansible.builtin.command: swapon --all
+  changed_when: false
+  become: true
+
+# Prefer reclaiming page cache over swapping; swap here is an emergency
+# backstop, not something we want the kernel reaching for routinely.
+- name: Set vm.swappiness
+  ansible.builtin.copy:
+    content: "vm.swappiness=10\n"
+    dest: /etc/sysctl.d/99-swappiness.conf
+    mode: "0644"
+  become: true
+  notify: Reload sysctl
+
+# Ubuntu's unattended-upgrades installs security fixes but never reboots, so a
+# patched kernel or libc sits unused until someone logs in and notices. Every
+# container is restart: unless-stopped, so a 04:00 reboot costs a minute.
+- name: Install unattended-upgrades
+  ansible.builtin.apt:
+    name: unattended-upgrades
+    state: present
+  become: true
+
+- name: Reboot after unattended upgrades when required
+  ansible.builtin.copy:
+    content: |
+      Unattended-Upgrade::Automatic-Reboot "true";
+      Unattended-Upgrade::Automatic-Reboot-Time "04:00";
+    dest: /etc/apt/apt.conf.d/52unattended-reboot
+    mode: "0644"
+  become: true

package/ansible/roles/compose-stacks/tasks/main.yml

@@ -0,0 +1,102 @@
+---
+# =============================================================================
+# compose-stacks role — runs the server's own compose stacks: for each name in
+# compose_stacks.stacks, servers/<server>/stacks/<name>/ (compose.yaml and any
+# files it mounts) goes to <dir>/<name>/, its secrets
+# (compose_stack_secrets.<name>) to a .env next to it, and `docker compose up -d` brings it to that state. Data lives wherever
+# each compose file mounts it; this role writes nothing but <dir>.
+#
+# Never deletes: a stack dropped from the list keeps running until it is
+# taken down by hand (docker compose -p <name> down).
+#
+# Private images: compose_registry_logins (in secrets.yml, a list of
+# {registry, username, password}) are logged in with a Docker config kept in
+# <dir>/.docker, which every compose call here uses (DOCKER_CONFIG) — the
+# host's own Docker config is never written.
+# =============================================================================
+
+- name: Create stack directories
+  ansible.builtin.file:
+    path: "{{ compose_stacks.dir }}/{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  loop: "{{ [''] + compose_stacks.stacks }}"
+
+# The whole stack folder: compose.yaml and any files it mounts (./…), which
+# the containers read whatever user they run as.
+- name: Copy stack files
+  ansible.builtin.copy:
+    src: "{{ server_dir }}/stacks/{{ item }}/"
+    dest: "{{ compose_stacks.dir }}/{{ item }}/"
+    mode: "0644"
+    directory_mode: "0755"
+  register: stack_files
+  loop: "{{ compose_stacks.stacks }}"
+
+# Single-quoted: Compose reads those literally, without escapes or ${…}.
+- name: Assert secrets fit in a .env
+  ansible.builtin.assert:
+    that: "\"'\" not in item.value | string and '\n' not in item.value | string"
+    fail_msg: "a secret in compose_stack_secrets contains a single quote or newline"
+    quiet: true
+  loop: "{{ compose_stack_secrets | default({}) | dict2items | map(attribute='value') | map('dict2items') | flatten }}"
+  loop_control:
+    label: "{{ item.key }}"
+  no_log: true
+
+- name: Render secrets
+  ansible.builtin.copy:
+    content: |
+      {% for key, value in compose_stack_secrets[item].items() %}
+      {{ key }}='{{ value }}'
+      {% endfor %}
+    dest: "{{ compose_stacks.dir }}/{{ item }}/.env"
+    mode: "0600"
+  loop: "{{ compose_stacks.stacks | select('in', compose_stack_secrets | default({})) }}"
+  no_log: true
+
+- name: Create the stacks' Docker config directory
+  ansible.builtin.file:
+    path: "{{ compose_stacks.dir }}/.docker"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  when: compose_registry_logins | default([]) | length > 0
+
+- name: Log in to private registries
+  community.docker.docker_login:
+    registry_url: "{{ item.registry }}"
+    username: "{{ item.username }}"
+    password: "{{ item.password }}"
+    config_path: "{{ compose_stacks.dir }}/.docker/config.json"
+  loop: "{{ compose_registry_logins | default([]) }}"
+  loop_control: {label: "{{ item.registry }}"}
+  no_log: true
+
+- name: Start stacks
+  environment:
+    DOCKER_CONFIG: "{{ compose_stacks.dir }}/.docker"
+  community.docker.docker_compose_v2:
+    project_src: "{{ compose_stacks.dir }}/{{ item }}"
+    project_name: "{{ item }}"
+    files: [compose.yaml]
+  register: compose_up
+  loop: "{{ compose_stacks.stacks }}"
+
+# `up` recreates a container only when its compose definition changed, not
+# when a file it mounts did (a Caddyfile, Homepage's config): those take
+# effect on a restart.
+- name: Restart stacks whose files changed
+  environment:
+    DOCKER_CONFIG: "{{ compose_stacks.dir }}/.docker"
+  ansible.builtin.command: docker compose --project-name {{ item.item }} --file compose.yaml restart
+  args:
+    chdir: "{{ compose_stacks.dir }}/{{ item.item }}"
+  changed_when: true
+  when: item.changed and not (compose_up.results | selectattr('item', 'eq', item.item) | first).changed
+  loop: "{{ stack_files.results }}"
+  loop_control:
+    label: "{{ item.item }}"

package/ansible/roles/docker/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# apt version pattern for docker-ce/docker-ce-cli: a major, held.
+docker_version: "5:29*"

package/ansible/roles/docker/files/buildkit-rootless

@@ -0,0 +1,23 @@
+# AppArmor profile for the rootless BuildKit container.
+#
+# Ubuntu 23.10+ refuses unprivileged user namespaces to processes running
+# unconfined (kernel.apparmor_restrict_unprivileged_userns=1) — which is
+# precisely what rootlesskit needs in order to start. BuildKit's own docs
+# suggest turning that sysctl off; this grants the capability to one container
+# instead, and leaves the protection in place for everything else on the host.
+#
+# flags=(unconfined) keeps the permissive behaviour BuildKit requires (mount,
+# unshare, and friends). Being a *named* profile rather than genuinely
+# unconfined is what makes the userns grant possible at all — the restriction
+# applies only to unconfined processes.
+#
+# Compose applies it by name: security_opt: apparmor=buildkit-rootless.
+abi <abi/4.0>,
+
+include <tunables/global>
+
+profile buildkit-rootless flags=(unconfined) {
+  userns,
+
+  include if exists <local/buildkit-rootless>
+}

package/ansible/roles/docker/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Restart Docker
+  ansible.builtin.systemd:
+    name: docker
+    state: restarted
+  become: true
+
+- name: Reload AppArmor profiles
+  ansible.builtin.command:
+    cmd: apparmor_parser -r -W /etc/apparmor.d/buildkit-rootless
+  changed_when: true
+  become: true

package/ansible/roles/docker/tasks/main.yml

@@ -0,0 +1,82 @@
+---
+# =============================================================================
+# Docker role — Install Docker CE + Compose plugin
+# =============================================================================
+
+- name: Add Docker apt repository
+  ansible.builtin.deb822_repository:
+    name: docker
+    uris: https://download.docker.com/linux/ubuntu
+    suites: "{{ ansible_facts.distribution_release }}"
+    components: stable
+    signed_by: https://download.docker.com/linux/ubuntu/gpg
+  become: true
+
+# Pinned to a major and held, so no apt upgrade, unattended or manual, moves
+# it; a new major is a tested change of docker_version.
+- name: Install Docker CE {{ docker_version }}
+  ansible.builtin.apt:
+    name:
+      - docker-ce={{ docker_version }}
+      - docker-ce-cli={{ docker_version }}
+      - containerd.io
+      - docker-compose-plugin
+    update_cache: true
+    allow_change_held_packages: true
+  become: true
+
+- name: Hold Docker
+  ansible.builtin.dpkg_selections:
+    name: "{{ item }}"
+    selection: hold
+  loop: [docker-ce, docker-ce-cli]
+  become: true
+
+# Unrotated json-file logs grow until the disk is full. `local` is compressed
+# and rotated; it applies to containers created after the change.
+- name: Configure the Docker daemon
+  ansible.builtin.copy:
+    content: "{{ {'log-driver': 'local', 'log-opts': {'max-size': '10m', 'max-file': '3'}} | to_nice_json }}\n"
+    dest: /etc/docker/daemon.json
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+  notify: Restart Docker
+
+# Now, not at the end of the play: in a single-play playbook (edge) the
+# containers started later in the play would otherwise be created before the
+# daemon picks up its config, and keep the old log driver.
+- name: Apply the daemon config before any container starts
+  ansible.builtin.meta: flush_handlers
+
+- name: Ensure Docker is running
+  ansible.builtin.systemd:
+    name: docker
+    state: started
+    enabled: true
+  become: true
+
+- name: Add deploy user to docker group
+  ansible.builtin.user:
+    name: deploy
+    groups: docker
+    append: true
+  become: true
+
+# Group membership only applies to new logins, and Ansible reuses its SSH
+# connection — without this the platform play cannot reach the docker socket.
+- name: Reconnect so the docker group takes effect
+  ansible.builtin.meta: reset_connection
+
+# Must be loaded before the platform play starts buildkitd — handlers flush at
+# the end of this play, which is the bootstrap one, so the ordering holds.
+- name: Install AppArmor profile for rootless BuildKit
+  ansible.builtin.copy:
+    src: buildkit-rootless
+    dest: /etc/apparmor.d/buildkit-rootless
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+  notify: Reload AppArmor profiles

package/ansible/roles/edge-proxy/files/docker-compose.yml

@@ -0,0 +1,13 @@
+# Managed by Ansible (edge-proxy role).
+services:
+  caddy:
+    image: caddy:2.11.7
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+      - "443:443/udp"
+    volumes:
+      - ./caddy:/etc/caddy:ro
+      - ./data:/data
+      - ./config:/config

package/ansible/roles/edge-proxy/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Reload Caddy
+  ansible.builtin.command: docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile
+  args:
+    chdir: /opt/edge-proxy
+  changed_when: true

package/ansible/roles/edge-proxy/tasks/main.yml

@@ -0,0 +1,34 @@
+---
+# =============================================================================
+# Edge proxy role — Caddy in front of whatever edge_routes point at (upstreams
+# on the far side of the WireGuard tunnel). Caddy gets its certs itself.
+# =============================================================================
+
+- name: Create edge proxy directories
+  ansible.builtin.file:
+    path: "/opt/edge-proxy/{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  loop: ["", caddy, data, config]
+
+- name: Copy compose file
+  ansible.builtin.copy:
+    src: docker-compose.yml
+    dest: /opt/edge-proxy/docker-compose.yml
+    mode: "0644"
+
+# Into caddy/, which is mounted as a directory: Ansible replaces files
+# atomically (new inode), and a single-file bind mount would keep showing the
+# container the old one.
+- name: Render Caddyfile
+  ansible.builtin.template:
+    src: Caddyfile.j2
+    dest: /opt/edge-proxy/caddy/Caddyfile
+    mode: "0644"
+  notify: Reload Caddy
+
+- name: Start Caddy
+  community.docker.docker_compose_v2:
+    project_src: /opt/edge-proxy

package/ansible/roles/edge-proxy/templates/Caddyfile.j2

@@ -0,0 +1,25 @@
+# Generated by Ansible from edge_routes in servers/<server>/vars.yml.
+{
+	email {{ acme_email }}
+
+	# Avoid QUIC/HTTP3 surprises on corporate networks.
+	servers {
+		protocols h1 h2
+	}
+}
+{% for route in edge_routes %}
+
+{{ route.subdomains | map('regex_replace', '$', '.' ~ domain) | join(',\n') }} {
+	encode zstd gzip
+{% if route.forwarded_headers | default(false) %}
+	reverse_proxy {{ route.upstream }} {
+		header_up Host {host}
+		header_up X-Forwarded-Proto https
+		header_up X-Forwarded-For {remote_host}
+		header_up X-Forwarded-Port 443
+	}
+{% else %}
+	reverse_proxy {{ route.upstream }}
+{% endif %}
+}
+{% endfor %}