@yunuseon/kayra 0.0.0-stage → 0.5.0

package/ansible/roles/platform/tasks/main.yml

@@ -0,0 +1,298 @@
+---
+# =============================================================================
+# Platform role — Traefik, Authelia, Registry, BuildKit and Jenkins as compose
+# project "platform" in {{ platform_dir }}. App-agnostic.
+#
+# {{ platform_dir }}/.env sets COMPOSE_FILE, so on the server every compose
+# command is just `docker compose ...` run in {{ platform_dir }}.
+# =============================================================================
+
+- name: Check Renovate has a token
+  ansible.builtin.assert:
+    that: renovate_token | length > 0 or 'RENOVATE_TOKEN' in (forgejo_config.tokens | default([]) | map(attribute='secret'))
+    fail_msg: renovate_repositories is set, so renovate_token must be set in servers/{{ server }}/secrets.yml
+  when: renovate_repositories | length > 0
+
+- name: Create platform directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: deploy
+    group: deploy
+    mode: "0755"
+  loop:
+    - "{{ apps_dir }}"
+    - "{{ platform_dir }}"
+    - "{{ platform_dir }}/traefik/dynamic"
+    - "{{ platform_dir }}/traefik/certs"
+    - "{{ platform_dir }}/authelia/secrets"
+  become: true
+
+# Written by bin/jenkins-app-apply as root and chowned per app to the
+# controller's uid, so the directory itself stays root's.
+- name: Create Jenkins app secrets directory
+  ansible.builtin.file:
+    path: "{{ platform_dir }}/jenkins-secrets"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  become: true
+
+# buildkit/ matters more than it looks: without buildkitd.toml the bind mount
+# has no source, Docker creates a directory in its place, and buildkitd
+# crash-loops until the container is recreated.
+- name: Copy platform files
+  ansible.builtin.copy:
+    src: "{{ kayra_root }}/compose/{{ item }}"
+    dest: "{{ platform_dir }}/{{ item }}"
+    owner: deploy
+    group: deploy
+    mode: "0644"
+  loop:
+    - docker-compose.yml
+    - docker-compose.ci.yml
+    - docker-compose.acme-dns.yml
+    - traefik/dynamic/
+    - authelia/configuration.yml
+    - jenkins/
+    - jenkins-agent/
+    - buildkit/
+  become: true
+  register: platform_files
+
+# Apps register with Jenkins through these, never by touching the controller.
+- name: Install platform scripts
+  ansible.builtin.copy:
+    src: "{{ kayra_root }}/scripts/server/"
+    dest: "{{ platform_dir }}/bin/"
+    owner: root
+    group: root
+    mode: "0755"
+  become: true
+
+- name: Render Traefik static config
+  ansible.builtin.copy:
+    content: "{{ (traefik_static_dns if acme_dns else traefik_static) ~ '\n' }}"
+    dest: "{{ platform_dir }}/traefik/traefik.yml"
+    owner: deploy
+    group: deploy
+    mode: "0644"
+  vars:
+    # Traefik does not expand env vars in its static file.
+    traefik_static: "{{ lookup('file', kayra_root ~ '/compose/traefik/traefik.yml') | regex_replace('(?m)^(\\s+email:).*$', '\\1 \"' ~ acme_email ~ '\"') }}"
+    # DNS-01: the challenge instead of HTTP-01, checked with public resolvers
+    # (a home server's own DNS may answer its zone itself), a minute after the
+    # record is set: Hetzner's three nameservers take that long to agree, and
+    # Let's Encrypt asks more than one.
+    traefik_static_dns: >-
+      {{ traefik_static | from_yaml
+         | combine({'certificatesResolvers': {'le': {'acme': {
+             'email': acme_email, 'storage': '/acme/acme.json',
+             'dnsChallenge': {'provider': acme_dns, 'resolvers': ['8.8.8.8:53', '9.9.9.9:53'],
+                              'propagation': {'delayBeforeChecks': '60s'}}}}}})
+         | to_nice_yaml(indent=2) }}
+  become: true
+  register: traefik_static_config
+
+# Vault values rendered to files rather than .env lines, which Authelia reads
+# via *_FILE. Authelia never writes the user database back — password change
+# and reset are disabled — so rendering it every run is safe.
+- name: Render Authelia user database
+  ansible.builtin.template:
+    src: authelia-users.yml.j2
+    dest: "{{ platform_dir }}/authelia/users_database.yml"
+    owner: deploy
+    group: deploy
+    mode: "0600"
+  become: true
+
+- name: Render Authelia secrets
+  ansible.builtin.copy:
+    content: "{{ item.value }}"
+    dest: "{{ platform_dir }}/authelia/secrets/{{ item.name }}"
+    owner: deploy
+    group: deploy
+    mode: "0600"
+  loop:
+    - { name: session, value: "{{ authelia_session_secret }}" }
+    - { name: storage_encryption, value: "{{ authelia_storage_encryption_key }}" }
+    - { name: smtp_password, value: "{{ authelia_smtp_password }}" }
+  loop_control:
+    label: "{{ item.name }}"
+  when: item.name != 'smtp_password' or authelia_smtp_address | length > 0
+  no_log: true
+  become: true
+
+# Servers not on the internet (multipass, home) without acme_dns. Elsewhere
+# Traefik's ACME resolver issues the certs, and it skips ACME for any hostname
+# a file-provided cert already covers — a stray mkcert cert would be served
+# as-is, hence removed below.
+- name: Copy mkcert TLS cert + key
+  ansible.builtin.copy:
+    src: "{{ fleet_root }}/.cache/certs/{{ item.name }}"
+    dest: "{{ platform_dir }}/traefik/certs/{{ item.name }}"
+    owner: deploy
+    group: deploy
+    mode: "{{ item.mode }}"
+  loop:
+    - { name: cert.pem, mode: "0644" }
+    - { name: key.pem, mode: "0600" }
+  when: target != 'hetzner' and not acme_dns
+  become: true
+
+- name: Remove a mkcert cert where DNS-01 issues them
+  ansible.builtin.file:
+    path: "{{ platform_dir }}/traefik/certs/{{ item }}"
+    state: absent
+  loop: [cert.pem, key.pem]
+  when: acme_dns | length > 0
+  become: true
+
+# Read by lego through HETZNER_API_TOKEN_FILE (docker-compose.acme-dns.yml).
+- name: Create the DNS-01 token directory
+  ansible.builtin.file:
+    path: "{{ platform_dir }}/traefik/secrets"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  when: acme_dns | length > 0
+  become: true
+
+- name: Render the DNS-01 token
+  ansible.builtin.copy:
+    content: "{{ acme_dns_token }}"
+    dest: "{{ platform_dir }}/traefik/secrets/acme-dns-token"
+    owner: root
+    group: root
+    mode: "0600"
+  when: acme_dns | length > 0
+  no_log: true
+  become: true
+
+- name: Look up the docker group
+  ansible.builtin.getent:
+    database: group
+    key: docker
+
+# Read by the controller as uid 1000. An empty var leaves the file as it is:
+# on a server whose Forgejo mints them (forgejo_config.tokens), that role
+# writes them.
+- name: Create platform secrets directory
+  ansible.builtin.file:
+    path: "{{ platform_dir }}/platform-secrets"
+    state: directory
+    owner: "1000"
+    group: "1000"
+    mode: "0500"
+  become: true
+
+- name: Render platform tokens
+  ansible.builtin.copy:
+    content: "{{ item.value }}"
+    dest: "{{ platform_dir }}/platform-secrets/{{ item.name }}"
+    owner: "1000"
+    group: "1000"
+    mode: "0400"
+    force: "{{ item.value | length > 0 }}"
+  loop:
+    - {name: GIT_TOKEN, value: "{{ git_token }}"}
+    - {name: RENOVATE_TOKEN, value: "{{ renovate_token }}"}
+  loop_control:
+    label: "{{ item.name }}"
+  no_log: true
+  become: true
+  register: platform_tokens
+
+- name: Render platform .env
+  ansible.builtin.template:
+    src: platform.env.j2
+    dest: "{{ platform_dir }}/.env"
+    owner: deploy
+    group: deploy
+    mode: "0600"
+  become: true
+
+# The registry used to have a public, basic-auth router. Nothing outside the
+# server pushes or pulls, so it is gone; the copy above never deletes files.
+- name: Remove the retired registry basic auth
+  ansible.builtin.file:
+    path: "{{ platform_dir }}/traefik/dynamic/{{ item }}"
+    state: absent
+  loop: [registry-auth.yml, registry-htpasswd]
+  become: true
+
+# ---------------------------------------------------------------------------
+# Start. `up` recreates any container whose interpolated config changed, so
+# a changed .env needs nothing extra.
+# ---------------------------------------------------------------------------
+- name: Pull platform images
+  community.docker.docker_compose_v2_pull:
+    project_src: "{{ platform_dir }}"
+    ignore_buildable: true
+
+- name: Start platform
+  community.docker.docker_compose_v2:
+    project_src: "{{ platform_dir }}"
+    build: always
+
+# Only dynamic/ is watched; the static file is read once at startup.
+- name: Restart Traefik when its static config changed
+  ansible.builtin.command: docker compose restart traefik
+  args:
+    chdir: "{{ platform_dir }}"
+  when: traefik_static_config is changed
+  changed_when: true
+
+# Inside the controller, not through Traefik: jenkins.<domain> sits behind
+# Authelia, which would answer every call here with a login redirect.
+- name: Wait for Jenkins
+  ansible.builtin.command: docker compose exec -T jenkins-controller curl -sf -o /dev/null http://127.0.0.1:8080/login
+  args:
+    chdir: "{{ platform_dir }}"
+  register: jenkins_health
+  retries: 30
+  delay: 10
+  until: jenkins_health.rc == 0
+  changed_when: false
+
+# casc/ is a bind mount, so a changed 00-platform.yaml only takes effect on
+# a reload; so do new tokens, read from files by ${readFile:…}.
+- name: Reload JCasC
+  ansible.builtin.command: "{{ platform_dir }}/bin/jenkins-reload"
+  when: >-
+    platform_files.results | selectattr('item', 'equalto', 'jenkins/') | selectattr('changed') | list | length > 0
+    or platform_tokens is changed
+  become: true
+  changed_when: true
+
+# ---------------------------------------------------------------------------
+# Agent secret. Fetched every run and kept in its own file: it is derived
+# from Jenkins' own state, so it has no var source, and re-rendering .env must
+# never be able to blank it. Refetching also heals a reset jenkins_home.
+# ---------------------------------------------------------------------------
+- name: Fetch Jenkins agent secret
+  ansible.builtin.command: "{{ platform_dir }}/bin/jenkins-api GET /computer/builder/jenkins-agent.jnlp"
+  register: agent_jnlp
+  no_log: true
+  changed_when: false
+  check_mode: false
+
+- name: Write Jenkins agent secret
+  ansible.builtin.copy:
+    content: "JENKINS_SECRET={{ agent_jnlp.stdout | regex_findall('<argument>([a-f0-9]{64})</argument>') | first }}\n"
+    dest: "{{ platform_dir }}/jenkins-agent.env"
+    owner: deploy
+    group: deploy
+    mode: "0600"
+  no_log: true
+  become: true
+  register: agent_secret
+
+- name: Recreate Jenkins agent with its secret
+  ansible.builtin.command: docker compose up -d --no-deps --force-recreate jenkins-agent
+  args:
+    chdir: "{{ platform_dir }}"
+  when: agent_secret is changed
+  changed_when: true

package/ansible/roles/platform/templates/authelia-users.yml.j2

@@ -0,0 +1,17 @@
+# =============================================================================
+# Authelia user database — rendered by Ansible from authelia_users.
+# Do not edit on the server: this file is regenerated on every platform run.
+#
+# Password hashes are argon2id, generated with:
+#   docker run --rm authelia/authelia:4.39 authelia crypto hash generate argon2 \
+#     --variant argon2id --iterations 3 --memory 32768 --parallelism 2
+# =============================================================================
+users:
+{% for username, user in authelia_users.items() %}
+  {{ username }}:
+    disabled: {{ user.disabled | default(false) | lower }}
+    displayname: '{{ user.displayname }}'
+    password: '{{ user.password }}'
+    email: '{{ user.email }}'
+    groups: [{% for group in user.groups %}'{{ group }}'{% if not loop.last %}, {% endif %}{% endfor %}]
+{% endfor %}

package/ansible/roles/platform/templates/platform.env.j2

@@ -0,0 +1,41 @@
+# =============================================================================
+# {{ platform_dir }}/.env — generated by Ansible, do not edit on the server.
+#
+# Compose reads it automatically in {{ platform_dir }}, and COMPOSE_FILE means
+# `docker compose <cmd>` there needs no -f flags. App-agnostic: app secrets
+# reach Jenkins through bin/jenkins-app-apply, app config lives in
+# {{ apps_dir }}/<app>/.env. The agent secret lives in jenkins-agent.env.
+# =============================================================================
+
+COMPOSE_PROJECT_NAME=platform
+COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml{{ ':docker-compose.acme-dns.yml' if acme_dns else '' }}
+DOMAIN={{ domain }}
+AUTH_POLICY={{ authelia_policy }}
+APPS_DIR={{ apps_dir }}
+DOCKER_GID={{ ansible_facts.getent_group.docker[1] }}
+# Branch the pipeline library and platform jobs are loaded from.
+PLATFORM_REF={{ platform_ref }}
+PLATFORM_RENOVATE={{ (renovate_repositories | length > 0) | lower }}
+RENOVATE_SCHEDULE={{ renovate_schedule }}
+RENOVATE_PLATFORM={{ renovate_platform }}
+RENOVATE_ENDPOINT={{ renovate_endpoint }}
+RENOVATE_GIT_URL={{ renovate_git_url }}
+RENOVATE_GITHUB_COM_TOKEN={{ renovate_github_com_token }}
+RENOVATE_HOST_RULES={{ renovate_host_rules | to_json if renovate_host_rules else '' }}
+RENOVATE_REPOSITORIES={{ renovate_repositories | to_json }}
+
+# Jenkins
+JENKINS_ADMIN_USER={{ jenkins_admin_user }}
+JENKINS_ADMIN_PASSWORD={{ jenkins_admin_password }}
+
+# Git
+GIT_REMOTE_URL={{ git_remote_url }}
+GITHUB_WEBHOOK_SECRET={{ github_webhook_secret }}
+
+# Authelia SMTP. Empty address falls the notifier back to a file — passkey
+# enrolment then needs the code read off the server. The password is NOT here:
+# it is rendered to authelia/secrets/smtp_password, because an empty
+# AUTHELIA_NOTIFIER_SMTP_PASSWORD would make Authelia think SMTP is configured.
+AUTHELIA_SMTP_ADDRESS={{ authelia_smtp_address }}
+AUTHELIA_SMTP_USERNAME={{ authelia_smtp_username }}
+AUTHELIA_SMTP_SENDER={{ authelia_smtp_sender }}

package/ansible/roles/restic-backup/files/docker-compose.postgres.yml

@@ -0,0 +1,27 @@
+# Managed by Ansible (restic-backup role), where restic_backup.postgres lists
+# databases to dump: copied as docker-compose.yml. Mounts and networks, which
+# differ per server, are in docker-compose.override.yml next to it.
+services:
+  # Hands its static restic binary to the backup container and exits.
+  restic:
+    image: restic/restic:0.19.1
+    entrypoint: ["cp", "/usr/bin/restic", "/opt/restic/restic"]
+    volumes:
+      - restic-bin:/opt/restic
+
+  # Postgres for pg_dump, busybox crond for the schedule. A newer pg_dump
+  # reads older servers, so its major only has to be the highest dumped.
+  backup:
+    image: postgres:18.6-alpine3.24
+    restart: unless-stopped
+    entrypoint: ["crond", "-f", "-d", "8"]
+    depends_on:
+      restic:
+        condition: service_completed_successfully
+    volumes:
+      - restic-bin:/opt/restic:ro
+      - ./config:/etc/backup:ro
+      - ./cron:/etc/crontabs:ro
+
+volumes:
+  restic-bin:

package/ansible/roles/restic-backup/files/docker-compose.yml

@@ -0,0 +1,12 @@
+# Managed by Ansible (restic-backup role). Mounts and networks, which differ
+# per server, are in docker-compose.override.yml next to it. Where databases
+# are dumped first, docker-compose.postgres.yml takes this file's place.
+services:
+  # restic's image with busybox crond for the schedule.
+  backup:
+    image: restic/restic:0.19.1
+    restart: unless-stopped
+    entrypoint: ["crond", "-f", "-d", "8"]
+    volumes:
+      - ./config:/etc/backup:ro
+      - ./cron:/etc/crontabs:ro

package/ansible/roles/restic-backup/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+# busybox crond reads its crontab at start only.
+- name: Restart backup
+  ansible.builtin.command: docker compose restart backup
+  args:
+    chdir: "{{ restic_backup.dir }}"
+  changed_when: true

package/ansible/roles/restic-backup/tasks/main.yml

@@ -0,0 +1,121 @@
+---
+# =============================================================================
+# restic-backup role — backs restic_backup.paths up to a restic repository (a
+# REST server, or SFTP with restic_backup.ssh), nightly, from a compose stack in restic_backup.dir. Postgres databases are
+# dumped first. Nothing outside restic_backup.dir is written, and the paths
+# are mounted read-only.
+#
+# By hand, on the server:
+#   docker compose -f <dir>/docker-compose.yml exec backup /etc/backup/backup.sh
+#   docker compose -f <dir>/docker-compose.yml logs backup
+# Restore test: kayra backup test <server> (starts one, follows its log)
+# =============================================================================
+
+- name: Assert every database has a password
+  ansible.builtin.assert:
+    that:
+      - item.name is match('^[a-z0-9_]+$')
+      - restic_backup_postgres_passwords[item.name] | default('') | length > 0
+    fail_msg: "postgres dump {{ item.name }}: name must be [a-z0-9_], and restic_backup_postgres_passwords.{{ item.name }} set in secrets.yml"
+    quiet: true
+  loop: "{{ restic_backup.postgres | default([]) }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+# Traversable, and log/ readable: the operator follows a restore test over
+# SSH as their own user (kayra backup test). Everything else stays root's.
+- name: Create the backup directory
+  ansible.builtin.file:
+    path: "{{ restic_backup.dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0711"
+
+- name: Create backup directories
+  ansible.builtin.file:
+    path: "{{ restic_backup.dir }}/{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "{{ '0755' if item == 'log' else '0700' }}"
+  loop: [config, cron, dumps, cache, restore-test, ssh, log]
+
+# With databases to dump, Postgres (pg_dump) with restic copied in; without,
+# restic's own image.
+- name: Copy compose file
+  ansible.builtin.copy:
+    src: "{{ 'docker-compose.postgres.yml' if restic_backup.postgres | default([]) else 'docker-compose.yml' }}"
+    dest: "{{ restic_backup.dir }}/docker-compose.yml"
+    mode: "0600"
+
+- name: Render compose override
+  ansible.builtin.template:
+    src: docker-compose.override.yml.j2
+    dest: "{{ restic_backup.dir }}/docker-compose.override.yml"
+    mode: "0600"
+
+- name: Render environment
+  ansible.builtin.template:
+    src: backup.env.j2
+    dest: "{{ restic_backup.dir }}/config/backup.env"
+    mode: "0600"
+  no_log: true
+
+- name: Render scripts
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "{{ restic_backup.dir }}/config/{{ item }}"
+    mode: "0700"
+  loop: [backup.sh, restore-test.sh]
+
+# On the host, not in the container: the operator's SSH user runs it.
+- name: Render the restore test's follower
+  ansible.builtin.template:
+    src: follow-restore-test.sh.j2
+    dest: "{{ restic_backup.dir }}/follow-restore-test.sh"
+    owner: root
+    group: root
+    mode: "0755"
+
+# For an SFTP repository: the target's alias, its pinned host key, our key.
+- name: Render SSH config
+  ansible.builtin.template:
+    src: ssh_config.j2
+    dest: "{{ restic_backup.dir }}/ssh/config"
+    mode: "0600"
+  when: restic_backup.ssh is defined
+
+- name: Pin the target's host key
+  ansible.builtin.copy:
+    content: "{{ restic_backup.ssh.known_host }}\n"
+    dest: "{{ restic_backup.dir }}/ssh/known_hosts"
+    mode: "0600"
+  when: restic_backup.ssh is defined
+
+- name: Install the backup's SSH key
+  ansible.builtin.copy:
+    content: "{{ restic_backup_ssh_key | trim }}\n"
+    dest: "{{ restic_backup.dir }}/ssh/id_ed25519"
+    mode: "0600"
+  when: restic_backup.ssh is defined
+  no_log: true
+
+- name: Render exclude list
+  ansible.builtin.copy:
+    content: "{{ restic_backup.exclude | default([]) | join('\n') }}\n"
+    dest: "{{ restic_backup.dir }}/config/excludes.txt"
+    mode: "0600"
+
+# Into cron/, which is mounted as a directory: a single-file bind mount would
+# keep showing the container the file Ansible just replaced.
+- name: Render crontab
+  ansible.builtin.template:
+    src: root.crontab.j2
+    dest: "{{ restic_backup.dir }}/cron/root"
+    mode: "0600"
+  notify: Restart backup
+
+- name: Start backup stack
+  community.docker.docker_compose_v2:
+    project_src: "{{ restic_backup.dir }}"

package/ansible/roles/restic-backup/templates/backup.env.j2

@@ -0,0 +1,9 @@
+# Generated by Ansible (restic-backup role). Sourced by the scripts: cron
+# jobs do not inherit the container's environment.
+export PATH=/opt/restic:/usr/local/bin:/usr/bin:/bin
+export RESTIC_REPOSITORY={{ restic_backup.repository | quote }}
+export RESTIC_PASSWORD={{ restic_backup_password | quote }}
+export RESTIC_CACHE_DIR={{ (restic_backup.dir ~ '/cache') | quote }}
+{% for db in restic_backup.postgres | default([]) %}
+export PG_PASSWORD_{{ db.name | upper }}={{ restic_backup_postgres_passwords[db.name] | quote }}
+{% endfor %}

package/ansible/roles/restic-backup/templates/backup.sh.j2

@@ -0,0 +1,28 @@
+#!/bin/sh
+# Generated by Ansible (restic-backup role). Run by cron in the backup
+# container: dump the databases, back everything up, and on Sundays forget
+# and prune what restic_backup.keep no longer covers (where it is set; an
+# append-only repository is pruned on the receiving side instead).
+set -eu
+. /etc/backup/backup.env
+
+dumps='{{ restic_backup.dir }}/dumps'
+{% for db in restic_backup.postgres | default([]) %}
+PGPASSWORD="$PG_PASSWORD_{{ db.name | upper }}" pg_dump --format=custom \
+    --host={{ db.host }} --username={{ db.user }} --dbname={{ db.database }} \
+    --file="${dumps}/{{ db.name }}.dump.tmp"
+# UGOS's ACLs override the umask: new files would be world-readable.
+chmod 600 "${dumps}/{{ db.name }}.dump.tmp"
+mv "${dumps}/{{ db.name }}.dump.tmp" "${dumps}/{{ db.name }}.dump"
+{% endfor %}
+
+restic cat config >/dev/null 2>&1 || restic init
+restic backup --host '{{ inventory_hostname }}' --exclude-file /etc/backup/excludes.txt \
+    {{ restic_backup.paths | map('quote') | join(' ') }}
+{% if restic_backup.keep is defined %}
+
+if [ "$(date +%u)" = 7 ]; then
+    restic forget --prune --host '{{ inventory_hostname }}'{% for period, count in restic_backup.keep.items() %} --keep-{{ period }} {{ count }}{% endfor %}
+
+fi
+{% endif %}

package/ansible/roles/restic-backup/templates/docker-compose.override.yml.j2

@@ -0,0 +1,26 @@
+# Generated by Ansible from restic_backup in servers/<server>/vars.yml.
+{% set networks = restic_backup.postgres | default([]) | map(attribute='network') | unique | list %}
+services:
+  backup:
+    volumes:
+{% for path in restic_backup.paths %}
+      - {{ path }}:{{ path }}:ro
+{% endfor %}
+{% for dir in ['dumps', 'cache', 'restore-test', 'log'] %}
+      - {{ restic_backup.dir }}/{{ dir }}:{{ restic_backup.dir }}/{{ dir }}
+{% endfor %}
+{% if restic_backup.ssh is defined %}
+      - {{ restic_backup.dir }}/ssh:/root/.ssh:ro
+{% endif %}
+{% if networks %}
+    networks:
+{% for network in networks %}
+      - {{ network }}
+{% endfor %}
+
+networks:
+{% for network in networks %}
+  {{ network }}:
+    external: true
+{% endfor %}
+{% endif %}

package/ansible/roles/restic-backup/templates/follow-restore-test.sh.j2

@@ -0,0 +1,28 @@
+#!/bin/sh
+# Generated by Ansible (restic-backup role). Follows the restore test that is
+# running, or about to: its log until it ends, then exits with its status.
+# Runs on the host as the operator's SSH user (kayra backup test), so it reads
+# only log/. Stopping it leaves the restore test running.
+set -eu
+
+log='{{ restic_backup.dir }}/log'
+lock="${log}/restore-test.lock"
+
+# A test that runs holds the lock, and removed the last one's status only
+# after starting its log. It was just started, so wait for that.
+waited=0
+until [ -e "$lock" ] && ! flock -n "$lock" true && [ ! -e "${log}/restore-test.status" ]; do
+    if [ "$waited" -ge 30 ]; then
+        echo "No restore test started; its first lines go to the backup container's log." >&2
+        exit 1
+    fi
+    waited=$((waited + 1))
+    sleep 1
+done
+
+tail -n +1 -F "${log}/restore-test.log" &
+follower=$!
+flock "$lock" true
+sleep 1
+kill "$follower"
+exit "$(cat "${log}/restore-test.status")"

package/ansible/roles/restic-backup/templates/restore-test.sh.j2

@@ -0,0 +1,51 @@
+#!/bin/sh
+# Generated by Ansible (restic-backup role). Proves the latest snapshot comes
+# back: checks the repository, restores it (or restic_backup.restore_test_paths
+# of it) below restore-test/, and has
+# pg_restore read every database dump. Deletes the restored copy afterwards.
+# Started by cron (restore_test_schedule) or by kayra backup test, which
+# follows its log with follow-restore-test.sh. One at a time.
+set -eu
+. /etc/backup/backup.env
+
+log='{{ restic_backup.dir }}/log'
+exec 9>"${log}/restore-test.lock"
+if ! flock -n 9; then
+    echo "A restore test is already running: ${log}/restore-test.log" >&2
+    exit 75
+fi
+# The log first, then the old status: a follower waits for both to be fresh.
+exec >"${log}/restore-test.log" 2>&1
+rm -f "${log}/restore-test.status"
+# restic leaves progress out without a terminal; every 10 s instead.
+export RESTIC_PROGRESS_FPS=0.1
+
+result=failed
+finish() {
+    status=$?
+    echo "$status" > "${log}/restore-test.status"
+{% if restic_backup.ssh is defined %}
+    # The result, next to the repository, for the backup-check role to read.
+    date -u '+%Y-%m-%dT%H:%M:%SZ' > "/tmp/restore-test.${result}"
+    echo "put /tmp/restore-test.${result} {{ restic_backup.repository.split(':', 2)[2] }}/restore-test.${result}" \
+        | sftp -q -b - backup-target
+{% endif %}
+}
+trap finish EXIT
+
+target='{{ restic_backup.dir }}/restore-test'
+find "$target" -mindepth 1 -delete
+
+# Waits out a backup's or prune's lock; reads a twelfth of the data, a
+# different one each month, so all of it once a year.
+restic check --retry-lock 1h --read-data-subset "$(date +%m | sed 's/^0//')/12"
+restic restore latest --host '{{ inventory_hostname }}' --target "$target"{% for path in restic_backup.restore_test_paths | default([]) %} --include {{ path | quote }}{% endfor %}
+
+{% for db in restic_backup.postgres | default([]) %}
+pg_restore --list "${target}{{ restic_backup.dir }}/dumps/{{ db.name }}.dump" >/dev/null
+echo "{{ db.name }}: dump readable"
+{% endfor %}
+echo "restored $(find "$target" -type f | wc -l) files, $(du -sh "$target" | cut -f1)"
+find "$target" -mindepth 1 -delete
+echo "restore test passed"
+result=passed