@yunuseon/kayra 0.21.1 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ansible/playbooks/edge.yml +7 -2
- package/ansible/playbooks/platform.yml +3 -0
- package/ansible/roles/backup-check/files/backup-check.service +1 -1
- package/ansible/roles/backup-check/tasks/main.yml +18 -4
- package/ansible/roles/backup-check/templates/backup-check.sh.j2 +25 -42
- package/ansible/roles/monitor/tasks/main.yml +58 -0
- package/ansible/roles/monitor/templates/config.yaml.j2 +33 -0
- package/ansible/roles/monitor/templates/docker-compose.yml.j2 +17 -0
- package/ansible/roles/monitor/templates/env.j2 +4 -0
- package/ansible/roles/ntfy/files/docker-compose.yml +25 -0
- package/ansible/roles/ntfy/handlers/main.yml +6 -0
- package/ansible/roles/ntfy/tasks/main.yml +37 -0
- package/ansible/roles/ntfy/templates/server.yml.j2 +23 -0
- package/ansible/roles/platform/defaults/main.yml +5 -3
- package/ansible/roles/platform/tasks/main.yml +16 -8
- package/cli/dist/core/fleet/settings.js +16 -8
- package/cli/dist/core/recipes/host.js +2 -1
- package/cli/dist/core/recipes/steps.js +7 -0
- package/cli/dist/deployer/build/builder.js +17 -1
- package/cli/dist/deployer/deployer.js +18 -4
- package/cli/dist/deployer/forges/notify.js +8 -8
- package/cli/dist/deployer/forges/ntfy.js +35 -0
- package/cli/dist/deployer/health.js +42 -0
- package/cli/dist/deployer/settings.js +11 -17
- package/compose/docker-compose.ci.yml +7 -1
- package/package.json +1 -1
- package/schema/defaults.json +102 -19
- package/schema/vars-edge.json +121 -9
- package/schema/vars-platform.json +55 -10
- package/scripts/local/monitor-endpoints.sh +98 -0
- package/cli/dist/deployer/forges/issues.js +0 -72
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
# Edge playbook — a server that exposes services from a private network (the
|
|
4
4
|
# home NAS behind a Fritzbox) to the internet: a WireGuard tunnel to the LAN
|
|
5
5
|
# and Caddy terminating TLS in front of it, plus the outside check on that
|
|
6
|
-
# network's backups
|
|
7
|
-
# (servers/<server>/terraform/); nothing on
|
|
6
|
+
# network's backups and, with ntfy set, the fleet's ntfy server. No platform,
|
|
7
|
+
# no apps. Its firewall is Hetzner's (servers/<server>/terraform/); nothing on
|
|
8
|
+
# the host filters on top.
|
|
8
9
|
#
|
|
9
10
|
# Usage: kayra setup <server> (for servers with kind: edge)
|
|
10
11
|
# =============================================================================
|
|
@@ -23,6 +24,10 @@
|
|
|
23
24
|
- docker
|
|
24
25
|
- wireguard
|
|
25
26
|
- edge-proxy
|
|
27
|
+
- role: ntfy
|
|
28
|
+
when: ntfy is defined
|
|
29
|
+
- role: monitor
|
|
30
|
+
when: monitor is defined
|
|
26
31
|
- backup-check
|
|
27
32
|
|
|
28
33
|
# ufw predates the repo on adopted servers: a second, drifting copy of the
|
|
@@ -2,11 +2,19 @@
|
|
|
2
2
|
# =============================================================================
|
|
3
3
|
# backup-check role — hourly, asks each of backup_check.repositories (a restic
|
|
4
4
|
# repository served read-only over HTTPS) for its newest snapshot, and
|
|
5
|
-
# reports one older than stale_after_hours
|
|
5
|
+
# reports one older than stale_after_hours to the notifier, and likewise a
|
|
6
6
|
# failed or overdue restore test (restore_test_max_days). Runs off the
|
|
7
7
|
# backed-up network on purpose: it notices a source that has gone quiet.
|
|
8
8
|
# =============================================================================
|
|
9
9
|
|
|
10
|
+
- name: Check the backup check has a notifier
|
|
11
|
+
ansible.builtin.assert:
|
|
12
|
+
that:
|
|
13
|
+
- notifier.kind | default('') == 'ntfy'
|
|
14
|
+
- notifier_ntfy_token | default('') | length > 0
|
|
15
|
+
fail_msg: the backup check sends its alerts to the notifier, so servers/{{ server }} needs notifier (kind ntfy) in vars.yml and notifier_ntfy_token in secrets.yml
|
|
16
|
+
quiet: true
|
|
17
|
+
|
|
10
18
|
- name: Install jq
|
|
11
19
|
ansible.builtin.apt:
|
|
12
20
|
name: jq
|
|
@@ -20,13 +28,19 @@
|
|
|
20
28
|
mode: "0700"
|
|
21
29
|
loop: ["", bin, state]
|
|
22
30
|
|
|
23
|
-
- name: Install
|
|
31
|
+
- name: Install the notifier's token
|
|
24
32
|
ansible.builtin.copy:
|
|
25
|
-
content: "{{
|
|
26
|
-
dest: /opt/backup-check/
|
|
33
|
+
content: "{{ notifier_ntfy_token }}"
|
|
34
|
+
dest: /opt/backup-check/ntfy-token
|
|
27
35
|
mode: "0600"
|
|
28
36
|
no_log: true
|
|
29
37
|
|
|
38
|
+
# Left by kayra before 0.22, when alerts were GitHub issues.
|
|
39
|
+
- name: Remove the GitHub App key
|
|
40
|
+
ansible.builtin.file:
|
|
41
|
+
path: /opt/backup-check/github-app.pem
|
|
42
|
+
state: absent
|
|
43
|
+
|
|
30
44
|
# curl reads the logins from here, so they never show in a process list.
|
|
31
45
|
- name: Render repository logins
|
|
32
46
|
ansible.builtin.copy:
|
|
@@ -1,33 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# Generated by Ansible (backup-check role). Hourly, from backup-check.timer.
|
|
3
3
|
#
|
|
4
|
-
#
|
|
5
|
-
# {{ backup_check.stale_after_hours }} hours, or (with restore_test_max_days) when its last
|
|
6
|
-
# test failed or is overdue, and
|
|
7
|
-
#
|
|
8
|
-
# App, not a personal token: GitHub notifies no one about their own actions.
|
|
4
|
+
# Sends an alert to the notifier (ntfy) when a repository's newest snapshot is
|
|
5
|
+
# older than {{ backup_check.stale_after_hours }} hours, or (with restore_test_max_days) when its last
|
|
6
|
+
# restore test failed or is overdue, and another once that has passed. Only
|
|
7
|
+
# changes are sent, which state/ records.
|
|
9
8
|
set -euo pipefail
|
|
10
9
|
|
|
11
10
|
state=/opt/backup-check/state
|
|
12
|
-
repo={{ backup_check.github_repository | quote }}
|
|
13
11
|
stale_after=$(( {{ backup_check.stale_after_hours | int }} * 3600 ))
|
|
14
12
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
jwt="${header}.${payload}.$(printf '%s.%s' "$header" "$payload" \
|
|
23
|
-
| openssl dgst -sha256 -sign /opt/backup-check/github-app.pem | b64url)"
|
|
24
|
-
installation=$(github "$jwt" "https://api.github.com/repos/${repo}/installation" | jq -r .id)
|
|
25
|
-
github "$jwt" -X POST "https://api.github.com/app/installations/${installation}/access_tokens" | jq -r .token
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
github() {
|
|
29
|
-
local token="$1"; shift
|
|
30
|
-
curl -fsS -H "Authorization: Bearer ${token}" -H "Accept: application/vnd.github+json" "$@"
|
|
13
|
+
publish() {
|
|
14
|
+
local title="$1" message="$2" priority="$3" tag="$4"
|
|
15
|
+
curl -fsS -o /dev/null -H "Authorization: Bearer $(cat /opt/backup-check/ntfy-token)" -H "Content-Type: application/json" \
|
|
16
|
+
-d "$(jq -n --arg topic {{ notifier.topic | quote }} --arg title "$title" --arg message "$message" \
|
|
17
|
+
--argjson priority "$priority" --arg tag "$tag" \
|
|
18
|
+
'{topic: $topic, title: $title, message: $message, priority: $priority, tags: [$tag]}')" \
|
|
19
|
+
{{ notifier.url | quote }}
|
|
31
20
|
}
|
|
32
21
|
|
|
33
22
|
# A file's Last-Modified as epoch seconds; nothing if the file is missing.
|
|
@@ -54,24 +43,18 @@ newest_snapshot() {
|
|
|
54
43
|
|
|
55
44
|
when() { date -d "@$1" '+%Y-%m-%d %H:%M %Z'; }
|
|
56
45
|
|
|
57
|
-
# One
|
|
58
|
-
|
|
59
|
-
local key="$1" title="$2"
|
|
46
|
+
# One alert per key while its problem lasts; state/<key> holds its title.
|
|
47
|
+
raise_alert() {
|
|
48
|
+
local key="$1" title="$2" message="$3"
|
|
60
49
|
[ ! -f "${state}/${key}" ] || return 0
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
cc @${repo%%/*}" '{title: $title, body: $body}')" \
|
|
65
|
-
| jq -r .number > "${state}/${key}"
|
|
50
|
+
publish "$title" "$message" 4 rotating_light
|
|
51
|
+
printf '%s' "$title" > "${state}/${key}"
|
|
66
52
|
}
|
|
67
53
|
|
|
68
|
-
|
|
69
|
-
local key="$1"
|
|
54
|
+
resolve_alert() {
|
|
55
|
+
local key="$1" note="$2"
|
|
70
56
|
[ -f "${state}/${key}" ] || return 0
|
|
71
|
-
|
|
72
|
-
issue="https://api.github.com/repos/${repo}/issues/$(cat "${state}/${key}")"
|
|
73
|
-
github "$token" -X POST "${issue}/comments" -d "$(jq -n --arg body "$comment" '{body: $body}')" >/dev/null
|
|
74
|
-
github "$token" -X PATCH "$issue" -d '{"state": "closed", "state_reason": "completed"}' >/dev/null
|
|
57
|
+
publish "Resolved: $(cat "${state}/${key}")" "$note" 3 white_check_mark
|
|
75
58
|
rm "${state}/${key}"
|
|
76
59
|
}
|
|
77
60
|
|
|
@@ -80,11 +63,11 @@ check_snapshots() {
|
|
|
80
63
|
newest=$(newest_snapshot "$url")
|
|
81
64
|
if [ $(( $(date +%s) - ${newest:-0} )) -gt "$stale_after" ]; then
|
|
82
65
|
if [ -n "$newest" ]; then last="The newest snapshot is from $(when "$newest")."; else last="There is no snapshot yet."; fi
|
|
83
|
-
|
|
66
|
+
raise_alert "$name" "Backup stale: ${name}" "No backup of \`${name}\` has reached its repository for more than {{ backup_check.stale_after_hours }} hours. ${last}
|
|
84
67
|
|
|
85
|
-
Look at the backup container's log on the source.
|
|
68
|
+
Look at the backup container's log on the source. Resolved by itself when the next snapshot arrives."
|
|
86
69
|
else
|
|
87
|
-
|
|
70
|
+
resolve_alert "$name" "A new snapshot arrived."
|
|
88
71
|
fi
|
|
89
72
|
}
|
|
90
73
|
|
|
@@ -101,11 +84,11 @@ check_restore_test() {
|
|
|
101
84
|
problem="No restore test of \`${name}\` has passed for more than ${max_days} days. ${last}"
|
|
102
85
|
fi
|
|
103
86
|
if [ -n "${problem:-}" ]; then
|
|
104
|
-
|
|
87
|
+
raise_alert "${name}-restore-test" "Restore test: ${name}" "${problem}
|
|
105
88
|
|
|
106
|
-
Its log is log/restore-test.log in the backup directory on the source; \`kayra backup test ${name}\` runs one by hand and follows it.
|
|
89
|
+
Its log is log/restore-test.log in the backup directory on the source; \`kayra backup test ${name}\` runs one by hand and follows it. Resolved by itself when the next one passes."
|
|
107
90
|
else
|
|
108
|
-
|
|
91
|
+
resolve_alert "${name}-restore-test" "A restore test passed."
|
|
109
92
|
fi
|
|
110
93
|
}
|
|
111
94
|
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
---
|
|
2
|
+
# =============================================================================
|
|
3
|
+
# monitor role — Gatus, checking every minute what kayra derives for the
|
|
4
|
+
# servers in monitor.watch (scripts/local/monitor-endpoints.sh, passed in as
|
|
5
|
+
# monitor_endpoints and monitor_hosts) plus monitor.extra, and alerting through
|
|
6
|
+
# the notifier's ntfy after three failures, again once it has recovered.
|
|
7
|
+
# Watched names are pinned to their servers' addresses (extra_hosts), so home
|
|
8
|
+
# names resolve through the tunnel from the edge as they do at home.
|
|
9
|
+
# =============================================================================
|
|
10
|
+
|
|
11
|
+
- name: Check the monitor has a notifier and its tokens
|
|
12
|
+
ansible.builtin.assert:
|
|
13
|
+
that:
|
|
14
|
+
- notifier.kind | default('') == 'ntfy'
|
|
15
|
+
- notifier_ntfy_token | default('') | length > 0
|
|
16
|
+
- monitor_endpoints is defined
|
|
17
|
+
- monitor_endpoints | selectattr('health') | map(attribute='group') | difference(monitor_health_tokens | default({}) | list) | length == 0
|
|
18
|
+
fail_msg: >-
|
|
19
|
+
the monitor alerts through the notifier (kind ntfy, notifier_ntfy_token in secrets.yml) and reads each
|
|
20
|
+
watched platform server's deploy.<domain>/health with monitor_health_tokens.<server> (that server's
|
|
21
|
+
health_token) in servers/{{ server }}/secrets.yml
|
|
22
|
+
quiet: true
|
|
23
|
+
|
|
24
|
+
- name: Create monitor directories
|
|
25
|
+
ansible.builtin.file:
|
|
26
|
+
path: "/opt/monitor/{{ item }}"
|
|
27
|
+
state: directory
|
|
28
|
+
owner: root
|
|
29
|
+
group: root
|
|
30
|
+
mode: "0755"
|
|
31
|
+
loop: ["", config]
|
|
32
|
+
|
|
33
|
+
- name: Render the monitor's compose file
|
|
34
|
+
ansible.builtin.template:
|
|
35
|
+
src: docker-compose.yml.j2
|
|
36
|
+
dest: /opt/monitor/docker-compose.yml
|
|
37
|
+
mode: "0644"
|
|
38
|
+
|
|
39
|
+
- name: Render the monitor's tokens
|
|
40
|
+
ansible.builtin.template:
|
|
41
|
+
src: env.j2
|
|
42
|
+
dest: /opt/monitor/.env
|
|
43
|
+
mode: "0600"
|
|
44
|
+
diff: false
|
|
45
|
+
register: monitor_env
|
|
46
|
+
|
|
47
|
+
# Into config/, mounted as a directory: Gatus reloads the file when it
|
|
48
|
+
# changes, which a single-file bind mount would never show it.
|
|
49
|
+
- name: Render Gatus's config
|
|
50
|
+
ansible.builtin.template:
|
|
51
|
+
src: config.yaml.j2
|
|
52
|
+
dest: /opt/monitor/config/config.yaml
|
|
53
|
+
mode: "0644"
|
|
54
|
+
|
|
55
|
+
- name: Start the monitor
|
|
56
|
+
community.docker.docker_compose_v2:
|
|
57
|
+
project_src: /opt/monitor
|
|
58
|
+
recreate: "{{ 'always' if monitor_env is changed else 'auto' }}"
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Generated by Ansible (monitor role) from monitor in servers/{{ server }}/vars.yml.
|
|
2
|
+
alerting:
|
|
3
|
+
ntfy:
|
|
4
|
+
url: {{ notifier.url | to_json }}
|
|
5
|
+
topic: {{ notifier.topic | to_json }}
|
|
6
|
+
token: "${NTFY_TOKEN}"
|
|
7
|
+
priority: 4
|
|
8
|
+
default-alert:
|
|
9
|
+
failure-threshold: 3
|
|
10
|
+
success-threshold: 2
|
|
11
|
+
send-on-resolved: true
|
|
12
|
+
description: {{ ('seen from ' ~ server) | to_json }}
|
|
13
|
+
|
|
14
|
+
endpoints:
|
|
15
|
+
{% for endpoint in monitor_endpoints %}
|
|
16
|
+
- name: {{ endpoint.name | to_json }}
|
|
17
|
+
group: {{ endpoint.group | to_json }}
|
|
18
|
+
url: {{ endpoint.url | to_json }}
|
|
19
|
+
interval: 1m
|
|
20
|
+
{% if not endpoint.redirects %}
|
|
21
|
+
client:
|
|
22
|
+
ignore-redirect: true
|
|
23
|
+
{% endif %}
|
|
24
|
+
# As a browser: Authelia answers anything else with 401, not the login.
|
|
25
|
+
headers:
|
|
26
|
+
Accept: text/html
|
|
27
|
+
{% if endpoint.health %}
|
|
28
|
+
Authorization: "Bearer ${HEALTH_TOKEN_{{ endpoint.group | upper | replace('-', '_') }}}"
|
|
29
|
+
{% endif %}
|
|
30
|
+
conditions: {{ endpoint.conditions | to_json }}
|
|
31
|
+
alerts:
|
|
32
|
+
- type: ntfy
|
|
33
|
+
{% endfor %}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Generated by Ansible (monitor role).
|
|
2
|
+
services:
|
|
3
|
+
gatus:
|
|
4
|
+
image: twinproduction/gatus:v5.37.0
|
|
5
|
+
restart: unless-stopped
|
|
6
|
+
env_file: .env
|
|
7
|
+
environment:
|
|
8
|
+
GATUS_CONFIG_PATH: /config/config.yaml
|
|
9
|
+
volumes:
|
|
10
|
+
- ./config:/config:ro
|
|
11
|
+
mem_limit: 128m
|
|
12
|
+
{% if monitor_hosts | length > 0 %}
|
|
13
|
+
extra_hosts:
|
|
14
|
+
{% for host, address in monitor_hosts | dictsort %}
|
|
15
|
+
- "{{ host }}:{{ address }}"
|
|
16
|
+
{% endfor %}
|
|
17
|
+
{% endif %}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Managed by Ansible (ntfy role).
|
|
2
|
+
services:
|
|
3
|
+
ntfy:
|
|
4
|
+
image: binwiederhier/ntfy:v2.33.1
|
|
5
|
+
container_name: ntfy
|
|
6
|
+
restart: unless-stopped
|
|
7
|
+
command: ["serve"]
|
|
8
|
+
volumes:
|
|
9
|
+
- ./etc:/etc/ntfy:ro
|
|
10
|
+
- ./cache:/var/cache/ntfy
|
|
11
|
+
- ./lib:/var/lib/ntfy
|
|
12
|
+
mem_limit: 256m
|
|
13
|
+
healthcheck:
|
|
14
|
+
test: ["CMD-SHELL", "wget -q --tries=1 -O- http://localhost:80/v1/health | grep -q '\"healthy\":true'"]
|
|
15
|
+
interval: 60s
|
|
16
|
+
timeout: 10s
|
|
17
|
+
retries: 3
|
|
18
|
+
start_period: 20s
|
|
19
|
+
networks: [edge-proxy]
|
|
20
|
+
|
|
21
|
+
# The edge proxy's: Caddy reaches ntfy by its name there.
|
|
22
|
+
networks:
|
|
23
|
+
edge-proxy:
|
|
24
|
+
name: edge-proxy_default
|
|
25
|
+
external: true
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
---
|
|
2
|
+
# =============================================================================
|
|
3
|
+
# ntfy role — an ntfy server for the fleet's alerts, behind the edge's Caddy:
|
|
4
|
+
# its users, their access and their tokens as code (ntfy reads them at
|
|
5
|
+
# start), everything else denied. Caddy reaches it as ntfy:80 on the edge
|
|
6
|
+
# proxy's network, through an edge_routes entry like any other upstream.
|
|
7
|
+
# =============================================================================
|
|
8
|
+
|
|
9
|
+
- name: Create ntfy directories
|
|
10
|
+
ansible.builtin.file:
|
|
11
|
+
path: "/opt/ntfy/{{ item }}"
|
|
12
|
+
state: directory
|
|
13
|
+
owner: root
|
|
14
|
+
group: root
|
|
15
|
+
mode: "0755"
|
|
16
|
+
loop: ["", etc, cache, lib]
|
|
17
|
+
|
|
18
|
+
- name: Copy compose file
|
|
19
|
+
ansible.builtin.copy:
|
|
20
|
+
src: docker-compose.yml
|
|
21
|
+
dest: /opt/ntfy/docker-compose.yml
|
|
22
|
+
mode: "0644"
|
|
23
|
+
notify: Restart ntfy
|
|
24
|
+
|
|
25
|
+
# Into etc/, mounted as a directory, for the same reason as the Caddyfile.
|
|
26
|
+
- name: Render ntfy's server.yml
|
|
27
|
+
ansible.builtin.template:
|
|
28
|
+
src: server.yml.j2
|
|
29
|
+
dest: /opt/ntfy/etc/server.yml
|
|
30
|
+
mode: "0600"
|
|
31
|
+
diff: false
|
|
32
|
+
notify: Restart ntfy
|
|
33
|
+
|
|
34
|
+
- name: Start ntfy
|
|
35
|
+
community.docker.docker_compose_v2:
|
|
36
|
+
project_src: /opt/ntfy
|
|
37
|
+
wait: true
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Generated by Ansible from ntfy in servers/{{ server }}/vars.yml and the
|
|
2
|
+
# ntfy_* secrets.
|
|
3
|
+
base-url: {{ ntfy.base_url | to_json }}
|
|
4
|
+
listen-http: ":80"
|
|
5
|
+
behind-proxy: true
|
|
6
|
+
cache-file: /var/cache/ntfy/cache.db
|
|
7
|
+
auth-file: /var/lib/ntfy/user.db
|
|
8
|
+
auth-default-access: deny-all
|
|
9
|
+
{% if ntfy.upstream_base_url is defined %}
|
|
10
|
+
upstream-base-url: {{ ntfy.upstream_base_url | to_json }}
|
|
11
|
+
{% endif %}
|
|
12
|
+
auth-users:
|
|
13
|
+
{% for name, hash in ntfy_password_hashes.items() %}
|
|
14
|
+
- {{ (name ~ ':' ~ hash ~ ':user') | to_json }}
|
|
15
|
+
{% endfor %}
|
|
16
|
+
auth-access:
|
|
17
|
+
{% for entry in ntfy.access %}
|
|
18
|
+
- {{ (entry.user ~ ':' ~ entry.topic ~ ':' ~ entry.permission) | to_json }}
|
|
19
|
+
{% endfor %}
|
|
20
|
+
auth-tokens:
|
|
21
|
+
{% for name, token in ntfy_tokens.items() %}
|
|
22
|
+
- {{ (name ~ ':' ~ token ~ ':kayra') | to_json }}
|
|
23
|
+
{% endfor %}
|
|
@@ -27,10 +27,12 @@ acme_dns_token: ""
|
|
|
27
27
|
git_token: ""
|
|
28
28
|
# The secret forges sign webhooks with (deploy.<domain>/hook/<app>).
|
|
29
29
|
webhook_secret: ""
|
|
30
|
-
#
|
|
31
|
-
|
|
30
|
+
# What a monitor shows to read deploy.<domain>/health; empty, there is no such page.
|
|
31
|
+
health_token: ""
|
|
32
|
+
# Where the deployer raises alerts: {} for nowhere, or {kind: ntfy, url: …,
|
|
33
|
+
# topic: …} with notifier_ntfy_token in secrets.yml.
|
|
32
34
|
notifier: {}
|
|
33
|
-
|
|
35
|
+
notifier_ntfy_token: ""
|
|
34
36
|
|
|
35
37
|
# owner/repo of the repos this server's Renovate job keeps up to date
|
|
36
38
|
# (weekly version-bump PRs); the job runs only where the list is not empty.
|
|
@@ -14,12 +14,12 @@
|
|
|
14
14
|
fail_msg: renovate_repositories is set, so renovate_token must be set in servers/{{ server }}/secrets.yml
|
|
15
15
|
when: renovate_repositories | length > 0
|
|
16
16
|
|
|
17
|
-
- name: Check the notifier has its
|
|
17
|
+
- name: Check the notifier has its token
|
|
18
18
|
ansible.builtin.assert:
|
|
19
|
-
that:
|
|
20
|
-
fail_msg: notifier is
|
|
19
|
+
that: notifier_ntfy_token | length > 0
|
|
20
|
+
fail_msg: notifier is an ntfy notifier, so notifier_ntfy_token must be set in servers/{{ server }}/secrets.yml
|
|
21
21
|
quiet: true
|
|
22
|
-
when: notifier.kind | default('') == '
|
|
22
|
+
when: notifier.kind | default('') == 'ntfy'
|
|
23
23
|
|
|
24
24
|
- name: Create platform directories
|
|
25
25
|
ansible.builtin.file:
|
|
@@ -254,6 +254,7 @@
|
|
|
254
254
|
loop:
|
|
255
255
|
- {name: GIT_TOKEN, value: "{{ git_token }}"}
|
|
256
256
|
- {name: RENOVATE_TOKEN, value: "{{ renovate_token }}"}
|
|
257
|
+
- {name: HEALTH_TOKEN, value: "{{ health_token }}"}
|
|
257
258
|
loop_control:
|
|
258
259
|
label: "{{ item.name }}"
|
|
259
260
|
no_log: true
|
|
@@ -270,17 +271,24 @@
|
|
|
270
271
|
mode: "0500"
|
|
271
272
|
become: true
|
|
272
273
|
|
|
273
|
-
- name: Render the notifier's
|
|
274
|
+
- name: Render the notifier's token
|
|
274
275
|
ansible.builtin.copy:
|
|
275
|
-
content: "{{
|
|
276
|
-
dest: "{{ platform_dir }}/deployer-secrets/
|
|
276
|
+
content: "{{ notifier_ntfy_token }}"
|
|
277
|
+
dest: "{{ platform_dir }}/deployer-secrets/NOTIFIER_NTFY_TOKEN"
|
|
277
278
|
owner: "1000"
|
|
278
279
|
group: "1000"
|
|
279
280
|
mode: "0400"
|
|
280
|
-
when:
|
|
281
|
+
when: notifier_ntfy_token | length > 0
|
|
281
282
|
no_log: true
|
|
282
283
|
become: true
|
|
283
284
|
|
|
285
|
+
# Left by kayra before 0.22, when alerts were GitHub issues.
|
|
286
|
+
- name: Remove the GitHub App key
|
|
287
|
+
ansible.builtin.file:
|
|
288
|
+
path: "{{ platform_dir }}/deployer-secrets/NOTIFIER_GITHUB_APP_KEY"
|
|
289
|
+
state: absent
|
|
290
|
+
become: true
|
|
291
|
+
|
|
284
292
|
- name: Render platform .env
|
|
285
293
|
ansible.builtin.template:
|
|
286
294
|
src: platform.env.j2
|
|
@@ -70,18 +70,24 @@ export const settings = [
|
|
|
70
70
|
// edge
|
|
71
71
|
setting("wireguard", edge, record({ address: text, port: z.number(), peer_public_key: text, peer_allowed_ips: texts, keepalive: z.number().optional() }), "The tunnel into the private network: the edge's address, port, and the one peer (its key, what it may reach)."),
|
|
72
72
|
setting("edge_routes", edge, z.array(record({ subdomains: texts, upstream: text, forwarded_headers: z.boolean().optional() })), "Public names the edge serves: subdomains of the domain and the upstream inside the tunnel they go to."),
|
|
73
|
-
setting("
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
setting("ntfy", edge, record({
|
|
74
|
+
base_url: text,
|
|
75
|
+
upstream_base_url: text.optional(),
|
|
76
|
+
access: z.array(record({ user: text, topic: text, permission: z.enum(["rw", "ro", "wo", "deny"]) })),
|
|
77
|
+
}), "An ntfy server for alerts, behind Caddy (an edge_routes entry to ntfy:80): its public base_url, who may read or publish which topic (everything else denied), and upstream_base_url (https://ntfy.sh) for iPhones, whose instant push only ntfy.sh can wake."),
|
|
78
|
+
setting("monitor", ["platform", "edge"], record({
|
|
79
|
+
watch: texts,
|
|
80
|
+
extra: z.array(record({ name: text, url: text, group: text.optional(), server: text.optional(), conditions: texts.optional() })).optional(),
|
|
81
|
+
}), "Gatus on this server, alerting through the notifier: every minute, what kayra derives for each server in watch (its logins, pages, app routes, deploy.<domain>/health; an edge's routes; certificates) with each name pinned to that server's address, plus extra checks (Gatus conditions, default [STATUS] == 200; server pins the name to that server's address)."),
|
|
82
|
+
setting("notifier", ["platform", "edge"], z.discriminatedUnion("kind", [z.strictObject({ kind: z.literal("ntfy"), url: text, topic: z.string().regex(/^[\w-]{1,64}$/) })]), "Where alerts go — the deployer's (a failed deploy of an app's ref, a failed job) and the backup check's: kind ntfy, a message to topic on the ntfy server at url when an alert opens and when it is resolved; the token in notifier_ntfy_token."),
|
|
76
83
|
setting("backup_check", edge, record({
|
|
77
84
|
stale_after_hours: z.number(),
|
|
78
|
-
github_repository: text,
|
|
79
|
-
github_app_id: text,
|
|
80
85
|
repositories: z.array(record({ name: text, url: text, user: text, restore_test_max_days: z.number().optional() })),
|
|
81
|
-
}), "The hourly check of the private network's backups: when a snapshot is stale,
|
|
86
|
+
}), "The hourly check of the private network's backups: when a snapshot is stale, which repositories. Its alerts go to the notifier."),
|
|
82
87
|
// Secrets
|
|
83
|
-
secret("
|
|
88
|
+
secret("notifier_ntfy_token", ["platform", "edge"], text, "With notifier kind ntfy: the access token alerts are published with."),
|
|
84
89
|
secret("git_token", platform, text, "Token the builder clones app repos with and the deployer posts commit statuses with."),
|
|
90
|
+
secret("health_token", platform, text, "Bearer token a monitor reads deploy.<domain>/health with: the platform's and every app's containers, the disk and memory. Unset, there is no such page."),
|
|
85
91
|
secret("webhook_secret", platform, text, "HMAC secret the apps' push webhooks (deploy.<domain>/hook/<app>) are signed with."),
|
|
86
92
|
secret("renovate_token", platform, text, "Token the Renovate job opens PRs with."),
|
|
87
93
|
secret("renovate_github_com_token", platform, text, "Public-read GitHub token for Renovate's lookups off GitHub."),
|
|
@@ -99,9 +105,11 @@ export const settings = [
|
|
|
99
105
|
secret("restic_backup_password", nas, text, "The restic repository's password."),
|
|
100
106
|
secret("restic_backup_ssh_key", nas, text, "The SSH key the backup reaches the Storage Box with."),
|
|
101
107
|
secret("restic_backup_postgres_passwords", nas, z.record(z.string(), text), "Per database container: the password its dump connects with."),
|
|
108
|
+
secret("ntfy_password_hashes", edge, z.record(z.string(), text), "Per ntfy user: the bcrypt hash of its password (ntfy user hash)."),
|
|
109
|
+
secret("ntfy_tokens", edge, z.record(z.string(), text), "Per ntfy user: an access token (ntfy token generate), e.g. the one monitors publish with."),
|
|
110
|
+
secret("monitor_health_tokens", ["platform", "edge"], z.record(z.string(), text), "Per watched platform server: its health_token, which the monitor reads deploy.<domain>/health with."),
|
|
102
111
|
secret("wireguard_private_key", edge, text, "The edge's WireGuard private key."),
|
|
103
112
|
secret("wireguard_peer_psk", edge, text, "The tunnel's preshared key."),
|
|
104
|
-
secret("backup_check_github_app_key", edge, text, "The GitHub App's private key the backup check opens issues with."),
|
|
105
113
|
secret("backup_check_passwords", edge, z.record(z.string(), text), "Per watched repository: its read-only password."),
|
|
106
114
|
];
|
|
107
115
|
/** Ansible's own variables (connection, privilege escalation) are always allowed. */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { partOf, run } from "../plan/model.js";
|
|
2
2
|
import { err, map, ok } from "../../lib/result.js";
|
|
3
|
-
import { ansibleEnv, ansiblePath, appSubdomains, certs, collections, fleetEnv, fromMain, inventory, latestState, needing, reachable, script, sequence, terraform, withSecrets, withVariables } from "./steps.js";
|
|
3
|
+
import { ansibleEnv, ansiblePath, appSubdomains, certs, collections, fleetEnv, fromMain, inventory, latestState, monitorEndpoints, needing, reachable, script, sequence, terraform, withSecrets, withVariables } from "./steps.js";
|
|
4
4
|
import { apps } from "./apps.js";
|
|
5
5
|
import { typed } from "../../lib/typed.js";
|
|
6
6
|
export const provision = (engine, server) => ok(partOf("provision", [
|
|
@@ -30,6 +30,7 @@ const hostPlaybook = (engine, server, extra = []) => withSecrets(run(`Host layer
|
|
|
30
30
|
`target=${server.target}`,
|
|
31
31
|
"-e",
|
|
32
32
|
`kayra_version=${engine.version}`,
|
|
33
|
+
...(server.vars["monitor"] === undefined ? [] : ["-e", monitorEndpoints(engine, server)]),
|
|
33
34
|
...extra,
|
|
34
35
|
], ansibleEnv(engine, {})), server);
|
|
35
36
|
/** The host layer: the playbook for the server's kind. */
|
|
@@ -65,5 +65,12 @@ export const appSubdomains = (engine, server) => typed({
|
|
|
65
65
|
words: "many",
|
|
66
66
|
env: fleetEnv(engine),
|
|
67
67
|
});
|
|
68
|
+
/** What the server's monitor checks (scripts/local/monitor-endpoints.sh), one JSON argument for ansible-playbook -e. */
|
|
69
|
+
export const monitorEndpoints = (engine, server) => typed({
|
|
70
|
+
kind: "substitution",
|
|
71
|
+
argv: ["bash", script(engine, "monitor-endpoints.sh"), server.name],
|
|
72
|
+
words: "one",
|
|
73
|
+
env: fleetEnv(engine),
|
|
74
|
+
});
|
|
68
75
|
export const certs = (engine, domain, subdomains) => run("mkcert certificates", ["bash", script(engine, "certs.sh"), domain, within(engine.fleet, ".cache", "certs"), ...subdomains]);
|
|
69
76
|
export const sequence = (plans) => map(all(plans), (each) => Object.freeze(each.flat()));
|
|
@@ -9,7 +9,7 @@ import { page, runPage } from "../http/page.js";
|
|
|
9
9
|
import { spawnLines } from "../process.js";
|
|
10
10
|
import { signed } from "../forges/forge.js";
|
|
11
11
|
import { forgeRepo } from "../../lib/forge-repo.js";
|
|
12
|
-
import { pushedRef } from "../http/webhook.js";
|
|
12
|
+
import { header, pushedRef, secretMatches } from "../http/webhook.js";
|
|
13
13
|
import { typed } from "../../lib/typed.js";
|
|
14
14
|
const fromFile = (path) => () => (existsSync(path) ? readFileSync(path, "utf8").trim() : "");
|
|
15
15
|
const settings = (env) => {
|
|
@@ -21,6 +21,7 @@ const settings = (env) => {
|
|
|
21
21
|
resolveImages: env["RESOLVE_IMAGES"] ?? "/opt/kayra/scripts/platform/resolve-images.sh",
|
|
22
22
|
webhookSecret: () => env["WEBHOOK_SECRET"] ?? "",
|
|
23
23
|
gitToken: fromFile("/run/platform-secrets/GIT_TOKEN"),
|
|
24
|
+
healthToken: fromFile("/run/platform-secrets/HEALTH_TOKEN"),
|
|
24
25
|
webhooks: env["WEBHOOKS"] !== "false",
|
|
25
26
|
});
|
|
26
27
|
};
|
|
@@ -136,6 +137,19 @@ const hook = async (s, name, request, response) => {
|
|
|
136
137
|
const forwarded = await fetch(`${s.deployer}/push/${app.value.name}`, { method: "POST" }).catch(() => undefined);
|
|
137
138
|
send(response, forwarded?.ok === true ? 202 : 502, forwarded?.ok === true ? "queued\n" : "the deployer did not answer\n");
|
|
138
139
|
};
|
|
140
|
+
/** The deployer's view of the server, for a monitor that shows the health token; Traefik routes /health past the login. */
|
|
141
|
+
const health = async (s, request, response) => {
|
|
142
|
+
const given = /^Bearer (.+)$/.exec(header(request.headers, "authorization") ?? "")?.[1];
|
|
143
|
+
if (!secretMatches(s.healthToken(), given)) {
|
|
144
|
+
send(response, 404, "not found\n");
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
const answer = await fetch(`${s.deployer}/health`).catch(() => undefined);
|
|
148
|
+
if (answer === undefined)
|
|
149
|
+
send(response, 503, JSON.stringify({ healthy: false, problems: ["the deployer did not answer"], warnings: [] }), "application/json");
|
|
150
|
+
else
|
|
151
|
+
send(response, answer.status, await answer.text(), "application/json");
|
|
152
|
+
};
|
|
139
153
|
const pages = (s, path, response) => {
|
|
140
154
|
const [, runs, who, id] = path.split("/");
|
|
141
155
|
if (path === "/") {
|
|
@@ -194,6 +208,8 @@ export const serveBuilder = (env) => {
|
|
|
194
208
|
else
|
|
195
209
|
send(response, 404, "this server takes no webhooks: it polls (webhooks: false)\n");
|
|
196
210
|
}
|
|
211
|
+
else if (request.method === "GET" && path === "/health")
|
|
212
|
+
await health(s, request, response);
|
|
197
213
|
else if (request.method === "GET")
|
|
198
214
|
pages(s, path, response);
|
|
199
215
|
else
|