@yunuseon/kayra 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/ansible/playbooks/edge.yml +7 -2
  2. package/ansible/playbooks/platform.yml +3 -0
  3. package/ansible/roles/backup-check/files/backup-check.service +1 -1
  4. package/ansible/roles/backup-check/tasks/main.yml +18 -4
  5. package/ansible/roles/backup-check/templates/backup-check.sh.j2 +25 -42
  6. package/ansible/roles/monitor/tasks/main.yml +58 -0
  7. package/ansible/roles/monitor/templates/config.yaml.j2 +33 -0
  8. package/ansible/roles/monitor/templates/docker-compose.yml.j2 +17 -0
  9. package/ansible/roles/monitor/templates/env.j2 +4 -0
  10. package/ansible/roles/ntfy/files/docker-compose.yml +25 -0
  11. package/ansible/roles/ntfy/handlers/main.yml +6 -0
  12. package/ansible/roles/ntfy/tasks/main.yml +37 -0
  13. package/ansible/roles/ntfy/templates/server.yml.j2 +23 -0
  14. package/ansible/roles/platform/defaults/main.yml +5 -3
  15. package/ansible/roles/platform/tasks/main.yml +16 -8
  16. package/cli/dist/core/fleet/settings.js +16 -8
  17. package/cli/dist/core/recipes/host.js +2 -1
  18. package/cli/dist/core/recipes/steps.js +7 -0
  19. package/cli/dist/deployer/build/builder.js +17 -1
  20. package/cli/dist/deployer/deployer.js +18 -4
  21. package/cli/dist/deployer/forges/notify.js +8 -8
  22. package/cli/dist/deployer/forges/ntfy.js +35 -0
  23. package/cli/dist/deployer/health.js +42 -0
  24. package/cli/dist/deployer/settings.js +11 -17
  25. package/compose/docker-compose.ci.yml +7 -1
  26. package/package.json +1 -1
  27. package/schema/defaults.json +102 -19
  28. package/schema/vars-edge.json +121 -9
  29. package/schema/vars-platform.json +55 -10
  30. package/scripts/local/monitor-endpoints.sh +98 -0
  31. package/cli/dist/deployer/forges/issues.js +0 -72
@@ -3,8 +3,9 @@
3
3
  # Edge playbook — a server that exposes services from a private network (the
4
4
  # home NAS behind a Fritzbox) to the internet: a WireGuard tunnel to the LAN
5
5
  # and Caddy terminating TLS in front of it, plus the outside check on that
6
- # network's backups. No platform, no apps. Its firewall is Hetzner's
7
- # (servers/<server>/terraform/); nothing on the host filters on top.
6
+ # network's backups and, with ntfy set, the fleet's ntfy server. No platform,
7
+ # no apps. Its firewall is Hetzner's (servers/<server>/terraform/); nothing on
8
+ # the host filters on top.
8
9
  #
9
10
  # Usage: kayra setup <server> (for servers with kind: edge)
10
11
  # =============================================================================
@@ -23,6 +24,10 @@
23
24
  - docker
24
25
  - wireguard
25
26
  - edge-proxy
27
+ - role: ntfy
28
+ when: ntfy is defined
29
+ - role: monitor
30
+ when: monitor is defined
26
31
  - backup-check
27
32
 
28
33
  # ufw predates the repo on adopted servers: a second, drifting copy of the
@@ -51,6 +51,9 @@
51
51
  when: compose_stacks is defined
52
52
  - role: forgejo-config
53
53
  when: forgejo_config is defined
54
+ - role: monitor
55
+ become: true
56
+ when: monitor is defined
54
57
 
55
58
  post_tasks:
56
59
  - name: Record which kayra set this server up
@@ -1,6 +1,6 @@
1
1
  # Managed by Ansible (backup-check role).
2
2
  [Unit]
3
- Description=Report stale backups as GitHub issues
3
+ Description=Report stale backups to the notifier
4
4
  After=network-online.target
5
5
  Wants=network-online.target
6
6
 
@@ -2,11 +2,19 @@
2
2
  # =============================================================================
3
3
  # backup-check role — hourly, asks each of backup_check.repositories (a restic
4
4
  # repository served read-only over HTTPS) for its newest snapshot, and
5
- # reports one older than stale_after_hours as a GitHub issue, and likewise a
5
+ # reports one older than stale_after_hours to the notifier, and likewise a
6
6
  # failed or overdue restore test (restore_test_max_days). Runs off the
7
7
  # backed-up network on purpose: it notices a source that has gone quiet.
8
8
  # =============================================================================
9
9
 
10
+ - name: Check the backup check has a notifier
11
+ ansible.builtin.assert:
12
+ that:
13
+ - notifier.kind | default('') == 'ntfy'
14
+ - notifier_ntfy_token | default('') | length > 0
15
+ fail_msg: the backup check sends its alerts to the notifier, so servers/{{ server }} needs notifier (kind ntfy) in vars.yml and notifier_ntfy_token in secrets.yml
16
+ quiet: true
17
+
10
18
  - name: Install jq
11
19
  ansible.builtin.apt:
12
20
  name: jq
@@ -20,13 +28,19 @@
20
28
  mode: "0700"
21
29
  loop: ["", bin, state]
22
30
 
23
- - name: Install GitHub App key
31
+ - name: Install the notifier's token
24
32
  ansible.builtin.copy:
25
- content: "{{ backup_check_github_app_key }}"
26
- dest: /opt/backup-check/github-app.pem
33
+ content: "{{ notifier_ntfy_token }}"
34
+ dest: /opt/backup-check/ntfy-token
27
35
  mode: "0600"
28
36
  no_log: true
29
37
 
38
+ # Left by kayra before 0.22, when alerts were GitHub issues.
39
+ - name: Remove the GitHub App key
40
+ ansible.builtin.file:
41
+ path: /opt/backup-check/github-app.pem
42
+ state: absent
43
+
30
44
  # curl reads the logins from here, so they never show in a process list.
31
45
  - name: Render repository logins
32
46
  ansible.builtin.copy:
@@ -1,33 +1,22 @@
1
1
  #!/usr/bin/env bash
2
2
  # Generated by Ansible (backup-check role). Hourly, from backup-check.timer.
3
3
  #
4
- # Opens a GitHub issue when a repository's newest snapshot is older than
5
- # {{ backup_check.stale_after_hours }} hours, or (with restore_test_max_days) when its last restore
6
- # test failed or is overdue, and closes it once that has passed. GitHub is
7
- # only told about changes, which state/ records. The issue comes from a GitHub
8
- # App, not a personal token: GitHub notifies no one about their own actions.
4
+ # Sends an alert to the notifier (ntfy) when a repository's newest snapshot is
5
+ # older than {{ backup_check.stale_after_hours }} hours, or (with restore_test_max_days) when its last
6
+ # restore test failed or is overdue, and another once that has passed. Only
7
+ # changes are sent, which state/ records.
9
8
  set -euo pipefail
10
9
 
11
10
  state=/opt/backup-check/state
12
- repo={{ backup_check.github_repository | quote }}
13
11
  stale_after=$(( {{ backup_check.stale_after_hours | int }} * 3600 ))
14
12
 
15
- b64url() { openssl base64 -A | tr '+/' '-_' | tr -d '='; }
16
-
17
- github_token() {
18
- local now header payload jwt installation
19
- now=$(date +%s)
20
- header=$(printf '{"alg":"RS256","typ":"JWT"}' | b64url)
21
- payload=$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' $((now - 60)) $((now + 540)) {{ backup_check.github_app_id | quote }} | b64url)
22
- jwt="${header}.${payload}.$(printf '%s.%s' "$header" "$payload" \
23
- | openssl dgst -sha256 -sign /opt/backup-check/github-app.pem | b64url)"
24
- installation=$(github "$jwt" "https://api.github.com/repos/${repo}/installation" | jq -r .id)
25
- github "$jwt" -X POST "https://api.github.com/app/installations/${installation}/access_tokens" | jq -r .token
26
- }
27
-
28
- github() {
29
- local token="$1"; shift
30
- curl -fsS -H "Authorization: Bearer ${token}" -H "Accept: application/vnd.github+json" "$@"
13
+ publish() {
14
+ local title="$1" message="$2" priority="$3" tag="$4"
15
+ curl -fsS -o /dev/null -H "Authorization: Bearer $(cat /opt/backup-check/ntfy-token)" -H "Content-Type: application/json" \
16
+ -d "$(jq -n --arg topic {{ notifier.topic | quote }} --arg title "$title" --arg message "$message" \
17
+ --argjson priority "$priority" --arg tag "$tag" \
18
+ '{topic: $topic, title: $title, message: $message, priority: $priority, tags: [$tag]}')" \
19
+ {{ notifier.url | quote }}
31
20
  }
32
21
 
33
22
  # A file's Last-Modified as epoch seconds; nothing if the file is missing.
@@ -54,24 +43,18 @@ newest_snapshot() {
54
43
 
55
44
  when() { date -d "@$1" '+%Y-%m-%d %H:%M %Z'; }
56
45
 
57
- # One issue per key while its problem lasts; state/<key> holds its number.
58
- open_issue() {
59
- local key="$1" title="$2" body="$3"
46
+ # One alert per key while its problem lasts; state/<key> holds its title.
47
+ raise_alert() {
48
+ local key="$1" title="$2" message="$3"
60
49
  [ ! -f "${state}/${key}" ] || return 0
61
- github "$(github_token)" -X POST "https://api.github.com/repos/${repo}/issues" \
62
- -d "$(jq -n --arg title "$title" --arg body "${body}
63
-
64
- cc @${repo%%/*}" '{title: $title, body: $body}')" \
65
- | jq -r .number > "${state}/${key}"
50
+ publish "$title" "$message" 4 rotating_light
51
+ printf '%s' "$title" > "${state}/${key}"
66
52
  }
67
53
 
68
- close_issue() {
69
- local key="$1" comment="$2" token issue
54
+ resolve_alert() {
55
+ local key="$1" note="$2"
70
56
  [ -f "${state}/${key}" ] || return 0
71
- token=$(github_token)
72
- issue="https://api.github.com/repos/${repo}/issues/$(cat "${state}/${key}")"
73
- github "$token" -X POST "${issue}/comments" -d "$(jq -n --arg body "$comment" '{body: $body}')" >/dev/null
74
- github "$token" -X PATCH "$issue" -d '{"state": "closed", "state_reason": "completed"}' >/dev/null
57
+ publish "Resolved: $(cat "${state}/${key}")" "$note" 3 white_check_mark
75
58
  rm "${state}/${key}"
76
59
  }
77
60
 
@@ -80,11 +63,11 @@ check_snapshots() {
80
63
  newest=$(newest_snapshot "$url")
81
64
  if [ $(( $(date +%s) - ${newest:-0} )) -gt "$stale_after" ]; then
82
65
  if [ -n "$newest" ]; then last="The newest snapshot is from $(when "$newest")."; else last="There is no snapshot yet."; fi
83
- open_issue "$name" "Backup stale: ${name}" "No backup of \`${name}\` has reached its repository for more than {{ backup_check.stale_after_hours }} hours. ${last}
66
+ raise_alert "$name" "Backup stale: ${name}" "No backup of \`${name}\` has reached its repository for more than {{ backup_check.stale_after_hours }} hours. ${last}
84
67
 
85
- Look at the backup container's log on the source. This issue closes itself when the next snapshot arrives."
68
+ Look at the backup container's log on the source. Resolved by itself when the next snapshot arrives."
86
69
  else
87
- close_issue "$name" "A new snapshot arrived."
70
+ resolve_alert "$name" "A new snapshot arrived."
88
71
  fi
89
72
  }
90
73
 
@@ -101,11 +84,11 @@ check_restore_test() {
101
84
  problem="No restore test of \`${name}\` has passed for more than ${max_days} days. ${last}"
102
85
  fi
103
86
  if [ -n "${problem:-}" ]; then
104
- open_issue "${name}-restore-test" "Restore test: ${name}" "${problem}
87
+ raise_alert "${name}-restore-test" "Restore test: ${name}" "${problem}
105
88
 
106
- Its log is log/restore-test.log in the backup directory on the source; \`kayra backup test ${name}\` runs one by hand and follows it. This issue closes itself when the next one passes."
89
+ Its log is log/restore-test.log in the backup directory on the source; \`kayra backup test ${name}\` runs one by hand and follows it. Resolved by itself when the next one passes."
107
90
  else
108
- close_issue "${name}-restore-test" "A restore test passed."
91
+ resolve_alert "${name}-restore-test" "A restore test passed."
109
92
  fi
110
93
  }
111
94
 
@@ -0,0 +1,58 @@
1
+ ---
2
+ # =============================================================================
3
+ # monitor role — Gatus, checking every minute what kayra derives for the
4
+ # servers in monitor.watch (scripts/local/monitor-endpoints.sh, passed in as
5
+ # monitor_endpoints and monitor_hosts) plus monitor.extra, and alerting through
6
+ # the notifier's ntfy after three failures, again once it has recovered.
7
+ # Watched names are pinned to their servers' addresses (extra_hosts), so home
8
+ # names resolve through the tunnel from the edge as they do at home.
9
+ # =============================================================================
10
+
11
+ - name: Check the monitor has a notifier and its tokens
12
+ ansible.builtin.assert:
13
+ that:
14
+ - notifier.kind | default('') == 'ntfy'
15
+ - notifier_ntfy_token | default('') | length > 0
16
+ - monitor_endpoints is defined
17
+ - monitor_endpoints | selectattr('health') | map(attribute='group') | difference(monitor_health_tokens | default({}) | list) | length == 0
18
+ fail_msg: >-
19
+ the monitor alerts through the notifier (kind ntfy, notifier_ntfy_token in secrets.yml) and reads each
20
+ watched platform server's deploy.<domain>/health with monitor_health_tokens.<server> (that server's
21
+ health_token) in servers/{{ server }}/secrets.yml
22
+ quiet: true
23
+
24
+ - name: Create monitor directories
25
+ ansible.builtin.file:
26
+ path: "/opt/monitor/{{ item }}"
27
+ state: directory
28
+ owner: root
29
+ group: root
30
+ mode: "0755"
31
+ loop: ["", config]
32
+
33
+ - name: Render the monitor's compose file
34
+ ansible.builtin.template:
35
+ src: docker-compose.yml.j2
36
+ dest: /opt/monitor/docker-compose.yml
37
+ mode: "0644"
38
+
39
+ - name: Render the monitor's tokens
40
+ ansible.builtin.template:
41
+ src: env.j2
42
+ dest: /opt/monitor/.env
43
+ mode: "0600"
44
+ diff: false
45
+ register: monitor_env
46
+
47
+ # Into config/, mounted as a directory: Gatus reloads the file when it
48
+ # changes, which a single-file bind mount would never show it.
49
+ - name: Render Gatus's config
50
+ ansible.builtin.template:
51
+ src: config.yaml.j2
52
+ dest: /opt/monitor/config/config.yaml
53
+ mode: "0644"
54
+
55
+ - name: Start the monitor
56
+ community.docker.docker_compose_v2:
57
+ project_src: /opt/monitor
58
+ recreate: "{{ 'always' if monitor_env is changed else 'auto' }}"
@@ -0,0 +1,33 @@
1
+ # Generated by Ansible (monitor role) from monitor in servers/{{ server }}/vars.yml.
2
+ alerting:
3
+ ntfy:
4
+ url: {{ notifier.url | to_json }}
5
+ topic: {{ notifier.topic | to_json }}
6
+ token: "${NTFY_TOKEN}"
7
+ priority: 4
8
+ default-alert:
9
+ failure-threshold: 3
10
+ success-threshold: 2
11
+ send-on-resolved: true
12
+ description: {{ ('seen from ' ~ server) | to_json }}
13
+
14
+ endpoints:
15
+ {% for endpoint in monitor_endpoints %}
16
+ - name: {{ endpoint.name | to_json }}
17
+ group: {{ endpoint.group | to_json }}
18
+ url: {{ endpoint.url | to_json }}
19
+ interval: 1m
20
+ {% if not endpoint.redirects %}
21
+ client:
22
+ ignore-redirect: true
23
+ {% endif %}
24
+ # As a browser: Authelia answers anything else with 401, not the login.
25
+ headers:
26
+ Accept: text/html
27
+ {% if endpoint.health %}
28
+ Authorization: "Bearer ${HEALTH_TOKEN_{{ endpoint.group | upper | replace('-', '_') }}}"
29
+ {% endif %}
30
+ conditions: {{ endpoint.conditions | to_json }}
31
+ alerts:
32
+ - type: ntfy
33
+ {% endfor %}
@@ -0,0 +1,17 @@
1
+ # Generated by Ansible (monitor role).
2
+ services:
3
+ gatus:
4
+ image: twinproduction/gatus:v5.37.0
5
+ restart: unless-stopped
6
+ env_file: .env
7
+ environment:
8
+ GATUS_CONFIG_PATH: /config/config.yaml
9
+ volumes:
10
+ - ./config:/config:ro
11
+ mem_limit: 128m
12
+ {% if monitor_hosts | length > 0 %}
13
+ extra_hosts:
14
+ {% for host, address in monitor_hosts | dictsort %}
15
+ - "{{ host }}:{{ address }}"
16
+ {% endfor %}
17
+ {% endif %}
@@ -0,0 +1,4 @@
1
+ NTFY_TOKEN={{ notifier_ntfy_token }}
2
+ {% for watched, token in (monitor_health_tokens | default({})) | dictsort %}
3
+ HEALTH_TOKEN_{{ watched | upper | replace('-', '_') }}={{ token }}
4
+ {% endfor %}
@@ -0,0 +1,25 @@
1
+ # Managed by Ansible (ntfy role).
2
+ services:
3
+ ntfy:
4
+ image: binwiederhier/ntfy:v2.33.1
5
+ container_name: ntfy
6
+ restart: unless-stopped
7
+ command: ["serve"]
8
+ volumes:
9
+ - ./etc:/etc/ntfy:ro
10
+ - ./cache:/var/cache/ntfy
11
+ - ./lib:/var/lib/ntfy
12
+ mem_limit: 256m
13
+ healthcheck:
14
+ test: ["CMD-SHELL", "wget -q --tries=1 -O- http://localhost:80/v1/health | grep -q '\"healthy\":true'"]
15
+ interval: 60s
16
+ timeout: 10s
17
+ retries: 3
18
+ start_period: 20s
19
+ networks: [edge-proxy]
20
+
21
+ # The edge proxy's: Caddy reaches ntfy by its name there.
22
+ networks:
23
+ edge-proxy:
24
+ name: edge-proxy_default
25
+ external: true
@@ -0,0 +1,6 @@
1
+ ---
2
+ # ntfy reads server.yml, its users and tokens included, only when it starts.
3
+ - name: Restart ntfy
4
+ community.docker.docker_compose_v2:
5
+ project_src: /opt/ntfy
6
+ state: restarted
@@ -0,0 +1,37 @@
1
+ ---
2
+ # =============================================================================
3
+ # ntfy role — an ntfy server for the fleet's alerts, behind the edge's Caddy:
4
+ # its users, their access and their tokens as code (ntfy reads them at
5
+ # start), everything else denied. Caddy reaches it as ntfy:80 on the edge
6
+ # proxy's network, through an edge_routes entry like any other upstream.
7
+ # =============================================================================
8
+
9
+ - name: Create ntfy directories
10
+ ansible.builtin.file:
11
+ path: "/opt/ntfy/{{ item }}"
12
+ state: directory
13
+ owner: root
14
+ group: root
15
+ mode: "0755"
16
+ loop: ["", etc, cache, lib]
17
+
18
+ - name: Copy compose file
19
+ ansible.builtin.copy:
20
+ src: docker-compose.yml
21
+ dest: /opt/ntfy/docker-compose.yml
22
+ mode: "0644"
23
+ notify: Restart ntfy
24
+
25
+ # Into etc/, mounted as a directory, for the same reason as the Caddyfile.
26
+ - name: Render ntfy's server.yml
27
+ ansible.builtin.template:
28
+ src: server.yml.j2
29
+ dest: /opt/ntfy/etc/server.yml
30
+ mode: "0600"
31
+ diff: false
32
+ notify: Restart ntfy
33
+
34
+ - name: Start ntfy
35
+ community.docker.docker_compose_v2:
36
+ project_src: /opt/ntfy
37
+ wait: true
@@ -0,0 +1,23 @@
1
+ # Generated by Ansible from ntfy in servers/{{ server }}/vars.yml and the
2
+ # ntfy_* secrets.
3
+ base-url: {{ ntfy.base_url | to_json }}
4
+ listen-http: ":80"
5
+ behind-proxy: true
6
+ cache-file: /var/cache/ntfy/cache.db
7
+ auth-file: /var/lib/ntfy/user.db
8
+ auth-default-access: deny-all
9
+ {% if ntfy.upstream_base_url is defined %}
10
+ upstream-base-url: {{ ntfy.upstream_base_url | to_json }}
11
+ {% endif %}
12
+ auth-users:
13
+ {% for name, hash in ntfy_password_hashes.items() %}
14
+ - {{ (name ~ ':' ~ hash ~ ':user') | to_json }}
15
+ {% endfor %}
16
+ auth-access:
17
+ {% for entry in ntfy.access %}
18
+ - {{ (entry.user ~ ':' ~ entry.topic ~ ':' ~ entry.permission) | to_json }}
19
+ {% endfor %}
20
+ auth-tokens:
21
+ {% for name, token in ntfy_tokens.items() %}
22
+ - {{ (name ~ ':' ~ token ~ ':kayra') | to_json }}
23
+ {% endfor %}
@@ -27,10 +27,12 @@ acme_dns_token: ""
27
27
  git_token: ""
28
28
  # The secret forges sign webhooks with (deploy.<domain>/hook/<app>).
29
29
  webhook_secret: ""
30
- # Where the deployer raises alerts: {} for nowhere, or {kind: github-issue,
31
- # repository: owner/name, app_id: …} with notifier_github_app_key in secrets.yml.
30
+ # What a monitor shows to read deploy.<domain>/health; empty, there is no such page.
31
+ health_token: ""
32
+ # Where the deployer raises alerts: {} for nowhere, or {kind: ntfy, url: …,
33
+ # topic: …} with notifier_ntfy_token in secrets.yml.
32
34
  notifier: {}
33
- notifier_github_app_key: ""
35
+ notifier_ntfy_token: ""
34
36
 
35
37
  # owner/repo of the repos this server's Renovate job keeps up to date
36
38
  # (weekly version-bump PRs); the job runs only where the list is not empty.
@@ -14,12 +14,12 @@
14
14
  fail_msg: renovate_repositories is set, so renovate_token must be set in servers/{{ server }}/secrets.yml
15
15
  when: renovate_repositories | length > 0
16
16
 
17
- - name: Check the notifier has its key
17
+ - name: Check the notifier has its token
18
18
  ansible.builtin.assert:
19
- that: notifier_github_app_key | length > 0
20
- fail_msg: notifier is a github-issue notifier, so notifier_github_app_key must be set in servers/{{ server }}/secrets.yml
19
+ that: notifier_ntfy_token | length > 0
20
+ fail_msg: notifier is an ntfy notifier, so notifier_ntfy_token must be set in servers/{{ server }}/secrets.yml
21
21
  quiet: true
22
- when: notifier.kind | default('') == 'github-issue'
22
+ when: notifier.kind | default('') == 'ntfy'
23
23
 
24
24
  - name: Create platform directories
25
25
  ansible.builtin.file:
@@ -254,6 +254,7 @@
254
254
  loop:
255
255
  - {name: GIT_TOKEN, value: "{{ git_token }}"}
256
256
  - {name: RENOVATE_TOKEN, value: "{{ renovate_token }}"}
257
+ - {name: HEALTH_TOKEN, value: "{{ health_token }}"}
257
258
  loop_control:
258
259
  label: "{{ item.name }}"
259
260
  no_log: true
@@ -270,17 +271,24 @@
270
271
  mode: "0500"
271
272
  become: true
272
273
 
273
- - name: Render the notifier's key
274
+ - name: Render the notifier's token
274
275
  ansible.builtin.copy:
275
- content: "{{ notifier_github_app_key }}"
276
- dest: "{{ platform_dir }}/deployer-secrets/NOTIFIER_GITHUB_APP_KEY"
276
+ content: "{{ notifier_ntfy_token }}"
277
+ dest: "{{ platform_dir }}/deployer-secrets/NOTIFIER_NTFY_TOKEN"
277
278
  owner: "1000"
278
279
  group: "1000"
279
280
  mode: "0400"
280
- when: notifier_github_app_key | length > 0
281
+ when: notifier_ntfy_token | length > 0
281
282
  no_log: true
282
283
  become: true
283
284
 
285
+ # Left by kayra before 0.22, when alerts were GitHub issues.
286
+ - name: Remove the GitHub App key
287
+ ansible.builtin.file:
288
+ path: "{{ platform_dir }}/deployer-secrets/NOTIFIER_GITHUB_APP_KEY"
289
+ state: absent
290
+ become: true
291
+
284
292
  - name: Render platform .env
285
293
  ansible.builtin.template:
286
294
  src: platform.env.j2
@@ -70,18 +70,24 @@ export const settings = [
70
70
  // edge
71
71
  setting("wireguard", edge, record({ address: text, port: z.number(), peer_public_key: text, peer_allowed_ips: texts, keepalive: z.number().optional() }), "The tunnel into the private network: the edge's address, port, and the one peer (its key, what it may reach)."),
72
72
  setting("edge_routes", edge, z.array(record({ subdomains: texts, upstream: text, forwarded_headers: z.boolean().optional() })), "Public names the edge serves: subdomains of the domain and the upstream inside the tunnel they go to."),
73
- setting("notifier", platform, z.discriminatedUnion("kind", [
74
- z.strictObject({ kind: z.literal("github-issue"), repository: text, app_id: z.union([text, z.number()]) }),
75
- ]), "Where the deployer raises alerts (a failed deploy of an app's ref, a failed job) and resolves them: kind github-issue, an issue in repository (owner/name) opened by the GitHub App app_id; the key in notifier_github_app_key."),
73
+ setting("ntfy", edge, record({
74
+ base_url: text,
75
+ upstream_base_url: text.optional(),
76
+ access: z.array(record({ user: text, topic: text, permission: z.enum(["rw", "ro", "wo", "deny"]) })),
77
+ }), "An ntfy server for alerts, behind Caddy (an edge_routes entry to ntfy:80): its public base_url, who may read or publish which topic (everything else denied), and upstream_base_url (https://ntfy.sh) for iPhones, whose instant push only ntfy.sh can wake."),
78
+ setting("monitor", ["platform", "edge"], record({
79
+ watch: texts,
80
+ extra: z.array(record({ name: text, url: text, group: text.optional(), server: text.optional(), conditions: texts.optional() })).optional(),
81
+ }), "Gatus on this server, alerting through the notifier: every minute, what kayra derives for each server in watch (its logins, pages, app routes, deploy.<domain>/health; an edge's routes; certificates) with each name pinned to that server's address, plus extra checks (Gatus conditions, default [STATUS] == 200; server pins the name to that server's address)."),
82
+ setting("notifier", ["platform", "edge"], z.discriminatedUnion("kind", [z.strictObject({ kind: z.literal("ntfy"), url: text, topic: z.string().regex(/^[\w-]{1,64}$/) })]), "Where alerts go — the deployer's (a failed deploy of an app's ref, a failed job) and the backup check's: kind ntfy, a message to topic on the ntfy server at url when an alert opens and when it is resolved; the token in notifier_ntfy_token."),
76
83
  setting("backup_check", edge, record({
77
84
  stale_after_hours: z.number(),
78
- github_repository: text,
79
- github_app_id: text,
80
85
  repositories: z.array(record({ name: text, url: text, user: text, restore_test_max_days: z.number().optional() })),
81
- }), "The hourly check of the private network's backups: when a snapshot is stale, where issues go, which repositories."),
86
+ }), "The hourly check of the private network's backups: when a snapshot is stale, which repositories. Its alerts go to the notifier."),
82
87
  // Secrets
83
- secret("notifier_github_app_key", platform, text, "With notifier kind github-issue: the GitHub App's private key (PEM) the deployer opens issues with."),
88
+ secret("notifier_ntfy_token", ["platform", "edge"], text, "With notifier kind ntfy: the access token alerts are published with."),
84
89
  secret("git_token", platform, text, "Token the builder clones app repos with and the deployer posts commit statuses with."),
90
+ secret("health_token", platform, text, "Bearer token a monitor reads deploy.<domain>/health with: the platform's and every app's containers, the disk and memory. Unset, there is no such page."),
85
91
  secret("webhook_secret", platform, text, "HMAC secret the apps' push webhooks (deploy.<domain>/hook/<app>) are signed with."),
86
92
  secret("renovate_token", platform, text, "Token the Renovate job opens PRs with."),
87
93
  secret("renovate_github_com_token", platform, text, "Public-read GitHub token for Renovate's lookups off GitHub."),
@@ -99,9 +105,11 @@ export const settings = [
99
105
  secret("restic_backup_password", nas, text, "The restic repository's password."),
100
106
  secret("restic_backup_ssh_key", nas, text, "The SSH key the backup reaches the Storage Box with."),
101
107
  secret("restic_backup_postgres_passwords", nas, z.record(z.string(), text), "Per database container: the password its dump connects with."),
108
+ secret("ntfy_password_hashes", edge, z.record(z.string(), z.string().regex(/^\$2[aby]\$\d{2}\$/)), "Per ntfy user: the bcrypt hash of its password (ntfy user hash)."),
109
+ secret("ntfy_tokens", edge, z.record(z.string(), z.string().regex(/^tk_[a-z0-9]{29}$/)), "Per ntfy user: an access token (ntfy token generate), e.g. the one monitors publish with."),
110
+ secret("monitor_health_tokens", ["platform", "edge"], z.record(z.string(), text), "Per watched platform server: its health_token, which the monitor reads deploy.<domain>/health with."),
102
111
  secret("wireguard_private_key", edge, text, "The edge's WireGuard private key."),
103
112
  secret("wireguard_peer_psk", edge, text, "The tunnel's preshared key."),
104
- secret("backup_check_github_app_key", edge, text, "The GitHub App's private key the backup check opens issues with."),
105
113
  secret("backup_check_passwords", edge, z.record(z.string(), text), "Per watched repository: its read-only password."),
106
114
  ];
107
115
  /** Ansible's own variables (connection, privilege escalation) are always allowed. */
@@ -1,6 +1,6 @@
1
1
  import { partOf, run } from "../plan/model.js";
2
2
  import { err, map, ok } from "../../lib/result.js";
3
- import { ansibleEnv, ansiblePath, appSubdomains, certs, collections, fleetEnv, fromMain, inventory, latestState, needing, reachable, script, sequence, terraform, withSecrets, withVariables } from "./steps.js";
3
+ import { ansibleEnv, ansiblePath, appSubdomains, certs, collections, fleetEnv, fromMain, inventory, latestState, monitorEndpoints, needing, reachable, script, sequence, terraform, withSecrets, withVariables } from "./steps.js";
4
4
  import { apps } from "./apps.js";
5
5
  import { typed } from "../../lib/typed.js";
6
6
  export const provision = (engine, server) => ok(partOf("provision", [
@@ -30,6 +30,7 @@ const hostPlaybook = (engine, server, extra = []) => withSecrets(run(`Host layer
30
30
  `target=${server.target}`,
31
31
  "-e",
32
32
  `kayra_version=${engine.version}`,
33
+ ...(server.vars["monitor"] === undefined ? [] : ["-e", monitorEndpoints(engine, server)]),
33
34
  ...extra,
34
35
  ], ansibleEnv(engine, {})), server);
35
36
  /** The host layer: the playbook for the server's kind. */
@@ -65,5 +65,12 @@ export const appSubdomains = (engine, server) => typed({
65
65
  words: "many",
66
66
  env: fleetEnv(engine),
67
67
  });
68
+ /** What the server's monitor checks (scripts/local/monitor-endpoints.sh), one JSON argument for ansible-playbook -e. */
69
+ export const monitorEndpoints = (engine, server) => typed({
70
+ kind: "substitution",
71
+ argv: ["bash", script(engine, "monitor-endpoints.sh"), server.name],
72
+ words: "one",
73
+ env: fleetEnv(engine),
74
+ });
68
75
  export const certs = (engine, domain, subdomains) => run("mkcert certificates", ["bash", script(engine, "certs.sh"), domain, within(engine.fleet, ".cache", "certs"), ...subdomains]);
69
76
  export const sequence = (plans) => map(all(plans), (each) => Object.freeze(each.flat()));
@@ -9,7 +9,7 @@ import { page, runPage } from "../http/page.js";
9
9
  import { spawnLines } from "../process.js";
10
10
  import { signed } from "../forges/forge.js";
11
11
  import { forgeRepo } from "../../lib/forge-repo.js";
12
- import { pushedRef } from "../http/webhook.js";
12
+ import { header, pushedRef, secretMatches } from "../http/webhook.js";
13
13
  import { typed } from "../../lib/typed.js";
14
14
  const fromFile = (path) => () => (existsSync(path) ? readFileSync(path, "utf8").trim() : "");
15
15
  const settings = (env) => {
@@ -21,6 +21,7 @@ const settings = (env) => {
21
21
  resolveImages: env["RESOLVE_IMAGES"] ?? "/opt/kayra/scripts/platform/resolve-images.sh",
22
22
  webhookSecret: () => env["WEBHOOK_SECRET"] ?? "",
23
23
  gitToken: fromFile("/run/platform-secrets/GIT_TOKEN"),
24
+ healthToken: fromFile("/run/platform-secrets/HEALTH_TOKEN"),
24
25
  webhooks: env["WEBHOOKS"] !== "false",
25
26
  });
26
27
  };
@@ -136,6 +137,19 @@ const hook = async (s, name, request, response) => {
136
137
  const forwarded = await fetch(`${s.deployer}/push/${app.value.name}`, { method: "POST" }).catch(() => undefined);
137
138
  send(response, forwarded?.ok === true ? 202 : 502, forwarded?.ok === true ? "queued\n" : "the deployer did not answer\n");
138
139
  };
140
+ /** The deployer's view of the server, for a monitor that shows the health token; Traefik routes /health past the login. */
141
+ const health = async (s, request, response) => {
142
+ const given = /^Bearer (.+)$/.exec(header(request.headers, "authorization") ?? "")?.[1];
143
+ if (!secretMatches(s.healthToken(), given)) {
144
+ send(response, 404, "not found\n");
145
+ return;
146
+ }
147
+ const answer = await fetch(`${s.deployer}/health`).catch(() => undefined);
148
+ if (answer === undefined)
149
+ send(response, 503, JSON.stringify({ healthy: false, problems: ["the deployer did not answer"], warnings: [] }), "application/json");
150
+ else
151
+ send(response, answer.status, await answer.text(), "application/json");
152
+ };
139
153
  const pages = (s, path, response) => {
140
154
  const [, runs, who, id] = path.split("/");
141
155
  if (path === "/") {
@@ -194,6 +208,8 @@ export const serveBuilder = (env) => {
194
208
  else
195
209
  send(response, 404, "this server takes no webhooks: it polls (webhooks: false)\n");
196
210
  }
211
+ else if (request.method === "GET" && path === "/health")
212
+ await health(s, request, response);
197
213
  else if (request.method === "GET")
198
214
  pages(s, path, response);
199
215
  else