@crvouga/dev-fleet 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -17
- package/ansible/github/git.yml +66 -0
- package/ansible/macos/policy.json +12 -1
- package/ansible/monitoring/compose.yml +10 -0
- package/ansible/monitoring/container-watchdog.py +118 -0
- package/ansible/monitoring/hub-darwin.yml +146 -0
- package/ansible/net/remote-probe.sh +5 -0
- package/ansible/platforms.json +111 -0
- package/ansible/ubuntu/authorize-sudo.sh +7 -16
- package/ansible/ubuntu/baseline.yml +6 -28
- package/ansible/ubuntu/disk-gc.py +211 -7
- package/ansible/ubuntu/disk-guard.py +24 -5
- package/ansible/ubuntu/policy.json +68 -4
- package/ansible/ubuntu/t3-desktop.yml +2 -2
- package/ansible/ubuntu/tailscale.yml +20 -9
- package/ansible/ubuntu/tasks/browser.yml +56 -0
- package/ansible/ubuntu/tasks/docker.yml +52 -0
- package/ansible/ubuntu/tasks/dpkg.yml +46 -0
- package/ansible/ubuntu/tasks/install.yml +38 -0
- package/ansible/ubuntu/tasks/packages.yml +4 -55
- package/ansible/ubuntu/tasks/release.yml +25 -0
- package/ansible/ubuntu/tasks/resources.yml +7 -6
- package/ansible/ubuntu/tasks/services.yml +13 -0
- package/ansible/ubuntu/tasks/snapshot.yml +50 -0
- package/ansible/ubuntu/tasks/unattended-upgrades.yml +24 -0
- package/bootstrap/doctor.ts +32 -6
- package/bootstrap/fleet-remote-bootstrap.sh +49 -45
- package/bootstrap/setup.ts +23 -8
- package/bootstrap/worker-tools.ts +118 -0
- package/cliproxy/README.md +1 -1
- package/cliproxy/compose.yaml +4 -0
- package/cliproxy/scripts/checkpoint.py +1 -1
- package/docs/authentication.md +90 -0
- package/docs/consumer-setup.md +32 -30
- package/docs/declared.md +6 -5
- package/docs/machine-setup.md +22 -12
- package/docs/operations.md +87 -31
- package/docs/runbooks.md +377 -0
- package/docs/state.md +13 -7
- package/docs/support-matrix.md +36 -0
- package/docs/ubuntu-sync.md +16 -13
- package/docs/worker-contract.md +28 -1
- package/examples/fleet.json +19 -1
- package/llms.txt +2 -0
- package/mise.toml +7 -0
- package/package.json +2 -2
- package/schema/check-ids.json +526 -0
- package/schema/fleet.schema.json +185 -4
- package/schema/output/check.schema.json +23 -0
- package/schema/output/error.schema.json +14 -0
- package/schema/output/fleet-report.schema.json +79 -0
- package/schema/output/fleet-sync.schema.json +112 -0
- package/schema/output/report.schema.json +38 -0
- package/src/auth-checks.ts +3 -0
- package/src/cli/agent-guide.ts +5 -3
- package/src/cli/fleet-report.ts +126 -0
- package/src/cli/fleet-sync-manager.ts +11 -0
- package/src/cli/fleet-sync-plan.ts +200 -0
- package/src/cli/fleet-sync-remote.ts +4 -2
- package/src/cli/fleet-sync-schedule.ts +142 -35
- package/src/cli/fleet-sync.ts +90 -8
- package/src/cli/fleet-watch.ts +198 -0
- package/src/cli/fleet.ts +4 -0
- package/src/cli/register-fleet.ts +51 -0
- package/src/cli/registry.ts +13 -0
- package/src/cli/schedule-logs.ts +36 -0
- package/src/clock.ts +63 -0
- package/src/definition.ts +160 -12
- package/src/git-identity.ts +160 -0
- package/src/machine/platform.ts +48 -28
- package/src/machine/support-matrix.ts +76 -0
- package/src/machine-health.ts +51 -4
- package/src/manager-health.ts +205 -0
- package/src/manager-sync.ts +27 -9
- package/src/monitoring/hub-backup.ts +63 -17
- package/src/monitoring/manager-hub.ts +44 -2
- package/src/network-definition.ts +1 -1
- package/src/node-converge-linux.ts +16 -5
- package/src/node-converge.ts +1 -1
- package/src/notify.ts +239 -0
- package/src/platform/debian-version.ts +62 -0
- package/src/platform/ubuntu-audit.ts +11 -4
- package/src/platform/ubuntu-release.ts +41 -0
- package/src/remote-bootstrap.ts +89 -25
- package/src/remote-operation.ts +213 -0
- package/src/schedule-check.ts +40 -0
- package/src/secrets.ts +26 -8
- package/src/ssh.ts +10 -2
- package/src/state/history.ts +172 -0
- package/src/state/store.ts +11 -4
- package/src/tailscale.ts +51 -0
- package/src/toolchain-checks.ts +8 -3
- package/src/worker-capability-checks.ts +167 -0
- package/src/worker-capacity.ts +19 -7
- package/src/worker-host-checks.ts +68 -10
- package/src/worker-policy.ts +33 -0
- package/src/worker-readiness.ts +10 -1
package/README.md
CHANGED
|
@@ -42,7 +42,9 @@ dev-fleet is opinionated. These are part of it, not options:
|
|
|
42
42
|
- The **manager** is a macOS machine. You run `fleet-sync` on it, and it hosts
|
|
43
43
|
the shared services (the CLI proxy and the monitoring hub) in Docker Desktop.
|
|
44
44
|
- **Workers** are Ubuntu 26.04 machines reached over SSH. They are stateless:
|
|
45
|
-
everything on them is rebuilt from Git and your secrets.
|
|
45
|
+
everything on them is rebuilt from Git and your secrets. The
|
|
46
|
+
[support matrix](docs/support-matrix.md) lists every platform and what
|
|
47
|
+
happens to it.
|
|
46
48
|
- The fleet state is kept in an **S3-compatible bucket** you own.
|
|
47
49
|
- Machines reach each other over **Tailscale**.
|
|
48
50
|
- Coding agents (Claude Code, Codex, Grok, OpenCode, Antigravity) run through
|
|
@@ -98,7 +100,11 @@ Ansible policy and verifies the result. It prints one table and writes a full
|
|
|
98
100
|
JSON report. The first run on a machine installs everything, mise included, so
|
|
99
101
|
Git is the only tool it needs beforehand.
|
|
100
102
|
|
|
101
|
-
`--dry-run` changes no machine.
|
|
103
|
+
`--dry-run` changes no machine. It first brings the manager's copy of the
|
|
104
|
+
fleet state up to the bucket (never uploading), then prints, and adds to
|
|
105
|
+
`--json` as `plan`, what a sync would change: the revision, each tool pin that
|
|
106
|
+
moves, and what restarts because of it (a T3 restart ends its running agent
|
|
107
|
+
turns). On the manager it checks the release out into
|
|
102
108
|
dev-fleet's cache and installs the Bun that release pins beside any others, so
|
|
103
109
|
that it can run; it never installs the toolchain, publishes the global mise
|
|
104
110
|
configuration, edits shell profiles or touches a worker.
|
|
@@ -316,9 +322,11 @@ fleet.json contents.
|
|
|
316
322
|
`fleet-sync` fetches that copy again when it is deleted, but the proxy then
|
|
317
323
|
needs `devfleet cliproxy install` once more.
|
|
318
324
|
- The manager is a Mac host. There is no container image of the manager.
|
|
319
|
-
- `fleet-sync --schedule enable` (unattended runs
|
|
320
|
-
|
|
321
|
-
|
|
325
|
+
- `fleet-sync --schedule enable --apply --yes` (unattended runs every 30
|
|
326
|
+
minutes) is installed by that command on the manager, not yet by the
|
|
327
|
+
manager's playbook, and its interval is not yet part of `fleet.json`.
|
|
328
|
+
`fleet-sync --schedule status` lists its last runs, and `fleet-status` fails
|
|
329
|
+
`manager:schedule` when it has not converged the fleet for two intervals.
|
|
322
330
|
- Not everything on a machine is pinned. [What is declared, and
|
|
323
331
|
where](docs/declared.md) lists what is and what is not: most of a worker's
|
|
324
332
|
own packages, the manager's applications and every login are outside it.
|
|
@@ -327,18 +335,18 @@ fleet.json contents.
|
|
|
327
335
|
|
|
328
336
|
## Repository layout
|
|
329
337
|
|
|
330
|
-
| Path | What
|
|
331
|
-
| ------------ |
|
|
332
|
-
| `src/` | Inventory, orchestration, verification, the report; `src/cli` CLI
|
|
333
|
-
| `src/state/` | The fleet state database and its replication to the bucket
|
|
334
|
-
| `ansible/` | Machine policy: `ubuntu/`, `macos/`, `monitoring/`
|
|
335
|
-
| `agents/` | Agent launchers T3 runs, and their registry
|
|
336
|
-
| `cliproxy/` | The CLIProxyAPI service: install, audit, backup, model health
|
|
337
|
-
| `bootstrap/` | Toolchain bootstrap and the manager and worker entry scripts
|
|
338
|
-
| `schema/` | JSON schemas for `fleet.json
|
|
339
|
-
| `examples/` | A complete example configuration
|
|
340
|
-
| `features/` | What dev-fleet does, as Gherkin features: the tests
|
|
341
|
-
| `bin/` | The entry point of the published package
|
|
338
|
+
| Path | What |
|
|
339
|
+
| ------------ | ----------------------------------------------------------------------- |
|
|
340
|
+
| `src/` | Inventory, orchestration, verification, the report; `src/cli` CLI |
|
|
341
|
+
| `src/state/` | The fleet state database and its replication to the bucket |
|
|
342
|
+
| `ansible/` | Machine policy: `ubuntu/`, `macos/`, `monitoring/` |
|
|
343
|
+
| `agents/` | Agent launchers T3 runs, and their registry |
|
|
344
|
+
| `cliproxy/` | The CLIProxyAPI service: install, audit, backup, model health |
|
|
345
|
+
| `bootstrap/` | Toolchain bootstrap and the manager and worker entry scripts |
|
|
346
|
+
| `schema/` | JSON schemas for `fleet.json`, the roster and command output; check IDs |
|
|
347
|
+
| `examples/` | A complete example configuration |
|
|
348
|
+
| `features/` | What dev-fleet does, as Gherkin features: the tests |
|
|
349
|
+
| `bin/` | The entry point of the published package |
|
|
342
350
|
|
|
343
351
|
## Develop
|
|
344
352
|
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
---
|
|
2
|
+
# The Git identity and GitHub credentials every node commits and pushes with,
|
|
3
|
+
# the manager included, as the developer: no become. Written in Git's own
|
|
4
|
+
# per-user file (~/.config/git/config), inside a marked block, so a person's
|
|
5
|
+
# ~/.gitconfig is never touched. Variables arrive through a private 0600
|
|
6
|
+
# variables file, never argv.
|
|
7
|
+
- name: Converge the Git identity and GitHub credential helper
|
|
8
|
+
hosts: localhost
|
|
9
|
+
connection: local
|
|
10
|
+
become: false
|
|
11
|
+
gather_facts: false
|
|
12
|
+
vars:
|
|
13
|
+
git_config: '{{ git_home }}/.config/git/config'
|
|
14
|
+
# gh answers for GitHub through the pinned toolchain's shim, so any
|
|
15
|
+
# session's PATH works.
|
|
16
|
+
git_credential_helper: "!'{{ git_home }}/.local/share/mise/shims/gh' auth git-credential"
|
|
17
|
+
tasks:
|
|
18
|
+
- name: Require a declared identity
|
|
19
|
+
ansible.builtin.assert:
|
|
20
|
+
that:
|
|
21
|
+
- git_name | length > 0
|
|
22
|
+
- git_email is match('^[^@ ]+@[^@ ]+$')
|
|
23
|
+
- git_home is match('^/')
|
|
24
|
+
fail_msg: GIT_IDENTITY_UNDECLARED; declare git.name and git.email in fleet.json.
|
|
25
|
+
- name: Create the per-user Git configuration directory
|
|
26
|
+
ansible.builtin.file:
|
|
27
|
+
path: '{{ git_home }}/.config/git'
|
|
28
|
+
state: directory
|
|
29
|
+
mode: '0700'
|
|
30
|
+
- name: Declare the identity and the GitHub credential helper
|
|
31
|
+
ansible.builtin.blockinfile:
|
|
32
|
+
path: '{{ git_config }}'
|
|
33
|
+
create: true
|
|
34
|
+
mode: '0600'
|
|
35
|
+
marker: '# {mark} dev-fleet: Git identity and GitHub credentials'
|
|
36
|
+
block: |
|
|
37
|
+
[user]
|
|
38
|
+
name = "{{ git_name }}"
|
|
39
|
+
email = {{ git_email }}
|
|
40
|
+
[credential "https://github.com"]
|
|
41
|
+
helper =
|
|
42
|
+
helper = {{ git_credential_helper }}
|
|
43
|
+
[credential "https://gist.github.com"]
|
|
44
|
+
helper =
|
|
45
|
+
helper = {{ git_credential_helper }}
|
|
46
|
+
- name: Read the effective identity and GitHub credential helper
|
|
47
|
+
ansible.builtin.command:
|
|
48
|
+
argv: [git, config, --global, --get-all, '{{ item }}']
|
|
49
|
+
environment:
|
|
50
|
+
HOME: '{{ git_home }}'
|
|
51
|
+
loop:
|
|
52
|
+
- user.name
|
|
53
|
+
- user.email
|
|
54
|
+
- credential.https://github.com.helper
|
|
55
|
+
register: git_effective
|
|
56
|
+
check_mode: false
|
|
57
|
+
changed_when: false
|
|
58
|
+
failed_when: false
|
|
59
|
+
- name: Require the declared identity and helper to be the effective ones (worker:git-identity)
|
|
60
|
+
ansible.builtin.assert:
|
|
61
|
+
that:
|
|
62
|
+
- git_effective.results[0].stdout_lines | default([]) | last | default('') == git_name
|
|
63
|
+
- git_effective.results[1].stdout_lines | default([]) | last | default('') == git_email
|
|
64
|
+
- git_effective.results[2].stdout_lines | default([]) | last | default('') == git_credential_helper
|
|
65
|
+
fail_msg: GIT_IDENTITY_OVERRIDDEN; another Git configuration (often ~/.gitconfig) sets user.name, user.email or the GitHub credential helper after dev-fleet's. Remove those lines.
|
|
66
|
+
when: not ansible_check_mode
|
|
@@ -1,9 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 1,
|
|
3
3
|
"minimumMajorVersion": 14,
|
|
4
|
+
"architectures": ["arm64", "x86_64"],
|
|
4
5
|
"pmset": {
|
|
5
6
|
"sleep": "0",
|
|
6
7
|
"womp": "1"
|
|
7
8
|
},
|
|
8
|
-
"beszelDaemon": "dev.devfleet.beszel-agent"
|
|
9
|
+
"beszelDaemon": "dev.devfleet.beszel-agent",
|
|
10
|
+
"disk": {
|
|
11
|
+
"warnFreeMiB": 20480,
|
|
12
|
+
"failFreeMiB": 5120,
|
|
13
|
+
"dockerWarnGiB": 64
|
|
14
|
+
},
|
|
15
|
+
"containerWatchdog": {
|
|
16
|
+
"unhealthyChecks": 3,
|
|
17
|
+
"maxRestartsPerHour": 3,
|
|
18
|
+
"intervalSeconds": 60
|
|
19
|
+
}
|
|
9
20
|
}
|
|
@@ -5,6 +5,16 @@ services:
|
|
|
5
5
|
hub:
|
|
6
6
|
image: henrygd/beszel:0.21.1@sha256:8514a69d9bbbcb74f371a3205561ee231fc0289739cea64e8131f341d695b770
|
|
7
7
|
restart: unless-stopped
|
|
8
|
+
# Docker restarts a hub that exits; the container watchdog on the manager
|
|
9
|
+
# restarts one that stays unhealthy (container-watchdog.py).
|
|
10
|
+
labels:
|
|
11
|
+
dev.devfleet.watchdog: restart
|
|
12
|
+
healthcheck:
|
|
13
|
+
test: ['CMD', '/beszel', 'health', '--url', 'http://localhost:8090']
|
|
14
|
+
interval: 30s
|
|
15
|
+
timeout: 5s
|
|
16
|
+
retries: 3
|
|
17
|
+
start_period: 30s
|
|
8
18
|
ports:
|
|
9
19
|
- '127.0.0.1:8090:8090'
|
|
10
20
|
environment:
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
"""Restart a fleet container that stays unhealthy, a bounded number of times.
|
|
3
|
+
|
|
4
|
+
Managed by fleet-sync (ansible/monitoring/hub-darwin.yml) as the per-user
|
|
5
|
+
launchd agent dev.devfleet.container-watchdog on the manager. Docker restarts
|
|
6
|
+
a container that exits, never one that hangs while it still holds its port.
|
|
7
|
+
Every run reads the health of each container labelled
|
|
8
|
+
dev.devfleet.watchdog=restart. One that was unhealthy for --unhealthy-checks
|
|
9
|
+
runs in a row is restarted, at most --max-restarts-per-hour times an hour.
|
|
10
|
+
Past that cap it is left alone and recorded as capped: a restart storm is a
|
|
11
|
+
failing check (manager:container-watchdog), not a loop.
|
|
12
|
+
|
|
13
|
+
Every decision is one JSON line on stdout; nothing a container printed is
|
|
14
|
+
repeated. The state file holds counters only and is safe to delete.
|
|
15
|
+
|
|
16
|
+
Usage: container-watchdog.py --state FILE [--unhealthy-checks N]
|
|
17
|
+
[--max-restarts-per-hour N] [--dry-run]
|
|
18
|
+
"""
|
|
19
|
+
import argparse
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
import time
|
|
25
|
+
|
|
26
|
+
HOUR = 3600
|
|
27
|
+
LABEL = 'dev.devfleet.watchdog=restart'
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def log(**event):
|
|
31
|
+
print(json.dumps(event, sort_keys=True), flush=True)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def decide(memory, health, now, limit, cap):
|
|
35
|
+
"""The next memory, and the containers to restart now with the reason.
|
|
36
|
+
|
|
37
|
+
memory: {name: {"unhealthy": n, "restarts": [t, ...], "capped": t|None}}
|
|
38
|
+
health: {name: "healthy" | "unhealthy" | "starting" | "none"}
|
|
39
|
+
"""
|
|
40
|
+
after, restart = {}, []
|
|
41
|
+
for name, status in sorted(health.items()):
|
|
42
|
+
before = memory.get(name, {})
|
|
43
|
+
restarts = [t for t in before.get('restarts', []) if now - t < HOUR]
|
|
44
|
+
unhealthy = before.get('unhealthy', 0) + 1 if status == 'unhealthy' else 0
|
|
45
|
+
capped = before.get('capped')
|
|
46
|
+
if capped is not None and now - capped >= HOUR:
|
|
47
|
+
capped = None
|
|
48
|
+
if unhealthy >= limit:
|
|
49
|
+
if len(restarts) < cap:
|
|
50
|
+
restarts.append(now)
|
|
51
|
+
restart.append((name, f'unhealthy for {unhealthy} checks'))
|
|
52
|
+
unhealthy = 0
|
|
53
|
+
else:
|
|
54
|
+
capped = now
|
|
55
|
+
after[name] = {'unhealthy': unhealthy, 'restarts': restarts,
|
|
56
|
+
'capped': capped}
|
|
57
|
+
return after, restart
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def docker(*args):
|
|
61
|
+
return subprocess.run(['docker', *args], capture_output=True, text=True,
|
|
62
|
+
timeout=60)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def observe():
|
|
66
|
+
listed = docker('ps', '--filter', f'label={LABEL}', '--format', '{{.Names}}')
|
|
67
|
+
if listed.returncode:
|
|
68
|
+
return None
|
|
69
|
+
health = {}
|
|
70
|
+
for name in listed.stdout.split():
|
|
71
|
+
state = docker('inspect', '--format',
|
|
72
|
+
'{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}',
|
|
73
|
+
name)
|
|
74
|
+
health[name] = state.stdout.strip() if state.returncode == 0 else 'none'
|
|
75
|
+
return health
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def main(argv):
|
|
79
|
+
parser = argparse.ArgumentParser()
|
|
80
|
+
parser.add_argument('--state', required=True)
|
|
81
|
+
parser.add_argument('--unhealthy-checks', type=int, default=3)
|
|
82
|
+
parser.add_argument('--max-restarts-per-hour', type=int, default=3)
|
|
83
|
+
parser.add_argument('--dry-run', action='store_true')
|
|
84
|
+
args = parser.parse_args(argv)
|
|
85
|
+
try:
|
|
86
|
+
with open(args.state) as file:
|
|
87
|
+
memory = json.load(file)
|
|
88
|
+
if not isinstance(memory, dict):
|
|
89
|
+
memory = {}
|
|
90
|
+
except (OSError, ValueError):
|
|
91
|
+
memory = {}
|
|
92
|
+
health = observe()
|
|
93
|
+
if health is None:
|
|
94
|
+
log(event='docker_unavailable')
|
|
95
|
+
return 0
|
|
96
|
+
now = time.time()
|
|
97
|
+
memory, restart = decide(memory, health, now, args.unhealthy_checks,
|
|
98
|
+
args.max_restarts_per_hour)
|
|
99
|
+
for name, reason in restart:
|
|
100
|
+
ok = True
|
|
101
|
+
if not args.dry_run:
|
|
102
|
+
ok = docker('restart', name).returncode == 0
|
|
103
|
+
log(event='restarted', container=name, reason=reason, ok=ok,
|
|
104
|
+
dry_run=args.dry_run)
|
|
105
|
+
for name, entry in memory.items():
|
|
106
|
+
if entry['capped'] == now:
|
|
107
|
+
log(event='capped', container=name,
|
|
108
|
+
restarts=len(entry['restarts']))
|
|
109
|
+
os.makedirs(os.path.dirname(os.path.abspath(args.state)), exist_ok=True)
|
|
110
|
+
temporary = args.state + '.tmp'
|
|
111
|
+
with open(temporary, 'w') as file:
|
|
112
|
+
json.dump(memory, file, sort_keys=True)
|
|
113
|
+
os.replace(temporary, args.state)
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
if __name__ == '__main__':
|
|
118
|
+
sys.exit(main(sys.argv[1:]))
|
|
@@ -14,6 +14,12 @@
|
|
|
14
14
|
- compose
|
|
15
15
|
- '-f'
|
|
16
16
|
- '{{ playbook_dir }}/compose.yml'
|
|
17
|
+
watchdog_policy: "{{ (lookup('ansible.builtin.file', playbook_dir ~ '/../macos/policy.json') | from_json).containerWatchdog }}"
|
|
18
|
+
watchdog_label: dev.devfleet.container-watchdog
|
|
19
|
+
watchdog_root: '{{ ansible_facts.env.HOME }}/Library/Application Support/dev-fleet/container-watchdog'
|
|
20
|
+
watchdog_state: '{{ ansible_facts.env.HOME }}/Library/Caches/dev-fleet/container-watchdog.json'
|
|
21
|
+
watchdog_plist: '{{ ansible_facts.env.HOME }}/Library/LaunchAgents/{{ watchdog_label }}.plist'
|
|
22
|
+
watchdog_domain: 'gui/{{ ansible_facts.user_uid }}'
|
|
17
23
|
beszel_environment:
|
|
18
24
|
BESZEL_HUB_URL: '{{ beszel_hub_url }}'
|
|
19
25
|
BESZEL_APP_URL: '{{ beszel_public_url | default(beszel_hub_url) }}'
|
|
@@ -100,3 +106,143 @@
|
|
|
100
106
|
delay: 2
|
|
101
107
|
register: hub_health
|
|
102
108
|
until: hub_health.status == 200
|
|
109
|
+
# Scheduled off-machine backups, as the fleet configuration declares them
|
|
110
|
+
# (services.monitoring.backups), through the hub's own superuser API. Only
|
|
111
|
+
# what differs is changed; the storage keys never leave the variables file.
|
|
112
|
+
- name: Converge the hub's scheduled off-machine backups
|
|
113
|
+
when:
|
|
114
|
+
- beszel_backup_schedule is defined
|
|
115
|
+
- beszel_fleet_password | default('') | length > 0
|
|
116
|
+
block:
|
|
117
|
+
- name: Sign in to the hub as its superuser
|
|
118
|
+
ansible.builtin.uri:
|
|
119
|
+
url: http://127.0.0.1:8090/api/collections/_superusers/auth-with-password
|
|
120
|
+
method: POST
|
|
121
|
+
body_format: json
|
|
122
|
+
body:
|
|
123
|
+
identity: '{{ beszel_fleet_identity }}'
|
|
124
|
+
password: '{{ beszel_fleet_password }}'
|
|
125
|
+
return_content: true
|
|
126
|
+
check_mode: false
|
|
127
|
+
no_log: true
|
|
128
|
+
register: hub_superuser
|
|
129
|
+
- name: Read the hub's backup settings
|
|
130
|
+
ansible.builtin.uri:
|
|
131
|
+
url: http://127.0.0.1:8090/api/settings
|
|
132
|
+
headers:
|
|
133
|
+
Authorization: '{{ hub_superuser.json.token }}'
|
|
134
|
+
return_content: true
|
|
135
|
+
check_mode: false
|
|
136
|
+
no_log: true
|
|
137
|
+
register: hub_settings
|
|
138
|
+
- name: Schedule backups to the declared off-machine storage (BESZEL_BACKUP_CONVERGENCE_FAILED)
|
|
139
|
+
ansible.builtin.uri:
|
|
140
|
+
url: http://127.0.0.1:8090/api/settings
|
|
141
|
+
method: PATCH
|
|
142
|
+
headers:
|
|
143
|
+
Authorization: '{{ hub_superuser.json.token }}'
|
|
144
|
+
body_format: json
|
|
145
|
+
body:
|
|
146
|
+
backups:
|
|
147
|
+
cron: '{{ beszel_backup_schedule }}'
|
|
148
|
+
cronMaxKeep: '{{ beszel_backup_keep | int }}'
|
|
149
|
+
s3:
|
|
150
|
+
enabled: true
|
|
151
|
+
bucket: '{{ beszel_backup_bucket }}'
|
|
152
|
+
region: '{{ beszel_backup_region }}'
|
|
153
|
+
endpoint: '{{ beszel_backup_endpoint }}'
|
|
154
|
+
accessKey: '{{ beszel_backup_access_key }}'
|
|
155
|
+
secret: '{{ beszel_backup_secret_key }}'
|
|
156
|
+
forcePathStyle: '{{ beszel_backup_force_path_style | default(false) | bool }}'
|
|
157
|
+
no_log: true
|
|
158
|
+
changed_when: true
|
|
159
|
+
when: >-
|
|
160
|
+
(hub_settings.json.backups | default({})).cron | default('') != beszel_backup_schedule
|
|
161
|
+
or (hub_settings.json.backups | default({})).cronMaxKeep | default(0) | int != beszel_backup_keep | int
|
|
162
|
+
or not ((hub_settings.json.backups | default({})).s3 | default({})).enabled | default(false)
|
|
163
|
+
or ((hub_settings.json.backups | default({})).s3 | default({})).bucket | default('') != beszel_backup_bucket
|
|
164
|
+
or ((hub_settings.json.backups | default({})).s3 | default({})).region | default('') != beszel_backup_region
|
|
165
|
+
or ((hub_settings.json.backups | default({})).s3 | default({})).endpoint | default('') != beszel_backup_endpoint
|
|
166
|
+
or ((hub_settings.json.backups | default({})).s3 | default({})).accessKey | default('') != beszel_backup_access_key
|
|
167
|
+
or ((hub_settings.json.backups | default({})).s3 | default({})).forcePathStyle | default(false) != beszel_backup_force_path_style | default(false) | bool
|
|
168
|
+
# Docker restarts a container that exits, never one that hangs while it
|
|
169
|
+
# holds its port. The watchdog restarts a labelled container (the hub, the
|
|
170
|
+
# CLI proxy) that stays unhealthy, a bounded number of times an hour; the
|
|
171
|
+
# bounds are containerWatchdog in ansible/macos/policy.json, and
|
|
172
|
+
# manager:container-watchdog reports a container past them.
|
|
173
|
+
- name: Create the container watchdog's directories
|
|
174
|
+
ansible.builtin.file:
|
|
175
|
+
path: '{{ item }}'
|
|
176
|
+
state: directory
|
|
177
|
+
mode: '0700'
|
|
178
|
+
loop:
|
|
179
|
+
- '{{ watchdog_root }}'
|
|
180
|
+
- '{{ watchdog_state | dirname }}'
|
|
181
|
+
- '{{ ansible_facts.env.HOME }}/Library/LaunchAgents'
|
|
182
|
+
- name: Install the container watchdog
|
|
183
|
+
ansible.builtin.copy:
|
|
184
|
+
src: '{{ playbook_dir }}/container-watchdog.py'
|
|
185
|
+
dest: '{{ watchdog_root }}/container-watchdog.py'
|
|
186
|
+
mode: '0700'
|
|
187
|
+
- name: Install the container watchdog's launch agent
|
|
188
|
+
ansible.builtin.copy:
|
|
189
|
+
dest: '{{ watchdog_plist }}'
|
|
190
|
+
mode: '0644'
|
|
191
|
+
content: |
|
|
192
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
193
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
194
|
+
<!-- Managed by personal Fleet: container watchdog -->
|
|
195
|
+
<plist version="1.0"><dict>
|
|
196
|
+
<key>Label</key><string>{{ watchdog_label }}</string>
|
|
197
|
+
<key>ProgramArguments</key><array>
|
|
198
|
+
<string>/usr/bin/python3</string>
|
|
199
|
+
<string>-I</string>
|
|
200
|
+
<string>-B</string>
|
|
201
|
+
<string>{{ watchdog_root }}/container-watchdog.py</string>
|
|
202
|
+
<string>--state</string>
|
|
203
|
+
<string>{{ watchdog_state }}</string>
|
|
204
|
+
<string>--unhealthy-checks</string>
|
|
205
|
+
<string>{{ watchdog_policy.unhealthyChecks }}</string>
|
|
206
|
+
<string>--max-restarts-per-hour</string>
|
|
207
|
+
<string>{{ watchdog_policy.maxRestartsPerHour }}</string>
|
|
208
|
+
</array>
|
|
209
|
+
<key>EnvironmentVariables</key><dict>
|
|
210
|
+
<key>PATH</key><string>/usr/local/bin:/usr/bin:/bin:/Applications/Docker.app/Contents/Resources/bin</string>
|
|
211
|
+
</dict>
|
|
212
|
+
<key>StartInterval</key><integer>{{ watchdog_policy.intervalSeconds }}</integer>
|
|
213
|
+
<key>ProcessType</key><string>Background</string>
|
|
214
|
+
<key>Umask</key><integer>63</integer>
|
|
215
|
+
<key>StandardOutPath</key><string>{{ watchdog_root }}/watchdog.log</string>
|
|
216
|
+
<key>StandardErrorPath</key><string>{{ watchdog_root }}/watchdog.log</string>
|
|
217
|
+
</dict></plist>
|
|
218
|
+
register: watchdog_agent
|
|
219
|
+
- name: Inspect whether the container watchdog is loaded
|
|
220
|
+
ansible.builtin.command:
|
|
221
|
+
argv:
|
|
222
|
+
[/bin/launchctl, print, '{{ watchdog_domain }}/{{ watchdog_label }}']
|
|
223
|
+
changed_when: false
|
|
224
|
+
failed_when: false
|
|
225
|
+
register: watchdog_loaded
|
|
226
|
+
# launchd caches a loaded definition: a changed plist needs a reload.
|
|
227
|
+
- name: Unload a stale container watchdog definition
|
|
228
|
+
ansible.builtin.command:
|
|
229
|
+
argv:
|
|
230
|
+
[
|
|
231
|
+
/bin/launchctl,
|
|
232
|
+
bootout,
|
|
233
|
+
'{{ watchdog_domain }}/{{ watchdog_label }}',
|
|
234
|
+
]
|
|
235
|
+
when: watchdog_agent.changed and watchdog_loaded.rc == 0
|
|
236
|
+
failed_when: false
|
|
237
|
+
changed_when: true
|
|
238
|
+
- name: Load the container watchdog
|
|
239
|
+
ansible.builtin.command:
|
|
240
|
+
argv:
|
|
241
|
+
[
|
|
242
|
+
/bin/launchctl,
|
|
243
|
+
bootstrap,
|
|
244
|
+
'{{ watchdog_domain }}',
|
|
245
|
+
'{{ watchdog_plist }}',
|
|
246
|
+
]
|
|
247
|
+
when: watchdog_agent.changed or watchdog_loaded.rc != 0
|
|
248
|
+
changed_when: true
|
|
@@ -8,6 +8,11 @@ if [ "$(uname)" = Darwin ]; then
|
|
|
8
8
|
else
|
|
9
9
|
echo "ncpu=$(nproc)"
|
|
10
10
|
echo "load=$(cut -d' ' -f1 /proc/loadavg)"
|
|
11
|
+
# Free inodes as a percentage: bytes can be free while every write fails.
|
|
12
|
+
df -Pi "$HOME" | awk 'NR==2 && $2>0{print "inodefree=" int($4*100/$2)}'
|
|
13
|
+
# Clock synchronisation, as systemd sees it: 1 or 0.
|
|
14
|
+
timedatectl show -p NTPSynchronized -p NTP 2>/dev/null | awk -F= '{v=($2=="yes")?1:0} $1=="NTP"{print "ntp=" v} $1=="NTPSynchronized"{print "ntpsync=" v}'
|
|
11
15
|
awk '/MemTotal/{t=$2} /MemAvailable/{a=$2} /SwapTotal/{st=$2} /SwapFree/{sf=$2} END{print "memfree=" int(a*100/t); if (st>0) print "swap=" int((st-sf)*100/st); else print "swap=0"}' /proc/meminfo
|
|
12
16
|
fi
|
|
13
17
|
echo "diskmb=$(df -Pk "$HOME" | awk 'NR==2{print int($4/1024)}')"
|
|
18
|
+
echo "epoch=$(date +%s)"
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"rows": [
|
|
4
|
+
{
|
|
5
|
+
"role": "manager",
|
|
6
|
+
"platform": "any other",
|
|
7
|
+
"versions": "any",
|
|
8
|
+
"architectures": "any",
|
|
9
|
+
"status": "refused",
|
|
10
|
+
"note": "The fleet configuration accepts only a macOS manager."
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"role": "worker",
|
|
14
|
+
"platform": "Ubuntu",
|
|
15
|
+
"versions": "any other release",
|
|
16
|
+
"architectures": "any",
|
|
17
|
+
"status": "refused",
|
|
18
|
+
"note": "UBUNTU_BASELINE_MISMATCH"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"role": "worker",
|
|
22
|
+
"platform": "Ubuntu",
|
|
23
|
+
"versions": "an accepted release",
|
|
24
|
+
"architectures": "any other",
|
|
25
|
+
"status": "refused",
|
|
26
|
+
"note": "UBUNTU_BASELINE_MISMATCH"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"role": "worker",
|
|
30
|
+
"platform": "Debian",
|
|
31
|
+
"versions": "any",
|
|
32
|
+
"architectures": "any",
|
|
33
|
+
"status": "refused",
|
|
34
|
+
"note": "UBUNTU_BASELINE_MISMATCH"
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"role": "worker",
|
|
38
|
+
"platform": "macOS",
|
|
39
|
+
"versions": "any",
|
|
40
|
+
"architectures": "any",
|
|
41
|
+
"status": "refused",
|
|
42
|
+
"note": "The fleet configuration accepts only Linux workers."
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"role": "worker",
|
|
46
|
+
"platform": "Windows",
|
|
47
|
+
"versions": "any",
|
|
48
|
+
"architectures": "any",
|
|
49
|
+
"status": "refused",
|
|
50
|
+
"note": "The fleet configuration accepts only Linux workers."
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"role": "worker",
|
|
54
|
+
"platform": "WSL2",
|
|
55
|
+
"environment": "wsl",
|
|
56
|
+
"versions": "any",
|
|
57
|
+
"architectures": "any",
|
|
58
|
+
"status": "refused",
|
|
59
|
+
"note": "WORKER_ENVIRONMENT_UNSUPPORTED"
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"role": "worker",
|
|
63
|
+
"platform": "container",
|
|
64
|
+
"environment": "container",
|
|
65
|
+
"versions": "any",
|
|
66
|
+
"architectures": "any",
|
|
67
|
+
"status": "refused",
|
|
68
|
+
"note": "WORKER_ENVIRONMENT_UNSUPPORTED"
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"role": "toolchain only",
|
|
72
|
+
"platform": "macOS",
|
|
73
|
+
"versions": "any",
|
|
74
|
+
"architectures": "arm64, x86_64",
|
|
75
|
+
"status": "best effort",
|
|
76
|
+
"note": "No fleet role: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"role": "toolchain only",
|
|
80
|
+
"platform": "Linux",
|
|
81
|
+
"versions": "any",
|
|
82
|
+
"architectures": "x86_64, aarch64",
|
|
83
|
+
"status": "best effort",
|
|
84
|
+
"note": "No fleet role: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"role": "toolchain only",
|
|
88
|
+
"platform": "WSL2",
|
|
89
|
+
"versions": "any",
|
|
90
|
+
"architectures": "x86_64",
|
|
91
|
+
"status": "best effort",
|
|
92
|
+
"note": "Treated as Linux: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"role": "toolchain only",
|
|
96
|
+
"platform": "Windows",
|
|
97
|
+
"versions": "any",
|
|
98
|
+
"architectures": "x64",
|
|
99
|
+
"status": "best effort",
|
|
100
|
+
"note": "No fleet role: bootstrap/bootstrap.ps1 installs the pinned tools (docs/machine-setup.md)."
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
"role": "toolchain only",
|
|
104
|
+
"platform": "Alpine",
|
|
105
|
+
"versions": "any",
|
|
106
|
+
"architectures": "any",
|
|
107
|
+
"status": "refused",
|
|
108
|
+
"note": "musl: the pinned mise build needs glibc."
|
|
109
|
+
}
|
|
110
|
+
]
|
|
111
|
+
}
|
|
@@ -3,14 +3,11 @@
|
|
|
3
3
|
# SSH terminal so sudo reads the operator's password itself; never capture it.
|
|
4
4
|
set -eu
|
|
5
5
|
umask 077
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
[ "$(sw_vers -productVersion | cut -d. -f1)" -ge 14 ] || exit 1 ;;
|
|
12
|
-
*) exit 1 ;;
|
|
13
|
-
esac
|
|
6
|
+
# Workers are Ubuntu. Which releases is checked by the bootstrap that
|
|
7
|
+
# follows, against ansible/ubuntu/policy.json; this only grants sudo.
|
|
8
|
+
[ "$(uname -s)" = Linux ] || exit 1
|
|
9
|
+
. /etc/os-release
|
|
10
|
+
[ "$ID" = ubuntu ] || exit 1
|
|
14
11
|
user=$(id -un)
|
|
15
12
|
case "$user" in root|''|*[!a-z0-9_-]*) exit 1;; esac
|
|
16
13
|
command -v sudo >/dev/null 2>&1 || exit 1
|
|
@@ -23,11 +20,7 @@ mkdir -p "$root"
|
|
|
23
20
|
chmod 700 "$root"
|
|
24
21
|
[ ! -L "$root/remote-operation.lock" ] || exit 1
|
|
25
22
|
exec 9>"$root/remote-operation.lock"
|
|
26
|
-
|
|
27
|
-
flock -n 9 || exit 1
|
|
28
|
-
else
|
|
29
|
-
python3 -c 'import fcntl; fcntl.flock(9, fcntl.LOCK_EX | fcntl.LOCK_NB)' 9>&9 || exit 1
|
|
30
|
-
fi
|
|
23
|
+
flock -n 9 || exit 1
|
|
31
24
|
sudo -v
|
|
32
25
|
file=$(mktemp)
|
|
33
26
|
trap 'rm -f "$file"' EXIT
|
|
@@ -45,7 +38,5 @@ if sudo -n test -e "$target"; then
|
|
|
45
38
|
}
|
|
46
39
|
sudo -n cp -p -- "$target" "$target.before-fleet-enrollment"
|
|
47
40
|
fi
|
|
48
|
-
|
|
49
|
-
[ "$(uname -s)" != Darwin ] || group=wheel
|
|
50
|
-
sudo -n install -o root -g "$group" -m 0440 -- "$file" "$target"
|
|
41
|
+
sudo -n install -o root -g root -m 0440 -- "$file" "$target"
|
|
51
42
|
sudo -n true
|