@crvouga/dev-fleet 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +25 -17
  2. package/ansible/github/git.yml +66 -0
  3. package/ansible/macos/policy.json +12 -1
  4. package/ansible/monitoring/compose.yml +10 -0
  5. package/ansible/monitoring/container-watchdog.py +118 -0
  6. package/ansible/monitoring/hub-darwin.yml +146 -0
  7. package/ansible/net/remote-probe.sh +5 -0
  8. package/ansible/platforms.json +111 -0
  9. package/ansible/ubuntu/authorize-sudo.sh +7 -16
  10. package/ansible/ubuntu/baseline.yml +6 -28
  11. package/ansible/ubuntu/disk-gc.py +211 -7
  12. package/ansible/ubuntu/disk-guard.py +24 -5
  13. package/ansible/ubuntu/policy.json +68 -4
  14. package/ansible/ubuntu/t3-desktop.yml +2 -2
  15. package/ansible/ubuntu/tailscale.yml +20 -9
  16. package/ansible/ubuntu/tasks/browser.yml +56 -0
  17. package/ansible/ubuntu/tasks/docker.yml +52 -0
  18. package/ansible/ubuntu/tasks/dpkg.yml +46 -0
  19. package/ansible/ubuntu/tasks/install.yml +38 -0
  20. package/ansible/ubuntu/tasks/packages.yml +4 -55
  21. package/ansible/ubuntu/tasks/release.yml +25 -0
  22. package/ansible/ubuntu/tasks/resources.yml +7 -6
  23. package/ansible/ubuntu/tasks/services.yml +13 -0
  24. package/ansible/ubuntu/tasks/snapshot.yml +50 -0
  25. package/ansible/ubuntu/tasks/unattended-upgrades.yml +24 -0
  26. package/bootstrap/doctor.ts +32 -6
  27. package/bootstrap/fleet-remote-bootstrap.sh +49 -45
  28. package/bootstrap/setup.ts +23 -8
  29. package/bootstrap/worker-tools.ts +118 -0
  30. package/cliproxy/README.md +1 -1
  31. package/cliproxy/compose.yaml +4 -0
  32. package/cliproxy/scripts/checkpoint.py +1 -1
  33. package/docs/authentication.md +90 -0
  34. package/docs/consumer-setup.md +32 -30
  35. package/docs/declared.md +6 -5
  36. package/docs/machine-setup.md +22 -12
  37. package/docs/operations.md +87 -31
  38. package/docs/runbooks.md +377 -0
  39. package/docs/state.md +13 -7
  40. package/docs/support-matrix.md +36 -0
  41. package/docs/ubuntu-sync.md +16 -13
  42. package/docs/worker-contract.md +28 -1
  43. package/examples/fleet.json +19 -1
  44. package/llms.txt +2 -0
  45. package/mise.toml +7 -0
  46. package/package.json +2 -2
  47. package/schema/check-ids.json +526 -0
  48. package/schema/fleet.schema.json +185 -4
  49. package/schema/output/check.schema.json +23 -0
  50. package/schema/output/error.schema.json +14 -0
  51. package/schema/output/fleet-report.schema.json +79 -0
  52. package/schema/output/fleet-sync.schema.json +112 -0
  53. package/schema/output/report.schema.json +38 -0
  54. package/src/auth-checks.ts +3 -0
  55. package/src/cli/agent-guide.ts +5 -3
  56. package/src/cli/fleet-report.ts +126 -0
  57. package/src/cli/fleet-sync-manager.ts +11 -0
  58. package/src/cli/fleet-sync-plan.ts +200 -0
  59. package/src/cli/fleet-sync-remote.ts +4 -2
  60. package/src/cli/fleet-sync-schedule.ts +142 -35
  61. package/src/cli/fleet-sync.ts +90 -8
  62. package/src/cli/fleet-watch.ts +198 -0
  63. package/src/cli/fleet.ts +4 -0
  64. package/src/cli/register-fleet.ts +51 -0
  65. package/src/cli/registry.ts +13 -0
  66. package/src/cli/schedule-logs.ts +36 -0
  67. package/src/clock.ts +63 -0
  68. package/src/definition.ts +160 -12
  69. package/src/git-identity.ts +160 -0
  70. package/src/machine/platform.ts +48 -28
  71. package/src/machine/support-matrix.ts +76 -0
  72. package/src/machine-health.ts +51 -4
  73. package/src/manager-health.ts +205 -0
  74. package/src/manager-sync.ts +27 -9
  75. package/src/monitoring/hub-backup.ts +63 -17
  76. package/src/monitoring/manager-hub.ts +44 -2
  77. package/src/network-definition.ts +1 -1
  78. package/src/node-converge-linux.ts +16 -5
  79. package/src/node-converge.ts +1 -1
  80. package/src/notify.ts +239 -0
  81. package/src/platform/debian-version.ts +62 -0
  82. package/src/platform/ubuntu-audit.ts +11 -4
  83. package/src/platform/ubuntu-release.ts +41 -0
  84. package/src/remote-bootstrap.ts +89 -25
  85. package/src/remote-operation.ts +213 -0
  86. package/src/schedule-check.ts +40 -0
  87. package/src/secrets.ts +26 -8
  88. package/src/ssh.ts +10 -2
  89. package/src/state/history.ts +172 -0
  90. package/src/state/store.ts +11 -4
  91. package/src/tailscale.ts +51 -0
  92. package/src/toolchain-checks.ts +8 -3
  93. package/src/worker-capability-checks.ts +167 -0
  94. package/src/worker-capacity.ts +19 -7
  95. package/src/worker-host-checks.ts +68 -10
  96. package/src/worker-policy.ts +33 -0
  97. package/src/worker-readiness.ts +10 -1
package/README.md CHANGED
@@ -42,7 +42,9 @@ dev-fleet is opinionated. These are part of it, not options:
42
42
  - The **manager** is a macOS machine. You run `fleet-sync` on it, and it hosts
43
43
  the shared services (the CLI proxy and the monitoring hub) in Docker Desktop.
44
44
  - **Workers** are Ubuntu 26.04 machines reached over SSH. They are stateless:
45
- everything on them is rebuilt from Git and your secrets.
45
+ everything on them is rebuilt from Git and your secrets. The
46
+ [support matrix](docs/support-matrix.md) lists every platform and what
47
+ happens to it.
46
48
  - The fleet state is kept in an **S3-compatible bucket** you own.
47
49
  - Machines reach each other over **Tailscale**.
48
50
  - Coding agents (Claude Code, Codex, Grok, OpenCode, Antigravity) run through
@@ -98,7 +100,11 @@ Ansible policy and verifies the result. It prints one table and writes a full
98
100
  JSON report. The first run on a machine installs everything, mise included, so
99
101
  Git is the only tool it needs beforehand.
100
102
 
101
- `--dry-run` changes no machine. On the manager it checks the release out into
103
+ `--dry-run` changes no machine. It first brings the manager's copy of the
104
+ fleet state up to the bucket (never uploading), then prints, and adds to
105
+ `--json` as `plan`, what a sync would change: the revision, each tool pin that
106
+ moves, and what restarts because of it (a T3 restart ends its running agent
107
+ turns). On the manager it checks the release out into
102
108
  dev-fleet's cache and installs the Bun that release pins beside any others, so
103
109
  that it can run; it never installs the toolchain, publishes the global mise
104
110
  configuration, edits shell profiles or touches a worker.
@@ -316,9 +322,11 @@ fleet.json contents.
316
322
  `fleet-sync` fetches that copy again when it is deleted, but the proxy then
317
323
  needs `devfleet cliproxy install` once more.
318
324
  - The manager is a Mac host. There is no container image of the manager.
319
- - `fleet-sync --schedule enable` (unattended runs on a timer) is not available
320
- from the published package yet, so nothing re-applies the fleet until you
321
- run `fleet-sync`.
325
+ - `fleet-sync --schedule enable --apply --yes` (unattended runs every 30
326
+ minutes) is installed by that command on the manager, not yet by the
327
+ manager's playbook, and its interval is not yet part of `fleet.json`.
328
+ `fleet-sync --schedule status` lists its last runs, and `fleet-status` fails
329
+ `manager:schedule` when it has not converged the fleet for two intervals.
322
330
  - Not everything on a machine is pinned. [What is declared, and
323
331
  where](docs/declared.md) lists what is and what is not: most of a worker's
324
332
  own packages, the manager's applications and every login are outside it.
@@ -327,18 +335,18 @@ fleet.json contents.
327
335
 
328
336
  ## Repository layout
329
337
 
330
- | Path | What |
331
- | ------------ | ----------------------------------------------------------------- |
332
- | `src/` | Inventory, orchestration, verification, the report; `src/cli` CLI |
333
- | `src/state/` | The fleet state database and its replication to the bucket |
334
- | `ansible/` | Machine policy: `ubuntu/`, `macos/`, `monitoring/` |
335
- | `agents/` | Agent launchers T3 runs, and their registry |
336
- | `cliproxy/` | The CLIProxyAPI service: install, audit, backup, model health |
337
- | `bootstrap/` | Toolchain bootstrap and the manager and worker entry scripts |
338
- | `schema/` | JSON schemas for `fleet.json` and the private roster |
339
- | `examples/` | A complete example configuration |
340
- | `features/` | What dev-fleet does, as Gherkin features: the tests |
341
- | `bin/` | The entry point of the published package |
338
+ | Path | What |
339
+ | ------------ | ----------------------------------------------------------------------- |
340
+ | `src/` | Inventory, orchestration, verification, the report; `src/cli` CLI |
341
+ | `src/state/` | The fleet state database and its replication to the bucket |
342
+ | `ansible/` | Machine policy: `ubuntu/`, `macos/`, `monitoring/` |
343
+ | `agents/` | Agent launchers T3 runs, and their registry |
344
+ | `cliproxy/` | The CLIProxyAPI service: install, audit, backup, model health |
345
+ | `bootstrap/` | Toolchain bootstrap and the manager and worker entry scripts |
346
+ | `schema/` | JSON schemas for `fleet.json`, the roster and command output; check IDs |
347
+ | `examples/` | A complete example configuration |
348
+ | `features/` | What dev-fleet does, as Gherkin features: the tests |
349
+ | `bin/` | The entry point of the published package |
342
350
 
343
351
  ## Develop
344
352
 
@@ -0,0 +1,66 @@
1
+ ---
2
+ # The Git identity and GitHub credentials every node commits and pushes with,
3
+ # the manager included, as the developer: no become. Written in Git's own
4
+ # per-user file (~/.config/git/config), inside a marked block, so a person's
5
+ # ~/.gitconfig is never touched. Variables arrive through a private 0600
6
+ # variables file, never argv.
7
+ - name: Converge the Git identity and GitHub credential helper
8
+ hosts: localhost
9
+ connection: local
10
+ become: false
11
+ gather_facts: false
12
+ vars:
13
+ git_config: '{{ git_home }}/.config/git/config'
14
+ # gh answers for GitHub through the pinned toolchain's shim, so any
15
+ # session's PATH works.
16
+ git_credential_helper: "!'{{ git_home }}/.local/share/mise/shims/gh' auth git-credential"
17
+ tasks:
18
+ - name: Require a declared identity
19
+ ansible.builtin.assert:
20
+ that:
21
+ - git_name | length > 0
22
+ - git_email is match('^[^@ ]+@[^@ ]+$')
23
+ - git_home is match('^/')
24
+ fail_msg: GIT_IDENTITY_UNDECLARED; declare git.name and git.email in fleet.json.
25
+ - name: Create the per-user Git configuration directory
26
+ ansible.builtin.file:
27
+ path: '{{ git_home }}/.config/git'
28
+ state: directory
29
+ mode: '0700'
30
+ - name: Declare the identity and the GitHub credential helper
31
+ ansible.builtin.blockinfile:
32
+ path: '{{ git_config }}'
33
+ create: true
34
+ mode: '0600'
35
+ marker: '# {mark} dev-fleet: Git identity and GitHub credentials'
36
+ block: |
37
+ [user]
38
+ name = "{{ git_name }}"
39
+ email = {{ git_email }}
40
+ [credential "https://github.com"]
41
+ helper =
42
+ helper = {{ git_credential_helper }}
43
+ [credential "https://gist.github.com"]
44
+ helper =
45
+ helper = {{ git_credential_helper }}
46
+ - name: Read the effective identity and GitHub credential helper
47
+ ansible.builtin.command:
48
+ argv: [git, config, --global, --get-all, '{{ item }}']
49
+ environment:
50
+ HOME: '{{ git_home }}'
51
+ loop:
52
+ - user.name
53
+ - user.email
54
+ - credential.https://github.com.helper
55
+ register: git_effective
56
+ check_mode: false
57
+ changed_when: false
58
+ failed_when: false
59
+ - name: Require the declared identity and helper to be the effective ones (worker:git-identity)
60
+ ansible.builtin.assert:
61
+ that:
62
+ - git_effective.results[0].stdout_lines | default([]) | last | default('') == git_name
63
+ - git_effective.results[1].stdout_lines | default([]) | last | default('') == git_email
64
+ - git_effective.results[2].stdout_lines | default([]) | last | default('') == git_credential_helper
65
+ fail_msg: GIT_IDENTITY_OVERRIDDEN; another Git configuration (often ~/.gitconfig) sets user.name, user.email or the GitHub credential helper after dev-fleet's. Remove those lines.
66
+ when: not ansible_check_mode
@@ -1,9 +1,20 @@
1
1
  {
2
2
  "version": 1,
3
3
  "minimumMajorVersion": 14,
4
+ "architectures": ["arm64", "x86_64"],
4
5
  "pmset": {
5
6
  "sleep": "0",
6
7
  "womp": "1"
7
8
  },
8
- "beszelDaemon": "dev.devfleet.beszel-agent"
9
+ "beszelDaemon": "dev.devfleet.beszel-agent",
10
+ "disk": {
11
+ "warnFreeMiB": 20480,
12
+ "failFreeMiB": 5120,
13
+ "dockerWarnGiB": 64
14
+ },
15
+ "containerWatchdog": {
16
+ "unhealthyChecks": 3,
17
+ "maxRestartsPerHour": 3,
18
+ "intervalSeconds": 60
19
+ }
9
20
  }
@@ -5,6 +5,16 @@ services:
5
5
  hub:
6
6
  image: henrygd/beszel:0.21.1@sha256:8514a69d9bbbcb74f371a3205561ee231fc0289739cea64e8131f341d695b770
7
7
  restart: unless-stopped
8
+ # Docker restarts a hub that exits; the container watchdog on the manager
9
+ # restarts one that stays unhealthy (container-watchdog.py).
10
+ labels:
11
+ dev.devfleet.watchdog: restart
12
+ healthcheck:
13
+ test: ['CMD', '/beszel', 'health', '--url', 'http://localhost:8090']
14
+ interval: 30s
15
+ timeout: 5s
16
+ retries: 3
17
+ start_period: 30s
8
18
  ports:
9
19
  - '127.0.0.1:8090:8090'
10
20
  environment:
@@ -0,0 +1,118 @@
1
+ #!/usr/bin/python3
2
+ """Restart a fleet container that stays unhealthy, a bounded number of times.
3
+
4
+ Managed by fleet-sync (ansible/monitoring/hub-darwin.yml) as the per-user
5
+ launchd agent dev.devfleet.container-watchdog on the manager. Docker restarts
6
+ a container that exits, never one that hangs while it still holds its port.
7
+ Every run reads the health of each container labelled
8
+ dev.devfleet.watchdog=restart. One that was unhealthy for --unhealthy-checks
9
+ runs in a row is restarted, at most --max-restarts-per-hour times an hour.
10
+ Past that cap it is left alone and recorded as capped: a restart storm is a
11
+ failing check (manager:container-watchdog), not a loop.
12
+
13
+ Every decision is one JSON line on stdout; nothing a container printed is
14
+ repeated. The state file holds counters only and is safe to delete.
15
+
16
+ Usage: container-watchdog.py --state FILE [--unhealthy-checks N]
17
+ [--max-restarts-per-hour N] [--dry-run]
18
+ """
19
+ import argparse
20
+ import json
21
+ import os
22
+ import subprocess
23
+ import sys
24
+ import time
25
+
26
+ HOUR = 3600
27
+ LABEL = 'dev.devfleet.watchdog=restart'
28
+
29
+
30
+ def log(**event):
31
+ print(json.dumps(event, sort_keys=True), flush=True)
32
+
33
+
34
+ def decide(memory, health, now, limit, cap):
35
+ """The next memory, and the containers to restart now with the reason.
36
+
37
+ memory: {name: {"unhealthy": n, "restarts": [t, ...], "capped": t|None}}
38
+ health: {name: "healthy" | "unhealthy" | "starting" | "none"}
39
+ """
40
+ after, restart = {}, []
41
+ for name, status in sorted(health.items()):
42
+ before = memory.get(name, {})
43
+ restarts = [t for t in before.get('restarts', []) if now - t < HOUR]
44
+ unhealthy = before.get('unhealthy', 0) + 1 if status == 'unhealthy' else 0
45
+ capped = before.get('capped')
46
+ if capped is not None and now - capped >= HOUR:
47
+ capped = None
48
+ if unhealthy >= limit:
49
+ if len(restarts) < cap:
50
+ restarts.append(now)
51
+ restart.append((name, f'unhealthy for {unhealthy} checks'))
52
+ unhealthy = 0
53
+ else:
54
+ capped = now
55
+ after[name] = {'unhealthy': unhealthy, 'restarts': restarts,
56
+ 'capped': capped}
57
+ return after, restart
58
+
59
+
60
+ def docker(*args):
61
+ return subprocess.run(['docker', *args], capture_output=True, text=True,
62
+ timeout=60)
63
+
64
+
65
+ def observe():
66
+ listed = docker('ps', '--filter', f'label={LABEL}', '--format', '{{.Names}}')
67
+ if listed.returncode:
68
+ return None
69
+ health = {}
70
+ for name in listed.stdout.split():
71
+ state = docker('inspect', '--format',
72
+ '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}',
73
+ name)
74
+ health[name] = state.stdout.strip() if state.returncode == 0 else 'none'
75
+ return health
76
+
77
+
78
+ def main(argv):
79
+ parser = argparse.ArgumentParser()
80
+ parser.add_argument('--state', required=True)
81
+ parser.add_argument('--unhealthy-checks', type=int, default=3)
82
+ parser.add_argument('--max-restarts-per-hour', type=int, default=3)
83
+ parser.add_argument('--dry-run', action='store_true')
84
+ args = parser.parse_args(argv)
85
+ try:
86
+ with open(args.state) as file:
87
+ memory = json.load(file)
88
+ if not isinstance(memory, dict):
89
+ memory = {}
90
+ except (OSError, ValueError):
91
+ memory = {}
92
+ health = observe()
93
+ if health is None:
94
+ log(event='docker_unavailable')
95
+ return 0
96
+ now = time.time()
97
+ memory, restart = decide(memory, health, now, args.unhealthy_checks,
98
+ args.max_restarts_per_hour)
99
+ for name, reason in restart:
100
+ ok = True
101
+ if not args.dry_run:
102
+ ok = docker('restart', name).returncode == 0
103
+ log(event='restarted', container=name, reason=reason, ok=ok,
104
+ dry_run=args.dry_run)
105
+ for name, entry in memory.items():
106
+ if entry['capped'] == now:
107
+ log(event='capped', container=name,
108
+ restarts=len(entry['restarts']))
109
+ os.makedirs(os.path.dirname(os.path.abspath(args.state)), exist_ok=True)
110
+ temporary = args.state + '.tmp'
111
+ with open(temporary, 'w') as file:
112
+ json.dump(memory, file, sort_keys=True)
113
+ os.replace(temporary, args.state)
114
+ return 0
115
+
116
+
117
+ if __name__ == '__main__':
118
+ sys.exit(main(sys.argv[1:]))
@@ -14,6 +14,12 @@
14
14
  - compose
15
15
  - '-f'
16
16
  - '{{ playbook_dir }}/compose.yml'
17
+ watchdog_policy: "{{ (lookup('ansible.builtin.file', playbook_dir ~ '/../macos/policy.json') | from_json).containerWatchdog }}"
18
+ watchdog_label: dev.devfleet.container-watchdog
19
+ watchdog_root: '{{ ansible_facts.env.HOME }}/Library/Application Support/dev-fleet/container-watchdog'
20
+ watchdog_state: '{{ ansible_facts.env.HOME }}/Library/Caches/dev-fleet/container-watchdog.json'
21
+ watchdog_plist: '{{ ansible_facts.env.HOME }}/Library/LaunchAgents/{{ watchdog_label }}.plist'
22
+ watchdog_domain: 'gui/{{ ansible_facts.user_uid }}'
17
23
  beszel_environment:
18
24
  BESZEL_HUB_URL: '{{ beszel_hub_url }}'
19
25
  BESZEL_APP_URL: '{{ beszel_public_url | default(beszel_hub_url) }}'
@@ -100,3 +106,143 @@
100
106
  delay: 2
101
107
  register: hub_health
102
108
  until: hub_health.status == 200
109
+ # Scheduled off-machine backups, as the fleet configuration declares them
110
+ # (services.monitoring.backups), through the hub's own superuser API. Only
111
+ # what differs is changed; the storage keys never leave the variables file.
112
+ - name: Converge the hub's scheduled off-machine backups
113
+ when:
114
+ - beszel_backup_schedule is defined
115
+ - beszel_fleet_password | default('') | length > 0
116
+ block:
117
+ - name: Sign in to the hub as its superuser
118
+ ansible.builtin.uri:
119
+ url: http://127.0.0.1:8090/api/collections/_superusers/auth-with-password
120
+ method: POST
121
+ body_format: json
122
+ body:
123
+ identity: '{{ beszel_fleet_identity }}'
124
+ password: '{{ beszel_fleet_password }}'
125
+ return_content: true
126
+ check_mode: false
127
+ no_log: true
128
+ register: hub_superuser
129
+ - name: Read the hub's backup settings
130
+ ansible.builtin.uri:
131
+ url: http://127.0.0.1:8090/api/settings
132
+ headers:
133
+ Authorization: '{{ hub_superuser.json.token }}'
134
+ return_content: true
135
+ check_mode: false
136
+ no_log: true
137
+ register: hub_settings
138
+ - name: Schedule backups to the declared off-machine storage (BESZEL_BACKUP_CONVERGENCE_FAILED)
139
+ ansible.builtin.uri:
140
+ url: http://127.0.0.1:8090/api/settings
141
+ method: PATCH
142
+ headers:
143
+ Authorization: '{{ hub_superuser.json.token }}'
144
+ body_format: json
145
+ body:
146
+ backups:
147
+ cron: '{{ beszel_backup_schedule }}'
148
+ cronMaxKeep: '{{ beszel_backup_keep | int }}'
149
+ s3:
150
+ enabled: true
151
+ bucket: '{{ beszel_backup_bucket }}'
152
+ region: '{{ beszel_backup_region }}'
153
+ endpoint: '{{ beszel_backup_endpoint }}'
154
+ accessKey: '{{ beszel_backup_access_key }}'
155
+ secret: '{{ beszel_backup_secret_key }}'
156
+ forcePathStyle: '{{ beszel_backup_force_path_style | default(false) | bool }}'
157
+ no_log: true
158
+ changed_when: true
159
+ when: >-
160
+ (hub_settings.json.backups | default({})).cron | default('') != beszel_backup_schedule
161
+ or (hub_settings.json.backups | default({})).cronMaxKeep | default(0) | int != beszel_backup_keep | int
162
+ or not ((hub_settings.json.backups | default({})).s3 | default({})).enabled | default(false)
163
+ or ((hub_settings.json.backups | default({})).s3 | default({})).bucket | default('') != beszel_backup_bucket
164
+ or ((hub_settings.json.backups | default({})).s3 | default({})).region | default('') != beszel_backup_region
165
+ or ((hub_settings.json.backups | default({})).s3 | default({})).endpoint | default('') != beszel_backup_endpoint
166
+ or ((hub_settings.json.backups | default({})).s3 | default({})).accessKey | default('') != beszel_backup_access_key
167
+ or ((hub_settings.json.backups | default({})).s3 | default({})).forcePathStyle | default(false) != beszel_backup_force_path_style | default(false) | bool
168
+ # Docker restarts a container that exits, never one that hangs while it
169
+ # holds its port. The watchdog restarts a labelled container (the hub, the
170
+ # CLI proxy) that stays unhealthy, a bounded number of times an hour; the
171
+ # bounds are containerWatchdog in ansible/macos/policy.json, and
172
+ # manager:container-watchdog reports a container past them.
173
+ - name: Create the container watchdog's directories
174
+ ansible.builtin.file:
175
+ path: '{{ item }}'
176
+ state: directory
177
+ mode: '0700'
178
+ loop:
179
+ - '{{ watchdog_root }}'
180
+ - '{{ watchdog_state | dirname }}'
181
+ - '{{ ansible_facts.env.HOME }}/Library/LaunchAgents'
182
+ - name: Install the container watchdog
183
+ ansible.builtin.copy:
184
+ src: '{{ playbook_dir }}/container-watchdog.py'
185
+ dest: '{{ watchdog_root }}/container-watchdog.py'
186
+ mode: '0700'
187
+ - name: Install the container watchdog's launch agent
188
+ ansible.builtin.copy:
189
+ dest: '{{ watchdog_plist }}'
190
+ mode: '0644'
191
+ content: |
192
+ <?xml version="1.0" encoding="UTF-8"?>
193
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
194
+ <!-- Managed by personal Fleet: container watchdog -->
195
+ <plist version="1.0"><dict>
196
+ <key>Label</key><string>{{ watchdog_label }}</string>
197
+ <key>ProgramArguments</key><array>
198
+ <string>/usr/bin/python3</string>
199
+ <string>-I</string>
200
+ <string>-B</string>
201
+ <string>{{ watchdog_root }}/container-watchdog.py</string>
202
+ <string>--state</string>
203
+ <string>{{ watchdog_state }}</string>
204
+ <string>--unhealthy-checks</string>
205
+ <string>{{ watchdog_policy.unhealthyChecks }}</string>
206
+ <string>--max-restarts-per-hour</string>
207
+ <string>{{ watchdog_policy.maxRestartsPerHour }}</string>
208
+ </array>
209
+ <key>EnvironmentVariables</key><dict>
210
+ <key>PATH</key><string>/usr/local/bin:/usr/bin:/bin:/Applications/Docker.app/Contents/Resources/bin</string>
211
+ </dict>
212
+ <key>StartInterval</key><integer>{{ watchdog_policy.intervalSeconds }}</integer>
213
+ <key>ProcessType</key><string>Background</string>
214
+ <key>Umask</key><integer>63</integer>
215
+ <key>StandardOutPath</key><string>{{ watchdog_root }}/watchdog.log</string>
216
+ <key>StandardErrorPath</key><string>{{ watchdog_root }}/watchdog.log</string>
217
+ </dict></plist>
218
+ register: watchdog_agent
219
+ - name: Inspect whether the container watchdog is loaded
220
+ ansible.builtin.command:
221
+ argv:
222
+ [/bin/launchctl, print, '{{ watchdog_domain }}/{{ watchdog_label }}']
223
+ changed_when: false
224
+ failed_when: false
225
+ register: watchdog_loaded
226
+ # launchd caches a loaded definition: a changed plist needs a reload.
227
+ - name: Unload a stale container watchdog definition
228
+ ansible.builtin.command:
229
+ argv:
230
+ [
231
+ /bin/launchctl,
232
+ bootout,
233
+ '{{ watchdog_domain }}/{{ watchdog_label }}',
234
+ ]
235
+ when: watchdog_agent.changed and watchdog_loaded.rc == 0
236
+ failed_when: false
237
+ changed_when: true
238
+ - name: Load the container watchdog
239
+ ansible.builtin.command:
240
+ argv:
241
+ [
242
+ /bin/launchctl,
243
+ bootstrap,
244
+ '{{ watchdog_domain }}',
245
+ '{{ watchdog_plist }}',
246
+ ]
247
+ when: watchdog_agent.changed or watchdog_loaded.rc != 0
248
+ changed_when: true
@@ -8,6 +8,11 @@ if [ "$(uname)" = Darwin ]; then
8
8
  else
9
9
  echo "ncpu=$(nproc)"
10
10
  echo "load=$(cut -d' ' -f1 /proc/loadavg)"
11
+ # Free inodes as a percentage: bytes can be free while every write fails.
12
+ df -Pi "$HOME" | awk 'NR==2 && $2>0{print "inodefree=" int($4*100/$2)}'
13
+ # Clock synchronisation, as systemd sees it: 1 or 0.
14
+ timedatectl show -p NTPSynchronized -p NTP 2>/dev/null | awk -F= '{v=($2=="yes")?1:0} $1=="NTP"{print "ntp=" v} $1=="NTPSynchronized"{print "ntpsync=" v}'
11
15
  awk '/MemTotal/{t=$2} /MemAvailable/{a=$2} /SwapTotal/{st=$2} /SwapFree/{sf=$2} END{print "memfree=" int(a*100/t); if (st>0) print "swap=" int((st-sf)*100/st); else print "swap=0"}' /proc/meminfo
12
16
  fi
13
17
  echo "diskmb=$(df -Pk "$HOME" | awk 'NR==2{print int($4/1024)}')"
18
+ echo "epoch=$(date +%s)"
@@ -0,0 +1,111 @@
1
+ {
2
+ "version": 1,
3
+ "rows": [
4
+ {
5
+ "role": "manager",
6
+ "platform": "any other",
7
+ "versions": "any",
8
+ "architectures": "any",
9
+ "status": "refused",
10
+ "note": "The fleet configuration accepts only a macOS manager."
11
+ },
12
+ {
13
+ "role": "worker",
14
+ "platform": "Ubuntu",
15
+ "versions": "any other release",
16
+ "architectures": "any",
17
+ "status": "refused",
18
+ "note": "UBUNTU_BASELINE_MISMATCH"
19
+ },
20
+ {
21
+ "role": "worker",
22
+ "platform": "Ubuntu",
23
+ "versions": "an accepted release",
24
+ "architectures": "any other",
25
+ "status": "refused",
26
+ "note": "UBUNTU_BASELINE_MISMATCH"
27
+ },
28
+ {
29
+ "role": "worker",
30
+ "platform": "Debian",
31
+ "versions": "any",
32
+ "architectures": "any",
33
+ "status": "refused",
34
+ "note": "UBUNTU_BASELINE_MISMATCH"
35
+ },
36
+ {
37
+ "role": "worker",
38
+ "platform": "macOS",
39
+ "versions": "any",
40
+ "architectures": "any",
41
+ "status": "refused",
42
+ "note": "The fleet configuration accepts only Linux workers."
43
+ },
44
+ {
45
+ "role": "worker",
46
+ "platform": "Windows",
47
+ "versions": "any",
48
+ "architectures": "any",
49
+ "status": "refused",
50
+ "note": "The fleet configuration accepts only Linux workers."
51
+ },
52
+ {
53
+ "role": "worker",
54
+ "platform": "WSL2",
55
+ "environment": "wsl",
56
+ "versions": "any",
57
+ "architectures": "any",
58
+ "status": "refused",
59
+ "note": "WORKER_ENVIRONMENT_UNSUPPORTED"
60
+ },
61
+ {
62
+ "role": "worker",
63
+ "platform": "container",
64
+ "environment": "container",
65
+ "versions": "any",
66
+ "architectures": "any",
67
+ "status": "refused",
68
+ "note": "WORKER_ENVIRONMENT_UNSUPPORTED"
69
+ },
70
+ {
71
+ "role": "toolchain only",
72
+ "platform": "macOS",
73
+ "versions": "any",
74
+ "architectures": "arm64, x86_64",
75
+ "status": "best effort",
76
+ "note": "No fleet role: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
77
+ },
78
+ {
79
+ "role": "toolchain only",
80
+ "platform": "Linux",
81
+ "versions": "any",
82
+ "architectures": "x86_64, aarch64",
83
+ "status": "best effort",
84
+ "note": "No fleet role: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
85
+ },
86
+ {
87
+ "role": "toolchain only",
88
+ "platform": "WSL2",
89
+ "versions": "any",
90
+ "architectures": "x86_64",
91
+ "status": "best effort",
92
+ "note": "Treated as Linux: devfleet bootstrap installs the pinned tools (docs/machine-setup.md)."
93
+ },
94
+ {
95
+ "role": "toolchain only",
96
+ "platform": "Windows",
97
+ "versions": "any",
98
+ "architectures": "x64",
99
+ "status": "best effort",
100
+ "note": "No fleet role: bootstrap/bootstrap.ps1 installs the pinned tools (docs/machine-setup.md)."
101
+ },
102
+ {
103
+ "role": "toolchain only",
104
+ "platform": "Alpine",
105
+ "versions": "any",
106
+ "architectures": "any",
107
+ "status": "refused",
108
+ "note": "musl: the pinned mise build needs glibc."
109
+ }
110
+ ]
111
+ }
@@ -3,14 +3,11 @@
3
3
  # SSH terminal so sudo reads the operator's password itself; never capture it.
4
4
  set -eu
5
5
  umask 077
6
- case "$(uname -s)" in
7
- Linux)
8
- . /etc/os-release
9
- [ "$ID" = ubuntu ] && [ "$VERSION_ID" = 26.04 ] || exit 1 ;;
10
- Darwin)
11
- [ "$(sw_vers -productVersion | cut -d. -f1)" -ge 14 ] || exit 1 ;;
12
- *) exit 1 ;;
13
- esac
6
+ # Workers are Ubuntu. Which releases is checked by the bootstrap that
7
+ # follows, against ansible/ubuntu/policy.json; this only grants sudo.
8
+ [ "$(uname -s)" = Linux ] || exit 1
9
+ . /etc/os-release
10
+ [ "$ID" = ubuntu ] || exit 1
14
11
  user=$(id -un)
15
12
  case "$user" in root|''|*[!a-z0-9_-]*) exit 1;; esac
16
13
  command -v sudo >/dev/null 2>&1 || exit 1
@@ -23,11 +20,7 @@ mkdir -p "$root"
23
20
  chmod 700 "$root"
24
21
  [ ! -L "$root/remote-operation.lock" ] || exit 1
25
22
  exec 9>"$root/remote-operation.lock"
26
- if command -v flock >/dev/null 2>&1; then
27
- flock -n 9 || exit 1
28
- else
29
- python3 -c 'import fcntl; fcntl.flock(9, fcntl.LOCK_EX | fcntl.LOCK_NB)' 9>&9 || exit 1
30
- fi
23
+ flock -n 9 || exit 1
31
24
  sudo -v
32
25
  file=$(mktemp)
33
26
  trap 'rm -f "$file"' EXIT
@@ -45,7 +38,5 @@ if sudo -n test -e "$target"; then
45
38
  }
46
39
  sudo -n cp -p -- "$target" "$target.before-fleet-enrollment"
47
40
  fi
48
- group=root
49
- [ "$(uname -s)" != Darwin ] || group=wheel
50
- sudo -n install -o root -g "$group" -m 0440 -- "$file" "$target"
41
+ sudo -n install -o root -g root -m 0440 -- "$file" "$target"
51
42
  sudo -n true