@crvouga/dev-fleet 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -17
- package/ansible/github/git.yml +66 -0
- package/ansible/macos/policy.json +12 -1
- package/ansible/monitoring/compose.yml +10 -0
- package/ansible/monitoring/container-watchdog.py +118 -0
- package/ansible/monitoring/hub-darwin.yml +146 -0
- package/ansible/net/remote-probe.sh +5 -0
- package/ansible/platforms.json +111 -0
- package/ansible/ubuntu/authorize-sudo.sh +7 -16
- package/ansible/ubuntu/baseline.yml +6 -28
- package/ansible/ubuntu/disk-gc.py +211 -7
- package/ansible/ubuntu/disk-guard.py +24 -5
- package/ansible/ubuntu/policy.json +68 -4
- package/ansible/ubuntu/t3-desktop.yml +2 -2
- package/ansible/ubuntu/tailscale.yml +20 -9
- package/ansible/ubuntu/tasks/browser.yml +56 -0
- package/ansible/ubuntu/tasks/docker.yml +52 -0
- package/ansible/ubuntu/tasks/dpkg.yml +46 -0
- package/ansible/ubuntu/tasks/install.yml +38 -0
- package/ansible/ubuntu/tasks/packages.yml +4 -55
- package/ansible/ubuntu/tasks/release.yml +25 -0
- package/ansible/ubuntu/tasks/resources.yml +7 -6
- package/ansible/ubuntu/tasks/services.yml +13 -0
- package/ansible/ubuntu/tasks/snapshot.yml +50 -0
- package/ansible/ubuntu/tasks/unattended-upgrades.yml +24 -0
- package/bootstrap/doctor.ts +32 -6
- package/bootstrap/fleet-remote-bootstrap.sh +49 -45
- package/bootstrap/setup.ts +23 -8
- package/bootstrap/worker-tools.ts +118 -0
- package/cliproxy/README.md +1 -1
- package/cliproxy/compose.yaml +4 -0
- package/cliproxy/scripts/checkpoint.py +1 -1
- package/docs/authentication.md +90 -0
- package/docs/consumer-setup.md +32 -30
- package/docs/declared.md +6 -5
- package/docs/machine-setup.md +22 -12
- package/docs/operations.md +87 -31
- package/docs/runbooks.md +377 -0
- package/docs/state.md +13 -7
- package/docs/support-matrix.md +36 -0
- package/docs/ubuntu-sync.md +16 -13
- package/docs/worker-contract.md +28 -1
- package/examples/fleet.json +19 -1
- package/llms.txt +2 -0
- package/mise.toml +7 -0
- package/package.json +2 -2
- package/schema/check-ids.json +526 -0
- package/schema/fleet.schema.json +185 -4
- package/schema/output/check.schema.json +23 -0
- package/schema/output/error.schema.json +14 -0
- package/schema/output/fleet-report.schema.json +79 -0
- package/schema/output/fleet-sync.schema.json +112 -0
- package/schema/output/report.schema.json +38 -0
- package/src/auth-checks.ts +3 -0
- package/src/cli/agent-guide.ts +5 -3
- package/src/cli/fleet-report.ts +126 -0
- package/src/cli/fleet-sync-manager.ts +11 -0
- package/src/cli/fleet-sync-plan.ts +200 -0
- package/src/cli/fleet-sync-remote.ts +4 -2
- package/src/cli/fleet-sync-schedule.ts +142 -35
- package/src/cli/fleet-sync.ts +90 -8
- package/src/cli/fleet-watch.ts +198 -0
- package/src/cli/fleet.ts +4 -0
- package/src/cli/register-fleet.ts +51 -0
- package/src/cli/registry.ts +13 -0
- package/src/cli/schedule-logs.ts +36 -0
- package/src/clock.ts +63 -0
- package/src/definition.ts +160 -12
- package/src/git-identity.ts +160 -0
- package/src/machine/platform.ts +48 -28
- package/src/machine/support-matrix.ts +76 -0
- package/src/machine-health.ts +51 -4
- package/src/manager-health.ts +205 -0
- package/src/manager-sync.ts +27 -9
- package/src/monitoring/hub-backup.ts +63 -17
- package/src/monitoring/manager-hub.ts +44 -2
- package/src/network-definition.ts +1 -1
- package/src/node-converge-linux.ts +16 -5
- package/src/node-converge.ts +1 -1
- package/src/notify.ts +239 -0
- package/src/platform/debian-version.ts +62 -0
- package/src/platform/ubuntu-audit.ts +11 -4
- package/src/platform/ubuntu-release.ts +41 -0
- package/src/remote-bootstrap.ts +89 -25
- package/src/remote-operation.ts +213 -0
- package/src/schedule-check.ts +40 -0
- package/src/secrets.ts +26 -8
- package/src/ssh.ts +10 -2
- package/src/state/history.ts +172 -0
- package/src/state/store.ts +11 -4
- package/src/tailscale.ts +51 -0
- package/src/toolchain-checks.ts +8 -3
- package/src/worker-capability-checks.ts +167 -0
- package/src/worker-capacity.ts +19 -7
- package/src/worker-host-checks.ts +68 -10
- package/src/worker-policy.ts +33 -0
- package/src/worker-readiness.ts +10 -1
|
@@ -7,15 +7,13 @@
|
|
|
7
7
|
vars_files:
|
|
8
8
|
- policy.json
|
|
9
9
|
pre_tasks:
|
|
10
|
-
-
|
|
10
|
+
- ansible.builtin.import_tasks: tasks/release.yml
|
|
11
|
+
- name: Require an explicit non-root user
|
|
11
12
|
ansible.builtin.assert:
|
|
12
13
|
that:
|
|
13
|
-
- ansible_facts.distribution == 'Ubuntu'
|
|
14
|
-
- ansible_facts.distribution_version == release
|
|
15
|
-
- ansible_facts.architecture in ['x86_64', 'aarch64']
|
|
16
14
|
- development_user is match('^[a-z_][a-z0-9_-]{0,31}$')
|
|
17
15
|
- development_user != 'root'
|
|
18
|
-
fail_msg: Unsupported
|
|
16
|
+
fail_msg: Unsupported user; name the non-root developer account.
|
|
19
17
|
- name: Read the designated existing account
|
|
20
18
|
ansible.builtin.getent:
|
|
21
19
|
database: passwd
|
|
@@ -25,6 +23,7 @@
|
|
|
25
23
|
that:
|
|
26
24
|
- ansible_facts.getent_passwd[development_user][1] | int == development_uid | int
|
|
27
25
|
- ansible_facts.getent_passwd[development_user][4] == development_home
|
|
26
|
+
- ansible.builtin.import_tasks: tasks/dpkg.yml
|
|
28
27
|
# Memory and disk safety must land even when the disk is nearly full.
|
|
29
28
|
- ansible.builtin.import_tasks: tasks/resources.yml
|
|
30
29
|
- name: Inspect capacity before package and service changes
|
|
@@ -55,32 +54,11 @@
|
|
|
55
54
|
fail_msg: DOCKER_OWNER_CONFLICT; migrate the existing Docker owner explicitly before applying the Ubuntu-native baseline.
|
|
56
55
|
- name: Inspect existing service ownership
|
|
57
56
|
ansible.builtin.service_facts:
|
|
58
|
-
- name: Wait for legitimate package transactions before inspecting dpkg
|
|
59
|
-
ansible.builtin.command:
|
|
60
|
-
argv:
|
|
61
|
-
- /usr/bin/python3
|
|
62
|
-
- '-c'
|
|
63
|
-
- |
|
|
64
|
-
import fcntl
|
|
65
|
-
files = [open(path, 'a') for path in ['/var/lib/dpkg/lock-frontend', '/var/lib/dpkg/lock']]
|
|
66
|
-
for handle in files:
|
|
67
|
-
fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
68
|
-
register: apt_idle
|
|
69
|
-
changed_when: false
|
|
70
|
-
retries: 24
|
|
71
|
-
delay: 5
|
|
72
|
-
until: apt_idle.rc == 0
|
|
73
|
-
- name: Inspect package database consistency
|
|
74
|
-
ansible.builtin.command: dpkg --audit
|
|
75
|
-
changed_when: false
|
|
76
|
-
register: dpkg_audit
|
|
77
|
-
- name: Require package database recovery before modifying packages
|
|
78
|
-
ansible.builtin.assert:
|
|
79
|
-
that: dpkg_audit.stdout | trim == ''
|
|
80
|
-
fail_msg: APT_STATE_BROKEN; inspect dpkg --audit and use reviewed package recovery. Never remove package lock files.
|
|
81
57
|
tasks:
|
|
82
58
|
- ansible.builtin.import_tasks: tasks/packages.yml
|
|
83
59
|
- ansible.builtin.import_tasks: tasks/docker.yml
|
|
60
|
+
- ansible.builtin.import_tasks: tasks/browser.yml
|
|
61
|
+
when: "'web' in worker_tool_profiles | default([])"
|
|
84
62
|
- ansible.builtin.import_tasks: tasks/services.yml
|
|
85
63
|
- ansible.builtin.import_tasks: tasks/power.yml
|
|
86
64
|
- ansible.builtin.import_tasks: tasks/user.yml
|
|
@@ -11,18 +11,31 @@ What it deletes:
|
|
|
11
11
|
process is running inside it;
|
|
12
12
|
* Docker images and build cache that no container uses and that are older
|
|
13
13
|
than --docker-age-hours (never containers or volumes);
|
|
14
|
-
* the apt package cache
|
|
15
|
-
|
|
14
|
+
* the apt package cache;
|
|
15
|
+
* agent worktrees under each --worktree-root that nobody has used for
|
|
16
|
+
--worktree-idle-days, only when Git says they hold no uncommitted,
|
|
17
|
+
untracked or unpushed work and no process runs in them (the branch is
|
|
18
|
+
kept);
|
|
19
|
+
* the oldest files of each --cache-cap DIR=MIB cache beyond its size;
|
|
20
|
+
* all but the newest N files of each --keep-newest DIR=N directory (the
|
|
21
|
+
downloaded T3 packages: the current one and the previous one);
|
|
22
|
+
* tool versions no mise configuration uses (--mise, run as --user).
|
|
23
|
+
When a watched filesystem has less than --pressure-mib free, or less than
|
|
24
|
+
--pressure-inode-percent of its inodes free, idle checkouts
|
|
16
25
|
qualify after --pressure-idle-hours, Docker leftovers after
|
|
17
26
|
--pressure-docker-age-hours, and aged entries of each --pressure-cache
|
|
18
27
|
directory are removed too.
|
|
19
28
|
|
|
20
|
-
It never deletes a checkout, a tracked file, uncommitted
|
|
21
|
-
a volume. Every decision is one JSON line on stdout
|
|
29
|
+
It never deletes a main checkout, a tracked file, uncommitted or unpushed
|
|
30
|
+
work, a container or a volume. Every decision is one JSON line on stdout, and
|
|
31
|
+
the last one says how many MiB were reclaimed.
|
|
22
32
|
|
|
23
33
|
Usage: disk-gc.py --home H [--path P ...] [--idle-days N] [--pressure-mib N]
|
|
24
|
-
[--pressure-idle-hours N] [--docker-age-hours N]
|
|
34
|
+
[--pressure-inode-percent N] [--pressure-idle-hours N] [--docker-age-hours N]
|
|
25
35
|
[--pressure-docker-age-hours N] [--pressure-cache DIR ...]
|
|
36
|
+
[--worktree-root DIR ...] [--worktree-idle-days N]
|
|
37
|
+
[--cache-cap DIR=MIB ...] [--keep-newest DIR=N ...]
|
|
38
|
+
[--mise PATH] [--user NAME]
|
|
26
39
|
[--max-depth N] [--proc DIR] [--home-only] [--dry-run]
|
|
27
40
|
|
|
28
41
|
--home-only skips the machine-wide Docker and apt steps.
|
|
@@ -61,6 +74,25 @@ def free_bytes(path):
|
|
|
61
74
|
return st.f_bavail * st.f_frsize
|
|
62
75
|
|
|
63
76
|
|
|
77
|
+
def free_inodes_percent(path):
|
|
78
|
+
"""Free inodes as a percentage; None on a filesystem without a fixed count."""
|
|
79
|
+
st = os.statvfs(path)
|
|
80
|
+
if st.f_files <= 0:
|
|
81
|
+
return None
|
|
82
|
+
return st.f_favail * 100 / st.f_files
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def under_pressure(paths, pressure_bytes, inode_percent):
|
|
86
|
+
"""Any watched filesystem short of free bytes or of free inodes."""
|
|
87
|
+
for path in paths:
|
|
88
|
+
if free_bytes(path) < pressure_bytes:
|
|
89
|
+
return True
|
|
90
|
+
inodes = free_inodes_percent(path)
|
|
91
|
+
if inodes is not None and inodes < inode_percent:
|
|
92
|
+
return True
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
|
|
64
96
|
def mtime(path):
|
|
65
97
|
try:
|
|
66
98
|
return os.lstat(path).st_mtime
|
|
@@ -277,6 +309,150 @@ def collect_cache(directory, age_seconds, now, dry_run):
|
|
|
277
309
|
return freed
|
|
278
310
|
|
|
279
311
|
|
|
312
|
+
def linked_worktrees(root, max_depth=3):
|
|
313
|
+
"""Linked worktrees (a .git file, never a repository) under root."""
|
|
314
|
+
found = []
|
|
315
|
+
|
|
316
|
+
def walk(directory, depth):
|
|
317
|
+
dot_git = os.path.join(directory, '.git')
|
|
318
|
+
if os.path.lexists(dot_git):
|
|
319
|
+
if not is_real_dir(dot_git) and not os.path.islink(dot_git):
|
|
320
|
+
found.append(directory)
|
|
321
|
+
return
|
|
322
|
+
if depth >= max_depth:
|
|
323
|
+
return
|
|
324
|
+
try:
|
|
325
|
+
entries = list(os.scandir(directory))
|
|
326
|
+
except OSError:
|
|
327
|
+
return
|
|
328
|
+
for entry in entries:
|
|
329
|
+
if entry.is_dir(follow_symlinks=False) and entry.name not in REGENERABLE:
|
|
330
|
+
walk(entry.path, depth + 1)
|
|
331
|
+
|
|
332
|
+
walk(root, 0)
|
|
333
|
+
return sorted(found)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def unsaved_work(worktree):
|
|
337
|
+
"""Why Git cannot vouch for a worktree, or None when everything in it is
|
|
338
|
+
committed and every commit is on a remote."""
|
|
339
|
+
status = run_git(worktree, ['status', '--porcelain=v1', '-z', '--untracked-files=all'])
|
|
340
|
+
if status is None or status.returncode != 0:
|
|
341
|
+
return 'git_status_failed'
|
|
342
|
+
if status.stdout.strip(b'\0'):
|
|
343
|
+
return 'uncommitted'
|
|
344
|
+
unpushed = run_git(worktree, ['rev-list', '-n', '1', 'HEAD', '--not', '--remotes'])
|
|
345
|
+
if unpushed is None or unpushed.returncode != 0:
|
|
346
|
+
return 'git_rev_list_failed'
|
|
347
|
+
if unpushed.stdout.strip():
|
|
348
|
+
return 'unpushed'
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def collect_worktrees(root, idle_seconds, proc, now, dry_run):
|
|
353
|
+
roots = busy_roots(proc)
|
|
354
|
+
freed = 0
|
|
355
|
+
for worktree in linked_worktrees(root):
|
|
356
|
+
gdir = git_dir(worktree)
|
|
357
|
+
if gdir is None or in_use(worktree, roots):
|
|
358
|
+
continue
|
|
359
|
+
if now - git_activity(worktree, gdir) < idle_seconds:
|
|
360
|
+
continue
|
|
361
|
+
latest = last_activity(worktree, gdir, [])
|
|
362
|
+
if latest is None or now - latest < idle_seconds:
|
|
363
|
+
continue
|
|
364
|
+
reason = unsaved_work(worktree)
|
|
365
|
+
if reason:
|
|
366
|
+
log(event='worktree_kept', worktree=worktree, reason=reason)
|
|
367
|
+
continue
|
|
368
|
+
size = size_of(worktree)
|
|
369
|
+
if not dry_run:
|
|
370
|
+
common = run_git(worktree, ['rev-parse', '--path-format=absolute', '--git-common-dir'])
|
|
371
|
+
if common is None or common.returncode != 0:
|
|
372
|
+
continue
|
|
373
|
+
out = subprocess.run(
|
|
374
|
+
GIT + ['--git-dir', common.stdout.decode().strip(), 'worktree', 'remove', '--force', worktree],
|
|
375
|
+
capture_output=True, timeout=600)
|
|
376
|
+
if out.returncode != 0 or os.path.lexists(worktree):
|
|
377
|
+
log(event='worktree_kept', worktree=worktree, reason='git_worktree_remove_failed')
|
|
378
|
+
continue
|
|
379
|
+
freed += size
|
|
380
|
+
log(event='worktree_removed', worktree=worktree, mib=size // MIB, dry_run=dry_run)
|
|
381
|
+
return freed
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def files_under(directory):
|
|
385
|
+
"""Every regular file under directory, never through a link."""
|
|
386
|
+
found = []
|
|
387
|
+
for current, dirs, files in os.walk(directory):
|
|
388
|
+
dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(current, d))]
|
|
389
|
+
for name in files:
|
|
390
|
+
path = os.path.join(current, name)
|
|
391
|
+
try:
|
|
392
|
+
st = os.lstat(path)
|
|
393
|
+
except OSError:
|
|
394
|
+
continue
|
|
395
|
+
if stat.S_ISREG(st.st_mode):
|
|
396
|
+
found.append((st.st_mtime, st.st_blocks * 512, path))
|
|
397
|
+
return found
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def unlink(path, dry_run):
|
|
401
|
+
if dry_run:
|
|
402
|
+
return True
|
|
403
|
+
try:
|
|
404
|
+
os.unlink(path)
|
|
405
|
+
return True
|
|
406
|
+
except OSError:
|
|
407
|
+
return False
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def cap_cache(directory, cap_bytes, dry_run):
|
|
411
|
+
"""Removes the oldest files of a cache until it fits its cap."""
|
|
412
|
+
entries = sorted(files_under(directory))
|
|
413
|
+
total = sum(size for _, size, _ in entries)
|
|
414
|
+
freed = 0
|
|
415
|
+
for _, size, path in entries:
|
|
416
|
+
if total - freed <= cap_bytes:
|
|
417
|
+
break
|
|
418
|
+
if unlink(path, dry_run):
|
|
419
|
+
freed += size
|
|
420
|
+
if not dry_run:
|
|
421
|
+
for current, dirs, files in os.walk(directory, topdown=False):
|
|
422
|
+
if current != directory and not dirs and not files:
|
|
423
|
+
try:
|
|
424
|
+
os.rmdir(current)
|
|
425
|
+
except OSError:
|
|
426
|
+
pass
|
|
427
|
+
if freed:
|
|
428
|
+
log(event='cache_capped', cache=directory, mib=freed // MIB,
|
|
429
|
+
cap_mib=cap_bytes // MIB, dry_run=dry_run)
|
|
430
|
+
return freed
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def keep_newest(directory, count, dry_run):
|
|
434
|
+
"""Removes all but the newest count files directly in directory."""
|
|
435
|
+
try:
|
|
436
|
+
entries = [e for e in os.scandir(directory) if e.is_file(follow_symlinks=False)]
|
|
437
|
+
except OSError:
|
|
438
|
+
return 0
|
|
439
|
+
entries.sort(key=lambda e: e.stat(follow_symlinks=False).st_mtime, reverse=True)
|
|
440
|
+
freed = 0
|
|
441
|
+
for entry in entries[count:]:
|
|
442
|
+
size = entry.stat(follow_symlinks=False).st_blocks * 512
|
|
443
|
+
if unlink(entry.path, dry_run):
|
|
444
|
+
freed += size
|
|
445
|
+
log(event='superseded_removed', path=entry.path, mib=size // MIB, dry_run=dry_run)
|
|
446
|
+
return freed
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def pair(text, kind):
|
|
450
|
+
name, _, value = text.rpartition('=')
|
|
451
|
+
if not name:
|
|
452
|
+
raise argparse.ArgumentTypeError(f'expected DIR={kind}')
|
|
453
|
+
return name, int(value)
|
|
454
|
+
|
|
455
|
+
|
|
280
456
|
def command(argv, dry_run, name):
|
|
281
457
|
if shutil.which(argv[0]) is None:
|
|
282
458
|
return
|
|
@@ -296,10 +472,19 @@ def main(argv):
|
|
|
296
472
|
parser.add_argument('--path', action='append', default=[])
|
|
297
473
|
parser.add_argument('--idle-days', type=float, default=7)
|
|
298
474
|
parser.add_argument('--pressure-mib', type=int, default=15360)
|
|
475
|
+
parser.add_argument('--pressure-inode-percent', type=float, default=0)
|
|
299
476
|
parser.add_argument('--pressure-idle-hours', type=float, default=12)
|
|
300
477
|
parser.add_argument('--docker-age-hours', type=int, default=168)
|
|
301
478
|
parser.add_argument('--pressure-docker-age-hours', type=int, default=24)
|
|
302
479
|
parser.add_argument('--pressure-cache', action='append', default=[])
|
|
480
|
+
parser.add_argument('--worktree-root', action='append', default=[])
|
|
481
|
+
parser.add_argument('--worktree-idle-days', type=float, default=30)
|
|
482
|
+
parser.add_argument('--cache-cap', action='append', default=[],
|
|
483
|
+
type=lambda text: pair(text, 'MIB'))
|
|
484
|
+
parser.add_argument('--keep-newest', action='append', default=[],
|
|
485
|
+
type=lambda text: pair(text, 'N'))
|
|
486
|
+
parser.add_argument('--mise')
|
|
487
|
+
parser.add_argument('--user')
|
|
303
488
|
parser.add_argument('--max-depth', type=int, default=4)
|
|
304
489
|
parser.add_argument('--proc', default='/proc')
|
|
305
490
|
parser.add_argument('--home-only', action='store_true')
|
|
@@ -309,7 +494,8 @@ def main(argv):
|
|
|
309
494
|
paths = args.path or ['/', home]
|
|
310
495
|
now = time.time()
|
|
311
496
|
before = {p: free_bytes(p) for p in paths}
|
|
312
|
-
pressure =
|
|
497
|
+
pressure = under_pressure(paths, args.pressure_mib * MIB,
|
|
498
|
+
args.pressure_inode_percent)
|
|
313
499
|
idle = args.pressure_idle_hours * HOUR if pressure else args.idle_days * 24 * HOUR
|
|
314
500
|
docker_age = args.pressure_docker_age_hours if pressure else args.docker_age_hours
|
|
315
501
|
log(event='started', pressure=pressure, idle_hours=idle / HOUR,
|
|
@@ -320,6 +506,23 @@ def main(argv):
|
|
|
320
506
|
if pressure:
|
|
321
507
|
for cache in args.pressure_cache:
|
|
322
508
|
freed += collect_cache(os.path.join(home, cache), idle, now, args.dry_run)
|
|
509
|
+
reclaimed = freed
|
|
510
|
+
for root in args.worktree_root:
|
|
511
|
+
reclaimed += collect_worktrees(os.path.join(home, root),
|
|
512
|
+
args.worktree_idle_days * 24 * HOUR,
|
|
513
|
+
args.proc, now, args.dry_run)
|
|
514
|
+
for cache, cap_mib in args.cache_cap:
|
|
515
|
+
reclaimed += cap_cache(os.path.join(home, cache), cap_mib * MIB, args.dry_run)
|
|
516
|
+
for directory, count in args.keep_newest:
|
|
517
|
+
reclaimed += keep_newest(os.path.join(home, directory), count, args.dry_run)
|
|
518
|
+
if args.mise:
|
|
519
|
+
installs = os.path.join(home, '.local', 'share', 'mise', 'installs')
|
|
520
|
+
before_mise = size_of(installs)
|
|
521
|
+
prune = [args.mise, 'prune', '--yes']
|
|
522
|
+
if args.user:
|
|
523
|
+
prune = ['runuser', '-u', args.user, '--', 'env', f'HOME={home}'] + prune
|
|
524
|
+
command(prune, args.dry_run, 'mise_prune')
|
|
525
|
+
reclaimed += max(0, before_mise - size_of(installs))
|
|
323
526
|
if not args.home_only:
|
|
324
527
|
command(['docker', 'image', 'prune', '--all', '--force', '--filter',
|
|
325
528
|
f'until={docker_age}h'], args.dry_run, 'docker_images')
|
|
@@ -328,7 +531,8 @@ def main(argv):
|
|
|
328
531
|
command(['apt-get', 'clean'], args.dry_run, 'apt_cache')
|
|
329
532
|
|
|
330
533
|
after = {p: free_bytes(p) for p in paths}
|
|
331
|
-
log(event='finished', checkout_mib=freed // MIB,
|
|
534
|
+
log(event='finished', checkout_mib=freed // MIB,
|
|
535
|
+
reclaimed_mib=reclaimed // MIB, dry_run=args.dry_run,
|
|
332
536
|
free_mib={p: v // MIB for p, v in after.items()})
|
|
333
537
|
|
|
334
538
|
|
|
@@ -3,14 +3,15 @@
|
|
|
3
3
|
|
|
4
4
|
Managed by personal fleet-sync (tasks/resources.yml) as
|
|
5
5
|
personal-fleet-disk-guard.service. Every interval it measures free space on the
|
|
6
|
-
watched filesystems and the bytes each leaf cgroup under user.slice wrote. When
|
|
7
|
-
any watched filesystem is below the floor
|
|
6
|
+
watched filesystems (bytes and inodes) and the bytes each leaf cgroup under user.slice wrote. When
|
|
7
|
+
any watched filesystem is below the floor of free bytes or of free inodes, it
|
|
8
|
+
kills only the single cgroup that
|
|
8
9
|
wrote the most during the last interval, and only if that is at least the write
|
|
9
10
|
threshold. It never deletes files, never touches system services and never
|
|
10
11
|
kills idle work. Free space must be recovered by a person.
|
|
11
12
|
|
|
12
|
-
Usage: disk-guard.py --path / --path /home/me [--floor-mib N]
|
|
13
|
-
[--interval S] [--cgroup-root DIR] [--dry-run]
|
|
13
|
+
Usage: disk-guard.py --path / --path /home/me [--floor-mib N]
|
|
14
|
+
[--inode-floor-percent N] [--write-mib N] [--interval S] [--cgroup-root DIR] [--dry-run]
|
|
14
15
|
"""
|
|
15
16
|
import argparse
|
|
16
17
|
import json
|
|
@@ -61,6 +62,21 @@ def free_bytes(path):
|
|
|
61
62
|
return st.f_bavail * st.f_frsize
|
|
62
63
|
|
|
63
64
|
|
|
65
|
+
def free_inodes_percent(path):
|
|
66
|
+
"""Free inodes as a percentage; None on a filesystem without a fixed count."""
|
|
67
|
+
st = os.statvfs(path)
|
|
68
|
+
if st.f_files <= 0:
|
|
69
|
+
return None
|
|
70
|
+
return st.f_favail * 100 / st.f_files
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def low_paths(free, inodes, floor, inode_floor):
|
|
74
|
+
"""The watched paths below either floor: bytes, or free inodes in percent."""
|
|
75
|
+
return [path for path in free
|
|
76
|
+
if free[path] < floor
|
|
77
|
+
or (inodes.get(path) is not None and inodes[path] < inode_floor)]
|
|
78
|
+
|
|
79
|
+
|
|
64
80
|
def log(**event):
|
|
65
81
|
print(json.dumps(event, sort_keys=True), flush=True)
|
|
66
82
|
|
|
@@ -69,6 +85,7 @@ def main(argv):
|
|
|
69
85
|
parser = argparse.ArgumentParser()
|
|
70
86
|
parser.add_argument('--path', action='append', required=True)
|
|
71
87
|
parser.add_argument('--floor-mib', type=int, default=3072)
|
|
88
|
+
parser.add_argument('--inode-floor-percent', type=float, default=2)
|
|
72
89
|
parser.add_argument('--write-mib', type=int, default=32)
|
|
73
90
|
parser.add_argument('--interval', type=float, default=5.0)
|
|
74
91
|
parser.add_argument('--cgroup-root', default='/sys/fs/cgroup/user.slice')
|
|
@@ -76,6 +93,7 @@ def main(argv):
|
|
|
76
93
|
args = parser.parse_args(argv)
|
|
77
94
|
floor, threshold = args.floor_mib * MIB, args.write_mib * MIB
|
|
78
95
|
log(event='started', paths=args.path, floor_mib=args.floor_mib,
|
|
96
|
+
inode_floor_percent=args.inode_floor_percent,
|
|
79
97
|
write_mib=args.write_mib, interval=args.interval, dry_run=args.dry_run)
|
|
80
98
|
previous = leaf_writes(args.cgroup_root)
|
|
81
99
|
while True:
|
|
@@ -83,7 +101,8 @@ def main(argv):
|
|
|
83
101
|
try:
|
|
84
102
|
current = leaf_writes(args.cgroup_root)
|
|
85
103
|
free = {path: free_bytes(path) for path in args.path}
|
|
86
|
-
|
|
104
|
+
inodes = {path: free_inodes_percent(path) for path in args.path}
|
|
105
|
+
low = low_paths(free, inodes, floor, args.inode_floor_percent)
|
|
87
106
|
victim = choose_victim(previous, current, bool(low), threshold)
|
|
88
107
|
previous = current
|
|
89
108
|
if victim is None:
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 1,
|
|
3
|
-
"
|
|
4
|
-
|
|
3
|
+
"releases": [
|
|
4
|
+
{
|
|
5
|
+
"release": "26.04",
|
|
6
|
+
"codename": "resolute",
|
|
7
|
+
"architectures": ["amd64", "arm64"]
|
|
8
|
+
}
|
|
9
|
+
],
|
|
5
10
|
"snapshot": "20261101T000000Z",
|
|
6
11
|
"packages": {
|
|
7
12
|
"sudo": "1.9.17p2-1ubuntu3.2",
|
|
@@ -17,7 +22,8 @@
|
|
|
17
22
|
"python3": "3.14.3-0ubuntu2",
|
|
18
23
|
"unzip": "6.0-29ubuntu1",
|
|
19
24
|
"xz-utils": "5.8.3-1ubuntu0.1",
|
|
20
|
-
"ansible-core": "2.20.1-1"
|
|
25
|
+
"ansible-core": "2.20.1-1",
|
|
26
|
+
"systemd-oomd": "259.9-0ubuntu0.1"
|
|
21
27
|
},
|
|
22
28
|
"tailscale": {
|
|
23
29
|
"version": "1.104.5",
|
|
@@ -47,7 +53,65 @@
|
|
|
47
53
|
"diskGcDockerAgeHours": 168,
|
|
48
54
|
"diskGcPressureDockerAgeHours": 24,
|
|
49
55
|
"diskGcPressureCaches": [".bun/install/cache"],
|
|
56
|
+
"diskGcWorktreeRoots": [".t3/worktrees"],
|
|
57
|
+
"diskGcWorktreeIdleDays": 30,
|
|
58
|
+
"diskGcCacheCapsMiB": {
|
|
59
|
+
".bun/install/cache": 4096,
|
|
60
|
+
".npm/_cacache": 2048,
|
|
61
|
+
".cache/uv": 4096,
|
|
62
|
+
".cache/pip": 1024
|
|
63
|
+
},
|
|
64
|
+
"diskGcPackagesKept": 2,
|
|
50
65
|
"dockerLogMaxSize": "50m",
|
|
51
|
-
"dockerLogMaxFiles": "3"
|
|
66
|
+
"dockerLogMaxFiles": "3",
|
|
67
|
+
"inodeWarnFreePercent": 10,
|
|
68
|
+
"inodeFailFreePercent": 2
|
|
69
|
+
},
|
|
70
|
+
"clock": {
|
|
71
|
+
"warnOffsetSeconds": 2,
|
|
72
|
+
"failOffsetSeconds": 30
|
|
73
|
+
},
|
|
74
|
+
"rebootPendingFailDays": 14,
|
|
75
|
+
"dockerSmokeTest": {
|
|
76
|
+
"image": "busybox:1.37.0@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec",
|
|
77
|
+
"timeoutSeconds": 60
|
|
78
|
+
},
|
|
79
|
+
"browser": {
|
|
80
|
+
"packages": {
|
|
81
|
+
"fonts-liberation": "1:2.1.5-3build1",
|
|
82
|
+
"libasound2t64": "1.2.15.3-1ubuntu1.1",
|
|
83
|
+
"libatk-bridge2.0-0t64": "2.60.4-0ubuntu0.1",
|
|
84
|
+
"libatk1.0-0t64": "2.60.4-0ubuntu0.1",
|
|
85
|
+
"libatspi2.0-0t64": "2.60.4-0ubuntu0.1",
|
|
86
|
+
"libcairo2": "1.18.4-3",
|
|
87
|
+
"libcups2t64": "2.4.16-1ubuntu1.3",
|
|
88
|
+
"libdbus-1-3": "1.16.2-2ubuntu4",
|
|
89
|
+
"libdrm2": "2.4.131-1",
|
|
90
|
+
"libgbm1": "26.0.8-1ubuntu0.3",
|
|
91
|
+
"libglib2.0-0t64": "2.88.0-1ubuntu0.1",
|
|
92
|
+
"libnspr4": "2:4.38.2-1ubuntu1",
|
|
93
|
+
"libnss3": "2:3.120-1ubuntu2.2",
|
|
94
|
+
"libpango-1.0-0": "1.57.0-1",
|
|
95
|
+
"libx11-6": "2:1.8.13-1ubuntu0.1",
|
|
96
|
+
"libxcb1": "1.17.0-2ubuntu1",
|
|
97
|
+
"libxcomposite1": "1:0.4.6-1build1",
|
|
98
|
+
"libxdamage1": "1:1.1.7-1",
|
|
99
|
+
"libxext6": "2:1.3.4-1build3",
|
|
100
|
+
"libxfixes3": "1:6.0.0-2build2",
|
|
101
|
+
"libxkbcommon0": "1.13.1-1",
|
|
102
|
+
"libxrandr2": "2:1.5.4-1build1"
|
|
103
|
+
},
|
|
104
|
+
"headlessShell": {
|
|
105
|
+
"version": "158.0.8127.18",
|
|
106
|
+
"amd64": {
|
|
107
|
+
"archive": "linux64/chrome-headless-shell-linux64",
|
|
108
|
+
"sha256": "18352d22cb0cdf520cbff2ad8c41391f0e02a72886cb53b0fde4bdfc4bd075d6"
|
|
109
|
+
},
|
|
110
|
+
"arm64": {
|
|
111
|
+
"archive": "linux-arm64/chrome-headless-shell-linux-arm64",
|
|
112
|
+
"sha256": "b136462898353d94c7fab50f9cdaa6d8f0d4c190153916936aa0a03353a26bf2"
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
"launchTimeoutSeconds": 60
|
|
52
116
|
}
|
|
53
117
|
}
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
# unit starts the app with the graphical session, so the UI is on screen and the
|
|
5
5
|
# server is up after every boot. Starting or migrating is left to fleet-sync.
|
|
6
6
|
#
|
|
7
|
-
# The upstream .deb declares libxss1, which Ubuntu
|
|
8
|
-
# binary does not link. Instead of patching the package or leaving apt with an
|
|
7
|
+
# The upstream .deb declares libxss1, which current Ubuntu releases no longer
|
|
8
|
+
# ship and the binary does not link. Instead of patching the package or leaving apt with an
|
|
9
9
|
# unmet dependency, the verified package payload is unpacked to its own path and
|
|
10
10
|
# its real runtime libraries and bundled AppArmor profile are installed.
|
|
11
11
|
- name: Install the pinned T3 Code desktop app and start it with the session
|
|
@@ -8,17 +8,28 @@
|
|
|
8
8
|
vars_files:
|
|
9
9
|
- policy.json
|
|
10
10
|
tasks:
|
|
11
|
-
-
|
|
12
|
-
ansible.builtin.assert:
|
|
13
|
-
that:
|
|
14
|
-
- ansible_facts.distribution == 'Ubuntu'
|
|
15
|
-
- ansible_facts.distribution_version == release
|
|
11
|
+
- ansible.builtin.import_tasks: tasks/release.yml
|
|
16
12
|
- name: Inspect installed packages before possible network interruption
|
|
17
13
|
ansible.builtin.package_facts:
|
|
18
14
|
manager: apt
|
|
19
|
-
|
|
15
|
+
# The pin is a minimum: a newer Tailscale is kept and reported as a
|
|
16
|
+
# warning by the worker checks; an older one needs a maintenance window.
|
|
17
|
+
- name: Compare the installed Tailscale with its pin
|
|
18
|
+
ansible.builtin.command:
|
|
19
|
+
argv:
|
|
20
|
+
- dpkg
|
|
21
|
+
- --compare-versions
|
|
22
|
+
- '{{ ansible_facts.packages.tailscale[0].version }}'
|
|
23
|
+
- ge
|
|
24
|
+
- '{{ tailscale.version }}'
|
|
25
|
+
register: tailscale_pin
|
|
26
|
+
changed_when: false
|
|
27
|
+
failed_when: tailscale_pin.rc not in [0, 1]
|
|
28
|
+
check_mode: false
|
|
29
|
+
when: "'tailscale' in ansible_facts.packages"
|
|
30
|
+
- name: Require maintenance for an upgrade to the pin
|
|
20
31
|
ansible.builtin.assert:
|
|
21
|
-
that: "'tailscale' not in ansible_facts.packages or
|
|
32
|
+
that: "'tailscale' not in ansible_facts.packages or tailscale_pin.rc == 0"
|
|
22
33
|
fail_msg: TAILSCALE_UPGRADE_PENDING; upgrading an active VPN can disconnect SSH. Review an explicit maintenance operation.
|
|
23
34
|
- name: Inspect existing package-source ownership before writing
|
|
24
35
|
ansible.builtin.find:
|
|
@@ -35,7 +46,7 @@
|
|
|
35
46
|
fail_msg: TAILSCALE_SOURCE_OWNER_CONFLICT; review the existing signed source and its migration before creating another repository entry.
|
|
36
47
|
- name: Install pinned upstream signing key
|
|
37
48
|
ansible.builtin.get_url:
|
|
38
|
-
url: https://pkgs.tailscale.com/stable/ubuntu/
|
|
49
|
+
url: https://pkgs.tailscale.com/stable/ubuntu/{{ ubuntu_release.codename }}.noarmor.gpg
|
|
39
50
|
dest: /usr/share/keyrings/tailscale-archive-keyring.gpg
|
|
40
51
|
checksum: 'sha256:{{ tailscale.keySha256 }}'
|
|
41
52
|
owner: root
|
|
@@ -44,7 +55,7 @@
|
|
|
44
55
|
timeout: 30
|
|
45
56
|
- name: Declare the signed upstream package source
|
|
46
57
|
ansible.builtin.apt_repository:
|
|
47
|
-
repo: deb [signed-by=/usr/share/keyrings/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/ubuntu
|
|
58
|
+
repo: deb [signed-by=/usr/share/keyrings/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/ubuntu {{ ubuntu_release.codename }} main
|
|
48
59
|
filename: tailscale
|
|
49
60
|
state: present
|
|
50
61
|
update_cache: false
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
---
|
|
2
|
+
# Headless Chromium for browser tests (Playwright, Storybook, end-to-end
|
|
3
|
+
# suites): the system libraries it loads, at the snapshot's exact versions,
|
|
4
|
+
# and one pinned Chrome Headless Shell build. Applied to workers that select
|
|
5
|
+
# the web tool profile; the launch below backs worker:browser.
|
|
6
|
+
- name: Install the exact libraries headless Chromium loads
|
|
7
|
+
ansible.builtin.apt:
|
|
8
|
+
name: '{{ item.key }}={{ item.value }}'
|
|
9
|
+
state: present
|
|
10
|
+
lock_timeout: '{{ aptLockTimeoutSeconds }}'
|
|
11
|
+
allow_downgrade: false
|
|
12
|
+
allow_unauthenticated: false
|
|
13
|
+
fail_on_autoremove: true
|
|
14
|
+
policy_rc_d: 101
|
|
15
|
+
environment:
|
|
16
|
+
APT_CONFIG: /etc/personal-fleet/apt.conf
|
|
17
|
+
loop: '{{ browser.packages | dict2items }}'
|
|
18
|
+
- name: Name the headless shell build of this architecture
|
|
19
|
+
ansible.builtin.set_fact:
|
|
20
|
+
browser_build: "{{ browser.headlessShell['arm64' if ansible_facts.architecture == 'aarch64' else 'amd64'] }}"
|
|
21
|
+
browser_root: '/opt/personal-fleet/browser/{{ browser.headlessShell.version }}'
|
|
22
|
+
- name: Create the package cache and the browser directory
|
|
23
|
+
ansible.builtin.file:
|
|
24
|
+
path: '{{ item }}'
|
|
25
|
+
state: directory
|
|
26
|
+
owner: root
|
|
27
|
+
group: root
|
|
28
|
+
mode: '0755'
|
|
29
|
+
loop:
|
|
30
|
+
- /var/cache/personal-fleet
|
|
31
|
+
- '{{ browser_root }}'
|
|
32
|
+
- name: Download the pinned headless shell
|
|
33
|
+
ansible.builtin.get_url:
|
|
34
|
+
url: 'https://cdn.playwright.dev/builds/cft/{{ browser.headlessShell.version }}/{{ browser_build.archive }}.zip'
|
|
35
|
+
dest: '/var/cache/personal-fleet/chrome-headless-shell-{{ browser.headlessShell.version }}-{{ browser_build.archive | basename }}.zip'
|
|
36
|
+
checksum: 'sha256:{{ browser_build.sha256 }}'
|
|
37
|
+
mode: '0644'
|
|
38
|
+
timeout: 300
|
|
39
|
+
- name: Unpack the verified headless shell
|
|
40
|
+
ansible.builtin.unarchive:
|
|
41
|
+
src: '/var/cache/personal-fleet/chrome-headless-shell-{{ browser.headlessShell.version }}-{{ browser_build.archive | basename }}.zip'
|
|
42
|
+
dest: '{{ browser_root }}'
|
|
43
|
+
remote_src: true
|
|
44
|
+
creates: '{{ browser_root }}/{{ browser_build.archive | basename }}/chrome-headless-shell'
|
|
45
|
+
- name: Launch headless as the developer, load a blank page and exit (BROWSER_LAUNCH_FAILED)
|
|
46
|
+
ansible.builtin.command:
|
|
47
|
+
argv:
|
|
48
|
+
- '{{ browser_root }}/{{ browser_build.archive | basename }}/chrome-headless-shell'
|
|
49
|
+
- --headless
|
|
50
|
+
- --no-sandbox
|
|
51
|
+
- --disable-gpu
|
|
52
|
+
- --dump-dom
|
|
53
|
+
- about:blank
|
|
54
|
+
become_user: '{{ development_user }}'
|
|
55
|
+
timeout: '{{ browser.launchTimeoutSeconds }}'
|
|
56
|
+
changed_when: false
|
|
@@ -29,3 +29,55 @@
|
|
|
29
29
|
- [docker, compose, version]
|
|
30
30
|
- [docker, buildx, version]
|
|
31
31
|
changed_when: false
|
|
32
|
+
# The engine answering is not proof that an agent can use it: run a pinned
|
|
33
|
+
# image and build a one-line image as the developer, in a fresh session that
|
|
34
|
+
# reads the docker group from scratch. The image is cached here so the same
|
|
35
|
+
# test works offline in worker:docker-run.
|
|
36
|
+
- name: Look for the cached smoke-test image
|
|
37
|
+
ansible.builtin.command:
|
|
38
|
+
argv:
|
|
39
|
+
[
|
|
40
|
+
docker,
|
|
41
|
+
image,
|
|
42
|
+
inspect,
|
|
43
|
+
--format,
|
|
44
|
+
'{{ "{{.Id}}" }}',
|
|
45
|
+
'{{ dockerSmokeTest.image }}',
|
|
46
|
+
]
|
|
47
|
+
become_user: '{{ development_user }}'
|
|
48
|
+
check_mode: false
|
|
49
|
+
changed_when: false
|
|
50
|
+
failed_when: false
|
|
51
|
+
register: docker_smoke_image
|
|
52
|
+
- name: Cache the pinned smoke-test image
|
|
53
|
+
ansible.builtin.command:
|
|
54
|
+
argv: [docker, pull, --quiet, '{{ dockerSmokeTest.image }}']
|
|
55
|
+
become_user: '{{ development_user }}'
|
|
56
|
+
when: docker_smoke_image.rc | default(0) != 0
|
|
57
|
+
changed_when: true
|
|
58
|
+
- name: Run a container as the developer, offline (DOCKER_RUN_FAILED)
|
|
59
|
+
ansible.builtin.command:
|
|
60
|
+
argv:
|
|
61
|
+
[
|
|
62
|
+
docker,
|
|
63
|
+
run,
|
|
64
|
+
--rm,
|
|
65
|
+
--network,
|
|
66
|
+
none,
|
|
67
|
+
--pull,
|
|
68
|
+
never,
|
|
69
|
+
'{{ dockerSmokeTest.image }}',
|
|
70
|
+
'true',
|
|
71
|
+
]
|
|
72
|
+
become_user: '{{ development_user }}'
|
|
73
|
+
timeout: '{{ dockerSmokeTest.timeoutSeconds }}'
|
|
74
|
+
changed_when: false
|
|
75
|
+
- name: Build a one-line image as the developer, offline (DOCKER_BUILD_FAILED)
|
|
76
|
+
ansible.builtin.command:
|
|
77
|
+
argv: [docker, build, --quiet, --pull=false, --network, none, '-']
|
|
78
|
+
stdin: |
|
|
79
|
+
FROM {{ dockerSmokeTest.image }}
|
|
80
|
+
RUN true
|
|
81
|
+
become_user: '{{ development_user }}'
|
|
82
|
+
timeout: '{{ dockerSmokeTest.timeoutSeconds }}'
|
|
83
|
+
changed_when: false
|