@crvouga/dev-fleet 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +25 -17
  2. package/ansible/github/git.yml +66 -0
  3. package/ansible/macos/policy.json +12 -1
  4. package/ansible/monitoring/compose.yml +10 -0
  5. package/ansible/monitoring/container-watchdog.py +118 -0
  6. package/ansible/monitoring/hub-darwin.yml +146 -0
  7. package/ansible/net/remote-probe.sh +5 -0
  8. package/ansible/platforms.json +111 -0
  9. package/ansible/ubuntu/authorize-sudo.sh +7 -16
  10. package/ansible/ubuntu/baseline.yml +6 -28
  11. package/ansible/ubuntu/disk-gc.py +211 -7
  12. package/ansible/ubuntu/disk-guard.py +24 -5
  13. package/ansible/ubuntu/policy.json +68 -4
  14. package/ansible/ubuntu/t3-desktop.yml +2 -2
  15. package/ansible/ubuntu/tailscale.yml +20 -9
  16. package/ansible/ubuntu/tasks/browser.yml +56 -0
  17. package/ansible/ubuntu/tasks/docker.yml +52 -0
  18. package/ansible/ubuntu/tasks/dpkg.yml +46 -0
  19. package/ansible/ubuntu/tasks/install.yml +38 -0
  20. package/ansible/ubuntu/tasks/packages.yml +4 -55
  21. package/ansible/ubuntu/tasks/release.yml +25 -0
  22. package/ansible/ubuntu/tasks/resources.yml +7 -6
  23. package/ansible/ubuntu/tasks/services.yml +13 -0
  24. package/ansible/ubuntu/tasks/snapshot.yml +50 -0
  25. package/ansible/ubuntu/tasks/unattended-upgrades.yml +24 -0
  26. package/bootstrap/doctor.ts +32 -6
  27. package/bootstrap/fleet-remote-bootstrap.sh +49 -45
  28. package/bootstrap/setup.ts +23 -8
  29. package/bootstrap/worker-tools.ts +118 -0
  30. package/cliproxy/README.md +1 -1
  31. package/cliproxy/compose.yaml +4 -0
  32. package/cliproxy/scripts/checkpoint.py +1 -1
  33. package/docs/authentication.md +90 -0
  34. package/docs/consumer-setup.md +32 -30
  35. package/docs/declared.md +6 -5
  36. package/docs/machine-setup.md +22 -12
  37. package/docs/operations.md +87 -31
  38. package/docs/runbooks.md +377 -0
  39. package/docs/state.md +13 -7
  40. package/docs/support-matrix.md +36 -0
  41. package/docs/ubuntu-sync.md +16 -13
  42. package/docs/worker-contract.md +28 -1
  43. package/examples/fleet.json +19 -1
  44. package/llms.txt +2 -0
  45. package/mise.toml +7 -0
  46. package/package.json +2 -2
  47. package/schema/check-ids.json +526 -0
  48. package/schema/fleet.schema.json +185 -4
  49. package/schema/output/check.schema.json +23 -0
  50. package/schema/output/error.schema.json +14 -0
  51. package/schema/output/fleet-report.schema.json +79 -0
  52. package/schema/output/fleet-sync.schema.json +112 -0
  53. package/schema/output/report.schema.json +38 -0
  54. package/src/auth-checks.ts +3 -0
  55. package/src/cli/agent-guide.ts +5 -3
  56. package/src/cli/fleet-report.ts +126 -0
  57. package/src/cli/fleet-sync-manager.ts +11 -0
  58. package/src/cli/fleet-sync-plan.ts +200 -0
  59. package/src/cli/fleet-sync-remote.ts +4 -2
  60. package/src/cli/fleet-sync-schedule.ts +142 -35
  61. package/src/cli/fleet-sync.ts +90 -8
  62. package/src/cli/fleet-watch.ts +198 -0
  63. package/src/cli/fleet.ts +4 -0
  64. package/src/cli/register-fleet.ts +51 -0
  65. package/src/cli/registry.ts +13 -0
  66. package/src/cli/schedule-logs.ts +36 -0
  67. package/src/clock.ts +63 -0
  68. package/src/definition.ts +160 -12
  69. package/src/git-identity.ts +160 -0
  70. package/src/machine/platform.ts +48 -28
  71. package/src/machine/support-matrix.ts +76 -0
  72. package/src/machine-health.ts +51 -4
  73. package/src/manager-health.ts +205 -0
  74. package/src/manager-sync.ts +27 -9
  75. package/src/monitoring/hub-backup.ts +63 -17
  76. package/src/monitoring/manager-hub.ts +44 -2
  77. package/src/network-definition.ts +1 -1
  78. package/src/node-converge-linux.ts +16 -5
  79. package/src/node-converge.ts +1 -1
  80. package/src/notify.ts +239 -0
  81. package/src/platform/debian-version.ts +62 -0
  82. package/src/platform/ubuntu-audit.ts +11 -4
  83. package/src/platform/ubuntu-release.ts +41 -0
  84. package/src/remote-bootstrap.ts +89 -25
  85. package/src/remote-operation.ts +213 -0
  86. package/src/schedule-check.ts +40 -0
  87. package/src/secrets.ts +26 -8
  88. package/src/ssh.ts +10 -2
  89. package/src/state/history.ts +172 -0
  90. package/src/state/store.ts +11 -4
  91. package/src/tailscale.ts +51 -0
  92. package/src/toolchain-checks.ts +8 -3
  93. package/src/worker-capability-checks.ts +167 -0
  94. package/src/worker-capacity.ts +19 -7
  95. package/src/worker-host-checks.ts +68 -10
  96. package/src/worker-policy.ts +33 -0
  97. package/src/worker-readiness.ts +10 -1
@@ -7,15 +7,13 @@
7
7
  vars_files:
8
8
  - policy.json
9
9
  pre_tasks:
10
- - name: Require the supported common release and an explicit non-root user
10
+ - ansible.builtin.import_tasks: tasks/release.yml
11
+ - name: Require an explicit non-root user
11
12
  ansible.builtin.assert:
12
13
  that:
13
- - ansible_facts.distribution == 'Ubuntu'
14
- - ansible_facts.distribution_version == release
15
- - ansible_facts.architecture in ['x86_64', 'aarch64']
16
14
  - development_user is match('^[a-z_][a-z0-9_-]{0,31}$')
17
15
  - development_user != 'root'
18
- fail_msg: Unsupported platform or user; no distribution upgrade is permitted.
16
+ fail_msg: Unsupported user; name the non-root developer account.
19
17
  - name: Read the designated existing account
20
18
  ansible.builtin.getent:
21
19
  database: passwd
@@ -25,6 +23,7 @@
25
23
  that:
26
24
  - ansible_facts.getent_passwd[development_user][1] | int == development_uid | int
27
25
  - ansible_facts.getent_passwd[development_user][4] == development_home
26
+ - ansible.builtin.import_tasks: tasks/dpkg.yml
28
27
  # Memory and disk safety must land even when the disk is nearly full.
29
28
  - ansible.builtin.import_tasks: tasks/resources.yml
30
29
  - name: Inspect capacity before package and service changes
@@ -55,32 +54,11 @@
55
54
  fail_msg: DOCKER_OWNER_CONFLICT; migrate the existing Docker owner explicitly before applying the Ubuntu-native baseline.
56
55
  - name: Inspect existing service ownership
57
56
  ansible.builtin.service_facts:
58
- - name: Wait for legitimate package transactions before inspecting dpkg
59
- ansible.builtin.command:
60
- argv:
61
- - /usr/bin/python3
62
- - '-c'
63
- - |
64
- import fcntl
65
- files = [open(path, 'a') for path in ['/var/lib/dpkg/lock-frontend', '/var/lib/dpkg/lock']]
66
- for handle in files:
67
- fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
68
- register: apt_idle
69
- changed_when: false
70
- retries: 24
71
- delay: 5
72
- until: apt_idle.rc == 0
73
- - name: Inspect package database consistency
74
- ansible.builtin.command: dpkg --audit
75
- changed_when: false
76
- register: dpkg_audit
77
- - name: Require package database recovery before modifying packages
78
- ansible.builtin.assert:
79
- that: dpkg_audit.stdout | trim == ''
80
- fail_msg: APT_STATE_BROKEN; inspect dpkg --audit and use reviewed package recovery. Never remove package lock files.
81
57
  tasks:
82
58
  - ansible.builtin.import_tasks: tasks/packages.yml
83
59
  - ansible.builtin.import_tasks: tasks/docker.yml
60
+ - ansible.builtin.import_tasks: tasks/browser.yml
61
+ when: "'web' in worker_tool_profiles | default([])"
84
62
  - ansible.builtin.import_tasks: tasks/services.yml
85
63
  - ansible.builtin.import_tasks: tasks/power.yml
86
64
  - ansible.builtin.import_tasks: tasks/user.yml
@@ -11,18 +11,31 @@ What it deletes:
11
11
  process is running inside it;
12
12
  * Docker images and build cache that no container uses and that are older
13
13
  than --docker-age-hours (never containers or volumes);
14
- * the apt package cache.
15
- When a watched filesystem has less than --pressure-mib free, idle checkouts
14
+ * the apt package cache;
15
+ * agent worktrees under each --worktree-root that nobody has used for
16
+ --worktree-idle-days, only when Git says they hold no uncommitted,
17
+ untracked or unpushed work and no process runs in them (the branch is
18
+ kept);
19
+ * the oldest files of each --cache-cap DIR=MIB cache beyond its size;
20
+ * all but the newest N files of each --keep-newest DIR=N directory (the
21
+ downloaded T3 packages: the current one and the previous one);
22
+ * tool versions no mise configuration uses (--mise, run as --user).
23
+ When a watched filesystem has less than --pressure-mib free, or less than
24
+ --pressure-inode-percent of its inodes free, idle checkouts
16
25
  qualify after --pressure-idle-hours, Docker leftovers after
17
26
  --pressure-docker-age-hours, and aged entries of each --pressure-cache
18
27
  directory are removed too.
19
28
 
20
- It never deletes a checkout, a tracked file, uncommitted work, a container or
21
- a volume. Every decision is one JSON line on stdout.
29
+ It never deletes a main checkout, a tracked file, uncommitted or unpushed
30
+ work, a container or a volume. Every decision is one JSON line on stdout, and
31
+ the last one says how many MiB were reclaimed.
22
32
 
23
33
  Usage: disk-gc.py --home H [--path P ...] [--idle-days N] [--pressure-mib N]
24
- [--pressure-idle-hours N] [--docker-age-hours N]
34
+ [--pressure-inode-percent N] [--pressure-idle-hours N] [--docker-age-hours N]
25
35
  [--pressure-docker-age-hours N] [--pressure-cache DIR ...]
36
+ [--worktree-root DIR ...] [--worktree-idle-days N]
37
+ [--cache-cap DIR=MIB ...] [--keep-newest DIR=N ...]
38
+ [--mise PATH] [--user NAME]
26
39
  [--max-depth N] [--proc DIR] [--home-only] [--dry-run]
27
40
 
28
41
  --home-only skips the machine-wide Docker and apt steps.
@@ -61,6 +74,25 @@ def free_bytes(path):
61
74
  return st.f_bavail * st.f_frsize
62
75
 
63
76
 
77
+ def free_inodes_percent(path):
78
+ """Free inodes as a percentage; None on a filesystem without a fixed count."""
79
+ st = os.statvfs(path)
80
+ if st.f_files <= 0:
81
+ return None
82
+ return st.f_favail * 100 / st.f_files
83
+
84
+
85
+ def under_pressure(paths, pressure_bytes, inode_percent):
86
+ """Any watched filesystem short of free bytes or of free inodes."""
87
+ for path in paths:
88
+ if free_bytes(path) < pressure_bytes:
89
+ return True
90
+ inodes = free_inodes_percent(path)
91
+ if inodes is not None and inodes < inode_percent:
92
+ return True
93
+ return False
94
+
95
+
64
96
  def mtime(path):
65
97
  try:
66
98
  return os.lstat(path).st_mtime
@@ -277,6 +309,150 @@ def collect_cache(directory, age_seconds, now, dry_run):
277
309
  return freed
278
310
 
279
311
 
312
+ def linked_worktrees(root, max_depth=3):
313
+ """Linked worktrees (a .git file, never a repository) under root."""
314
+ found = []
315
+
316
+ def walk(directory, depth):
317
+ dot_git = os.path.join(directory, '.git')
318
+ if os.path.lexists(dot_git):
319
+ if not is_real_dir(dot_git) and not os.path.islink(dot_git):
320
+ found.append(directory)
321
+ return
322
+ if depth >= max_depth:
323
+ return
324
+ try:
325
+ entries = list(os.scandir(directory))
326
+ except OSError:
327
+ return
328
+ for entry in entries:
329
+ if entry.is_dir(follow_symlinks=False) and entry.name not in REGENERABLE:
330
+ walk(entry.path, depth + 1)
331
+
332
+ walk(root, 0)
333
+ return sorted(found)
334
+
335
+
336
+ def unsaved_work(worktree):
337
+ """Why Git cannot vouch for a worktree, or None when everything in it is
338
+ committed and every commit is on a remote."""
339
+ status = run_git(worktree, ['status', '--porcelain=v1', '-z', '--untracked-files=all'])
340
+ if status is None or status.returncode != 0:
341
+ return 'git_status_failed'
342
+ if status.stdout.strip(b'\0'):
343
+ return 'uncommitted'
344
+ unpushed = run_git(worktree, ['rev-list', '-n', '1', 'HEAD', '--not', '--remotes'])
345
+ if unpushed is None or unpushed.returncode != 0:
346
+ return 'git_rev_list_failed'
347
+ if unpushed.stdout.strip():
348
+ return 'unpushed'
349
+ return None
350
+
351
+
352
+ def collect_worktrees(root, idle_seconds, proc, now, dry_run):
353
+ roots = busy_roots(proc)
354
+ freed = 0
355
+ for worktree in linked_worktrees(root):
356
+ gdir = git_dir(worktree)
357
+ if gdir is None or in_use(worktree, roots):
358
+ continue
359
+ if now - git_activity(worktree, gdir) < idle_seconds:
360
+ continue
361
+ latest = last_activity(worktree, gdir, [])
362
+ if latest is None or now - latest < idle_seconds:
363
+ continue
364
+ reason = unsaved_work(worktree)
365
+ if reason:
366
+ log(event='worktree_kept', worktree=worktree, reason=reason)
367
+ continue
368
+ size = size_of(worktree)
369
+ if not dry_run:
370
+ common = run_git(worktree, ['rev-parse', '--path-format=absolute', '--git-common-dir'])
371
+ if common is None or common.returncode != 0:
372
+ continue
373
+ out = subprocess.run(
374
+ GIT + ['--git-dir', common.stdout.decode().strip(), 'worktree', 'remove', '--force', worktree],
375
+ capture_output=True, timeout=600)
376
+ if out.returncode != 0 or os.path.lexists(worktree):
377
+ log(event='worktree_kept', worktree=worktree, reason='git_worktree_remove_failed')
378
+ continue
379
+ freed += size
380
+ log(event='worktree_removed', worktree=worktree, mib=size // MIB, dry_run=dry_run)
381
+ return freed
382
+
383
+
384
+ def files_under(directory):
385
+ """Every regular file under directory, never through a link."""
386
+ found = []
387
+ for current, dirs, files in os.walk(directory):
388
+ dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(current, d))]
389
+ for name in files:
390
+ path = os.path.join(current, name)
391
+ try:
392
+ st = os.lstat(path)
393
+ except OSError:
394
+ continue
395
+ if stat.S_ISREG(st.st_mode):
396
+ found.append((st.st_mtime, st.st_blocks * 512, path))
397
+ return found
398
+
399
+
400
+ def unlink(path, dry_run):
401
+ if dry_run:
402
+ return True
403
+ try:
404
+ os.unlink(path)
405
+ return True
406
+ except OSError:
407
+ return False
408
+
409
+
410
+ def cap_cache(directory, cap_bytes, dry_run):
411
+ """Removes the oldest files of a cache until it fits its cap."""
412
+ entries = sorted(files_under(directory))
413
+ total = sum(size for _, size, _ in entries)
414
+ freed = 0
415
+ for _, size, path in entries:
416
+ if total - freed <= cap_bytes:
417
+ break
418
+ if unlink(path, dry_run):
419
+ freed += size
420
+ if not dry_run:
421
+ for current, dirs, files in os.walk(directory, topdown=False):
422
+ if current != directory and not dirs and not files:
423
+ try:
424
+ os.rmdir(current)
425
+ except OSError:
426
+ pass
427
+ if freed:
428
+ log(event='cache_capped', cache=directory, mib=freed // MIB,
429
+ cap_mib=cap_bytes // MIB, dry_run=dry_run)
430
+ return freed
431
+
432
+
433
+ def keep_newest(directory, count, dry_run):
434
+ """Removes all but the newest count files directly in directory."""
435
+ try:
436
+ entries = [e for e in os.scandir(directory) if e.is_file(follow_symlinks=False)]
437
+ except OSError:
438
+ return 0
439
+ entries.sort(key=lambda e: e.stat(follow_symlinks=False).st_mtime, reverse=True)
440
+ freed = 0
441
+ for entry in entries[count:]:
442
+ size = entry.stat(follow_symlinks=False).st_blocks * 512
443
+ if unlink(entry.path, dry_run):
444
+ freed += size
445
+ log(event='superseded_removed', path=entry.path, mib=size // MIB, dry_run=dry_run)
446
+ return freed
447
+
448
+
449
+ def pair(text, kind):
450
+ name, _, value = text.rpartition('=')
451
+ if not name:
452
+ raise argparse.ArgumentTypeError(f'expected DIR={kind}')
453
+ return name, int(value)
454
+
455
+
280
456
  def command(argv, dry_run, name):
281
457
  if shutil.which(argv[0]) is None:
282
458
  return
@@ -296,10 +472,19 @@ def main(argv):
296
472
  parser.add_argument('--path', action='append', default=[])
297
473
  parser.add_argument('--idle-days', type=float, default=7)
298
474
  parser.add_argument('--pressure-mib', type=int, default=15360)
475
+ parser.add_argument('--pressure-inode-percent', type=float, default=0)
299
476
  parser.add_argument('--pressure-idle-hours', type=float, default=12)
300
477
  parser.add_argument('--docker-age-hours', type=int, default=168)
301
478
  parser.add_argument('--pressure-docker-age-hours', type=int, default=24)
302
479
  parser.add_argument('--pressure-cache', action='append', default=[])
480
+ parser.add_argument('--worktree-root', action='append', default=[])
481
+ parser.add_argument('--worktree-idle-days', type=float, default=30)
482
+ parser.add_argument('--cache-cap', action='append', default=[],
483
+ type=lambda text: pair(text, 'MIB'))
484
+ parser.add_argument('--keep-newest', action='append', default=[],
485
+ type=lambda text: pair(text, 'N'))
486
+ parser.add_argument('--mise')
487
+ parser.add_argument('--user')
303
488
  parser.add_argument('--max-depth', type=int, default=4)
304
489
  parser.add_argument('--proc', default='/proc')
305
490
  parser.add_argument('--home-only', action='store_true')
@@ -309,7 +494,8 @@ def main(argv):
309
494
  paths = args.path or ['/', home]
310
495
  now = time.time()
311
496
  before = {p: free_bytes(p) for p in paths}
312
- pressure = min(before.values()) < args.pressure_mib * MIB
497
+ pressure = under_pressure(paths, args.pressure_mib * MIB,
498
+ args.pressure_inode_percent)
313
499
  idle = args.pressure_idle_hours * HOUR if pressure else args.idle_days * 24 * HOUR
314
500
  docker_age = args.pressure_docker_age_hours if pressure else args.docker_age_hours
315
501
  log(event='started', pressure=pressure, idle_hours=idle / HOUR,
@@ -320,6 +506,23 @@ def main(argv):
320
506
  if pressure:
321
507
  for cache in args.pressure_cache:
322
508
  freed += collect_cache(os.path.join(home, cache), idle, now, args.dry_run)
509
+ reclaimed = freed
510
+ for root in args.worktree_root:
511
+ reclaimed += collect_worktrees(os.path.join(home, root),
512
+ args.worktree_idle_days * 24 * HOUR,
513
+ args.proc, now, args.dry_run)
514
+ for cache, cap_mib in args.cache_cap:
515
+ reclaimed += cap_cache(os.path.join(home, cache), cap_mib * MIB, args.dry_run)
516
+ for directory, count in args.keep_newest:
517
+ reclaimed += keep_newest(os.path.join(home, directory), count, args.dry_run)
518
+ if args.mise:
519
+ installs = os.path.join(home, '.local', 'share', 'mise', 'installs')
520
+ before_mise = size_of(installs)
521
+ prune = [args.mise, 'prune', '--yes']
522
+ if args.user:
523
+ prune = ['runuser', '-u', args.user, '--', 'env', f'HOME={home}'] + prune
524
+ command(prune, args.dry_run, 'mise_prune')
525
+ reclaimed += max(0, before_mise - size_of(installs))
323
526
  if not args.home_only:
324
527
  command(['docker', 'image', 'prune', '--all', '--force', '--filter',
325
528
  f'until={docker_age}h'], args.dry_run, 'docker_images')
@@ -328,7 +531,8 @@ def main(argv):
328
531
  command(['apt-get', 'clean'], args.dry_run, 'apt_cache')
329
532
 
330
533
  after = {p: free_bytes(p) for p in paths}
331
- log(event='finished', checkout_mib=freed // MIB, dry_run=args.dry_run,
534
+ log(event='finished', checkout_mib=freed // MIB,
535
+ reclaimed_mib=reclaimed // MIB, dry_run=args.dry_run,
332
536
  free_mib={p: v // MIB for p, v in after.items()})
333
537
 
334
538
 
@@ -3,14 +3,15 @@
3
3
 
4
4
  Managed by personal fleet-sync (tasks/resources.yml) as
5
5
  personal-fleet-disk-guard.service. Every interval it measures free space on the
6
- watched filesystems and the bytes each leaf cgroup under user.slice wrote. When
7
- any watched filesystem is below the floor, it kills only the single cgroup that
6
+ watched filesystems (bytes and inodes) and the bytes each leaf cgroup under user.slice wrote. When
7
+ any watched filesystem is below the floor of free bytes or of free inodes, it
8
+ kills only the single cgroup that
8
9
  wrote the most during the last interval, and only if that is at least the write
9
10
  threshold. It never deletes files, never touches system services and never
10
11
  kills idle work. Free space must be recovered by a person.
11
12
 
12
- Usage: disk-guard.py --path / --path /home/me [--floor-mib N] [--write-mib N]
13
- [--interval S] [--cgroup-root DIR] [--dry-run]
13
+ Usage: disk-guard.py --path / --path /home/me [--floor-mib N]
14
+ [--inode-floor-percent N] [--write-mib N] [--interval S] [--cgroup-root DIR] [--dry-run]
14
15
  """
15
16
  import argparse
16
17
  import json
@@ -61,6 +62,21 @@ def free_bytes(path):
61
62
  return st.f_bavail * st.f_frsize
62
63
 
63
64
 
65
+ def free_inodes_percent(path):
66
+ """Free inodes as a percentage; None on a filesystem without a fixed count."""
67
+ st = os.statvfs(path)
68
+ if st.f_files <= 0:
69
+ return None
70
+ return st.f_favail * 100 / st.f_files
71
+
72
+
73
+ def low_paths(free, inodes, floor, inode_floor):
74
+ """The watched paths below either floor: bytes, or free inodes in percent."""
75
+ return [path for path in free
76
+ if free[path] < floor
77
+ or (inodes.get(path) is not None and inodes[path] < inode_floor)]
78
+
79
+
64
80
  def log(**event):
65
81
  print(json.dumps(event, sort_keys=True), flush=True)
66
82
 
@@ -69,6 +85,7 @@ def main(argv):
69
85
  parser = argparse.ArgumentParser()
70
86
  parser.add_argument('--path', action='append', required=True)
71
87
  parser.add_argument('--floor-mib', type=int, default=3072)
88
+ parser.add_argument('--inode-floor-percent', type=float, default=2)
72
89
  parser.add_argument('--write-mib', type=int, default=32)
73
90
  parser.add_argument('--interval', type=float, default=5.0)
74
91
  parser.add_argument('--cgroup-root', default='/sys/fs/cgroup/user.slice')
@@ -76,6 +93,7 @@ def main(argv):
76
93
  args = parser.parse_args(argv)
77
94
  floor, threshold = args.floor_mib * MIB, args.write_mib * MIB
78
95
  log(event='started', paths=args.path, floor_mib=args.floor_mib,
96
+ inode_floor_percent=args.inode_floor_percent,
79
97
  write_mib=args.write_mib, interval=args.interval, dry_run=args.dry_run)
80
98
  previous = leaf_writes(args.cgroup_root)
81
99
  while True:
@@ -83,7 +101,8 @@ def main(argv):
83
101
  try:
84
102
  current = leaf_writes(args.cgroup_root)
85
103
  free = {path: free_bytes(path) for path in args.path}
86
- low = [path for path, value in free.items() if value < floor]
104
+ inodes = {path: free_inodes_percent(path) for path in args.path}
105
+ low = low_paths(free, inodes, floor, args.inode_floor_percent)
87
106
  victim = choose_victim(previous, current, bool(low), threshold)
88
107
  previous = current
89
108
  if victim is None:
@@ -1,7 +1,12 @@
1
1
  {
2
2
  "version": 1,
3
- "release": "26.04",
4
- "architectures": ["amd64", "arm64"],
3
+ "releases": [
4
+ {
5
+ "release": "26.04",
6
+ "codename": "resolute",
7
+ "architectures": ["amd64", "arm64"]
8
+ }
9
+ ],
5
10
  "snapshot": "20261101T000000Z",
6
11
  "packages": {
7
12
  "sudo": "1.9.17p2-1ubuntu3.2",
@@ -17,7 +22,8 @@
17
22
  "python3": "3.14.3-0ubuntu2",
18
23
  "unzip": "6.0-29ubuntu1",
19
24
  "xz-utils": "5.8.3-1ubuntu0.1",
20
- "ansible-core": "2.20.1-1"
25
+ "ansible-core": "2.20.1-1",
26
+ "systemd-oomd": "259.9-0ubuntu0.1"
21
27
  },
22
28
  "tailscale": {
23
29
  "version": "1.104.5",
@@ -47,7 +53,65 @@
47
53
  "diskGcDockerAgeHours": 168,
48
54
  "diskGcPressureDockerAgeHours": 24,
49
55
  "diskGcPressureCaches": [".bun/install/cache"],
56
+ "diskGcWorktreeRoots": [".t3/worktrees"],
57
+ "diskGcWorktreeIdleDays": 30,
58
+ "diskGcCacheCapsMiB": {
59
+ ".bun/install/cache": 4096,
60
+ ".npm/_cacache": 2048,
61
+ ".cache/uv": 4096,
62
+ ".cache/pip": 1024
63
+ },
64
+ "diskGcPackagesKept": 2,
50
65
  "dockerLogMaxSize": "50m",
51
- "dockerLogMaxFiles": "3"
66
+ "dockerLogMaxFiles": "3",
67
+ "inodeWarnFreePercent": 10,
68
+ "inodeFailFreePercent": 2
69
+ },
70
+ "clock": {
71
+ "warnOffsetSeconds": 2,
72
+ "failOffsetSeconds": 30
73
+ },
74
+ "rebootPendingFailDays": 14,
75
+ "dockerSmokeTest": {
76
+ "image": "busybox:1.37.0@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec",
77
+ "timeoutSeconds": 60
78
+ },
79
+ "browser": {
80
+ "packages": {
81
+ "fonts-liberation": "1:2.1.5-3build1",
82
+ "libasound2t64": "1.2.15.3-1ubuntu1.1",
83
+ "libatk-bridge2.0-0t64": "2.60.4-0ubuntu0.1",
84
+ "libatk1.0-0t64": "2.60.4-0ubuntu0.1",
85
+ "libatspi2.0-0t64": "2.60.4-0ubuntu0.1",
86
+ "libcairo2": "1.18.4-3",
87
+ "libcups2t64": "2.4.16-1ubuntu1.3",
88
+ "libdbus-1-3": "1.16.2-2ubuntu4",
89
+ "libdrm2": "2.4.131-1",
90
+ "libgbm1": "26.0.8-1ubuntu0.3",
91
+ "libglib2.0-0t64": "2.88.0-1ubuntu0.1",
92
+ "libnspr4": "2:4.38.2-1ubuntu1",
93
+ "libnss3": "2:3.120-1ubuntu2.2",
94
+ "libpango-1.0-0": "1.57.0-1",
95
+ "libx11-6": "2:1.8.13-1ubuntu0.1",
96
+ "libxcb1": "1.17.0-2ubuntu1",
97
+ "libxcomposite1": "1:0.4.6-1build1",
98
+ "libxdamage1": "1:1.1.7-1",
99
+ "libxext6": "2:1.3.4-1build3",
100
+ "libxfixes3": "1:6.0.0-2build2",
101
+ "libxkbcommon0": "1.13.1-1",
102
+ "libxrandr2": "2:1.5.4-1build1"
103
+ },
104
+ "headlessShell": {
105
+ "version": "158.0.8127.18",
106
+ "amd64": {
107
+ "archive": "linux64/chrome-headless-shell-linux64",
108
+ "sha256": "18352d22cb0cdf520cbff2ad8c41391f0e02a72886cb53b0fde4bdfc4bd075d6"
109
+ },
110
+ "arm64": {
111
+ "archive": "linux-arm64/chrome-headless-shell-linux-arm64",
112
+ "sha256": "b136462898353d94c7fab50f9cdaa6d8f0d4c190153916936aa0a03353a26bf2"
113
+ }
114
+ },
115
+ "launchTimeoutSeconds": 60
52
116
  }
53
117
  }
@@ -4,8 +4,8 @@
4
4
  # unit starts the app with the graphical session, so the UI is on screen and the
5
5
  # server is up after every boot. Starting or migrating is left to fleet-sync.
6
6
  #
7
- # The upstream .deb declares libxss1, which Ubuntu 26.04 no longer ships and the
8
- # binary does not link. Instead of patching the package or leaving apt with an
7
+ # The upstream .deb declares libxss1, which current Ubuntu releases no longer
8
+ # ship and the binary does not link. Instead of patching the package or leaving apt with an
9
9
  # unmet dependency, the verified package payload is unpacked to its own path and
10
10
  # its real runtime libraries and bundled AppArmor profile are installed.
11
11
  - name: Install the pinned T3 Code desktop app and start it with the session
@@ -8,17 +8,28 @@
8
8
  vars_files:
9
9
  - policy.json
10
10
  tasks:
11
- - name: Require supported Ubuntu
12
- ansible.builtin.assert:
13
- that:
14
- - ansible_facts.distribution == 'Ubuntu'
15
- - ansible_facts.distribution_version == release
11
+ - ansible.builtin.import_tasks: tasks/release.yml
16
12
  - name: Inspect installed packages before possible network interruption
17
13
  ansible.builtin.package_facts:
18
14
  manager: apt
19
- - name: Require maintenance for an installed version change
15
+ # The pin is a minimum: a newer Tailscale is kept and reported as a
16
+ # warning by the worker checks; an older one needs a maintenance window.
17
+ - name: Compare the installed Tailscale with its pin
18
+ ansible.builtin.command:
19
+ argv:
20
+ - dpkg
21
+ - --compare-versions
22
+ - '{{ ansible_facts.packages.tailscale[0].version }}'
23
+ - ge
24
+ - '{{ tailscale.version }}'
25
+ register: tailscale_pin
26
+ changed_when: false
27
+ failed_when: tailscale_pin.rc not in [0, 1]
28
+ check_mode: false
29
+ when: "'tailscale' in ansible_facts.packages"
30
+ - name: Require maintenance for an upgrade to the pin
20
31
  ansible.builtin.assert:
21
- that: "'tailscale' not in ansible_facts.packages or ansible_facts.packages.tailscale[0].version == tailscale.version"
32
+ that: "'tailscale' not in ansible_facts.packages or tailscale_pin.rc == 0"
22
33
  fail_msg: TAILSCALE_UPGRADE_PENDING; upgrading an active VPN can disconnect SSH. Review an explicit maintenance operation.
23
34
  - name: Inspect existing package-source ownership before writing
24
35
  ansible.builtin.find:
@@ -35,7 +46,7 @@
35
46
  fail_msg: TAILSCALE_SOURCE_OWNER_CONFLICT; review the existing signed source and its migration before creating another repository entry.
36
47
  - name: Install pinned upstream signing key
37
48
  ansible.builtin.get_url:
38
- url: https://pkgs.tailscale.com/stable/ubuntu/resolute.noarmor.gpg
49
+ url: https://pkgs.tailscale.com/stable/ubuntu/{{ ubuntu_release.codename }}.noarmor.gpg
39
50
  dest: /usr/share/keyrings/tailscale-archive-keyring.gpg
40
51
  checksum: 'sha256:{{ tailscale.keySha256 }}'
41
52
  owner: root
@@ -44,7 +55,7 @@
44
55
  timeout: 30
45
56
  - name: Declare the signed upstream package source
46
57
  ansible.builtin.apt_repository:
47
- repo: deb [signed-by=/usr/share/keyrings/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/ubuntu resolute main
58
+ repo: deb [signed-by=/usr/share/keyrings/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/ubuntu {{ ubuntu_release.codename }} main
48
59
  filename: tailscale
49
60
  state: present
50
61
  update_cache: false
@@ -0,0 +1,56 @@
1
+ ---
2
+ # Headless Chromium for browser tests (Playwright, Storybook, end-to-end
3
+ # suites): the system libraries it loads, at the snapshot's exact versions,
4
+ # and one pinned Chrome Headless Shell build. Applied to workers that select
5
+ # the web tool profile; the launch below backs worker:browser.
6
+ - name: Install the exact libraries headless Chromium loads
7
+ ansible.builtin.apt:
8
+ name: '{{ item.key }}={{ item.value }}'
9
+ state: present
10
+ lock_timeout: '{{ aptLockTimeoutSeconds }}'
11
+ allow_downgrade: false
12
+ allow_unauthenticated: false
13
+ fail_on_autoremove: true
14
+ policy_rc_d: 101
15
+ environment:
16
+ APT_CONFIG: /etc/personal-fleet/apt.conf
17
+ loop: '{{ browser.packages | dict2items }}'
18
+ - name: Name the headless shell build of this architecture
19
+ ansible.builtin.set_fact:
20
+ browser_build: "{{ browser.headlessShell['arm64' if ansible_facts.architecture == 'aarch64' else 'amd64'] }}"
21
+ browser_root: '/opt/personal-fleet/browser/{{ browser.headlessShell.version }}'
22
+ - name: Create the package cache and the browser directory
23
+ ansible.builtin.file:
24
+ path: '{{ item }}'
25
+ state: directory
26
+ owner: root
27
+ group: root
28
+ mode: '0755'
29
+ loop:
30
+ - /var/cache/personal-fleet
31
+ - '{{ browser_root }}'
32
+ - name: Download the pinned headless shell
33
+ ansible.builtin.get_url:
34
+ url: 'https://cdn.playwright.dev/builds/cft/{{ browser.headlessShell.version }}/{{ browser_build.archive }}.zip'
35
+ dest: '/var/cache/personal-fleet/chrome-headless-shell-{{ browser.headlessShell.version }}-{{ browser_build.archive | basename }}.zip'
36
+ checksum: 'sha256:{{ browser_build.sha256 }}'
37
+ mode: '0644'
38
+ timeout: 300
39
+ - name: Unpack the verified headless shell
40
+ ansible.builtin.unarchive:
41
+ src: '/var/cache/personal-fleet/chrome-headless-shell-{{ browser.headlessShell.version }}-{{ browser_build.archive | basename }}.zip'
42
+ dest: '{{ browser_root }}'
43
+ remote_src: true
44
+ creates: '{{ browser_root }}/{{ browser_build.archive | basename }}/chrome-headless-shell'
45
+ - name: Launch headless as the developer, load a blank page and exit (BROWSER_LAUNCH_FAILED)
46
+ ansible.builtin.command:
47
+ argv:
48
+ - '{{ browser_root }}/{{ browser_build.archive | basename }}/chrome-headless-shell'
49
+ - --headless
50
+ - --no-sandbox
51
+ - --disable-gpu
52
+ - --dump-dom
53
+ - about:blank
54
+ become_user: '{{ development_user }}'
55
+ timeout: '{{ browser.launchTimeoutSeconds }}'
56
+ changed_when: false
@@ -29,3 +29,55 @@
29
29
  - [docker, compose, version]
30
30
  - [docker, buildx, version]
31
31
  changed_when: false
32
+ # The engine answering is not proof that an agent can use it: run a pinned
33
+ # image and build a one-line image as the developer, in a fresh session that
34
+ # reads the docker group from scratch. The image is cached here so the same
35
+ # test works offline in worker:docker-run.
36
+ - name: Look for the cached smoke-test image
37
+ ansible.builtin.command:
38
+ argv:
39
+ [
40
+ docker,
41
+ image,
42
+ inspect,
43
+ --format,
44
+ '{{ "{{.Id}}" }}',
45
+ '{{ dockerSmokeTest.image }}',
46
+ ]
47
+ become_user: '{{ development_user }}'
48
+ check_mode: false
49
+ changed_when: false
50
+ failed_when: false
51
+ register: docker_smoke_image
52
+ - name: Cache the pinned smoke-test image
53
+ ansible.builtin.command:
54
+ argv: [docker, pull, --quiet, '{{ dockerSmokeTest.image }}']
55
+ become_user: '{{ development_user }}'
56
+ when: docker_smoke_image.rc | default(0) != 0
57
+ changed_when: true
58
+ - name: Run a container as the developer, offline (DOCKER_RUN_FAILED)
59
+ ansible.builtin.command:
60
+ argv:
61
+ [
62
+ docker,
63
+ run,
64
+ --rm,
65
+ --network,
66
+ none,
67
+ --pull,
68
+ never,
69
+ '{{ dockerSmokeTest.image }}',
70
+ 'true',
71
+ ]
72
+ become_user: '{{ development_user }}'
73
+ timeout: '{{ dockerSmokeTest.timeoutSeconds }}'
74
+ changed_when: false
75
+ - name: Build a one-line image as the developer, offline (DOCKER_BUILD_FAILED)
76
+ ansible.builtin.command:
77
+ argv: [docker, build, --quiet, --pull=false, --network, none, '-']
78
+ stdin: |
79
+ FROM {{ dockerSmokeTest.image }}
80
+ RUN true
81
+ become_user: '{{ development_user }}'
82
+ timeout: '{{ dockerSmokeTest.timeoutSeconds }}'
83
+ changed_when: false