@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/ansible/ubuntu/tasks/resources.yml

@@ -0,0 +1,312 @@
+---
+# Memory and disk safety: a runaway workload is stopped on its own before the
+# machine, SSH, Tailscale or the T3 server run out of memory or disk. These
+# tasks run before the capacity preflight because they matter most when a disk
+# is nearly full; they only write small files and never delete user data. The
+# disk GC timer they install deletes only build output and caches a build or
+# install recreates (ansible/ubuntu/disk-gc.py).
+- name: Inspect active swap
+  ansible.builtin.command:
+    argv: [swapon, --show=NAME, --noheadings]
+  changed_when: false
+  check_mode: false
+  register: active_swap
+- name: Measure root free space before creating swap
+  ansible.builtin.command:
+    argv: [df, --output=avail, -BM, /]
+  changed_when: false
+  check_mode: false
+  register: swap_capacity
+  when: active_swap.stdout | trim == ''
+- name: Refuse to create swap on a nearly full disk
+  ansible.builtin.assert:
+    that: (swap_capacity.stdout_lines | last | trim | regex_replace('M$', '') | int) >= resourceSafety.swapFileMiB + minimumFreeDiskMiB
+    fail_msg: SWAP_CAPACITY_REQUIRED; free disk space so a swap file can be created. No swap is active.
+  when: active_swap.stdout | trim == ''
+- name: Create a swap file when the node has no swap
+  ansible.builtin.command:
+    argv: [fallocate, -l, '{{ resourceSafety.swapFileMiB }}M', /swap.fleet]
+    creates: /swap.fleet
+  when: active_swap.stdout | trim == ''
+- name: Restrict the fleet swap file
+  ansible.builtin.file:
+    path: /swap.fleet
+    owner: root
+    group: root
+    mode: '0600'
+  when: active_swap.stdout | trim == ''
+- name: Format the fleet swap file
+  ansible.builtin.command:
+    argv: [mkswap, /swap.fleet]
+  when: active_swap.stdout | trim == ''
+- name: Activate the fleet swap file
+  ansible.builtin.command:
+    argv: [swapon, /swap.fleet]
+  when: active_swap.stdout | trim == ''
+- name: Activate the fleet swap file at boot
+  ansible.builtin.lineinfile:
+    path: /etc/fstab
+    line: /swap.fleet none swap sw 0 0
+  when: active_swap.stdout | trim == ''
+
+- name: Install the systemd userspace OOM killer
+  ansible.builtin.apt:
+    name: systemd-oomd
+    state: present
+- name: Create memory and disk policy directories
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+  loop:
+    - /etc/systemd/oomd.conf.d
+    - /etc/systemd/system/-.slice.d
+    - /etc/systemd/system/user.slice.d
+    - /etc/systemd/user/t3code-.scope.d
+    - /etc/systemd/journald.conf.d
+    - /etc/systemd/system/systemd-tmpfiles-clean.timer.d
+    - /etc/tmpfiles.d
+- name: Declare systemd-oomd limits
+  ansible.builtin.copy:
+    dest: /etc/systemd/oomd.conf.d/60-personal-fleet.conf
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      # Managed by personal fleet-sync: kill one cgroup before swap or memory pressure stalls the node.
+      [OOM]
+      SwapUsedLimit=90%
+      DefaultMemoryPressureLimit=60%
+      DefaultMemoryPressureDurationSec=20s
+  register: oomd_config
+- name: Declare memory reserve and OOM policy for system and user slices
+  ansible.builtin.copy:
+    dest: '{{ item.dest }}'
+    owner: root
+    group: root
+    mode: '0644'
+    content: '{{ item.content }}'
+  loop:
+    - dest: /etc/systemd/system/-.slice.d/60-personal-fleet.conf
+      content: |
+        # Managed by personal fleet-sync: when swap is exhausted, oomd kills the largest swap user.
+        [Slice]
+        ManagedOOMSwap=kill
+    - dest: /etc/systemd/system/user.slice.d/60-personal-fleet.conf
+      content: |
+        # Managed by personal fleet-sync: user workloads can never take the memory SSH, Tailscale and system services need.
+        [Slice]
+        MemoryMax={{ resourceSafety.userMemoryMax }}
+  loop_control:
+    label: '{{ item.dest }}'
+  register: slice_policy
+- name: Bound each T3 agent so a runaway loses one process, not the session or the node
+  ansible.builtin.copy:
+    dest: /etc/systemd/user/t3code-.scope.d/60-personal-fleet.conf
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      # Managed by personal fleet-sync. Applies to every t3code-*.scope T3 starts for an agent.
+      # OOMPolicy=continue: the kernel kills the largest process in the agent; the agent itself keeps running.
+      [Scope]
+      MemoryMax={{ resourceSafety.agentMemoryMax }}
+      MemorySwapMax={{ resourceSafety.agentSwapMax }}
+      OOMPolicy=continue
+  register: agent_policy
+- name: Apply slice limits without restarting services
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+  when: slice_policy.changed
+- name: Load the agent limits into the running user manager for new agents
+  become: true
+  become_user: '{{ development_user }}'
+  environment:
+    XDG_RUNTIME_DIR: '/run/user/{{ development_uid }}'
+  ansible.builtin.systemd_service:
+    scope: user
+    daemon_reload: true
+  when: agent_policy.changed
+  failed_when: false
+- name: Run systemd-oomd
+  ansible.builtin.systemd_service:
+    name: systemd-oomd.service
+    enabled: true
+    state: '{{ "restarted" if oomd_config.changed else "started" }}'
+
+- name: Cap the system journal
+  ansible.builtin.copy:
+    dest: /etc/systemd/journald.conf.d/60-personal-fleet.conf
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      # Managed by personal fleet-sync.
+      [Journal]
+      SystemMaxUse={{ resourceSafety.journalMaxUse }}
+      SystemKeepFree={{ resourceSafety.journalKeepFree }}
+      RuntimeMaxUse=256M
+  register: journal_policy
+- name: Apply the journal cap
+  ansible.builtin.systemd_service:
+    name: systemd-journald.service
+    state: restarted
+  when: journal_policy.changed
+- name: Age out abandoned temporary files
+  ansible.builtin.copy:
+    dest: /etc/tmpfiles.d/tmp.conf
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      # Managed by personal fleet-sync; overrides /usr/lib/tmpfiles.d/tmp.conf.
+      q /tmp 1777 root root {{ resourceSafety.tmpAge }}
+      q /var/tmp 1777 root root {{ resourceSafety.varTmpAge }}
+- name: Clean temporary files hourly instead of daily
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/systemd-tmpfiles-clean.timer.d/60-personal-fleet.conf
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      # Managed by personal fleet-sync.
+      [Timer]
+      OnUnitActiveSec=1h
+  register: tmpfiles_timer
+- name: Inspect the RAM-backed /tmp unit
+  ansible.builtin.stat:
+    path: /usr/lib/systemd/system/tmp.mount
+  register: tmp_mount
+- name: Create the /tmp mount policy directory
+  ansible.builtin.file:
+    path: /etc/systemd/system/tmp.mount.d
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+  when: tmp_mount.stat.exists
+- name: Bound RAM-backed /tmp (effective after the next reboot)
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/tmp.mount.d/60-personal-fleet.conf
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      # Managed by personal fleet-sync: /tmp is RAM; keep it from taking half of memory.
+      [Mount]
+      Options=mode=1777,strictatime,nosuid,nodev,size={{ resourceSafety.tmpMaxSize }},nr_inodes=1m,x-systemd.graceful-option=usrquota
+  when: tmp_mount.stat.exists
+  register: tmp_policy
+- name: Reload the cleanup timer and /tmp definitions
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+  when: tmpfiles_timer.changed or tmp_policy.changed
+
+- name: Read the tracked disk guard
+  ansible.builtin.stat:
+    path: '{{ development_source }}/ansible/ubuntu/disk-guard.py'
+    checksum_algorithm: sha256
+  register: disk_guard_source
+  when: development_source is defined
+- name: Declare the disk guard service
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/personal-fleet-disk-guard.service
+    owner: root
+    group: root
+    mode: '0644'
+    content: |
+      # Managed by personal fleet-sync: ansible/ubuntu/disk-guard.py
+      # Source sha256 {{ disk_guard_source.stat.checksum }}
+      [Unit]
+      Description=Personal Fleet disk guard (stops a runaway user writer before the disk fills)
+      StartLimitIntervalSec=0
+      [Service]
+      Type=simple
+      ExecStart=/usr/bin/python3 -I -B {{ disk_guard_source.stat.path }} --path / --path {{ development_home }} --floor-mib {{ resourceSafety.diskGuardFloorMiB }} --write-mib {{ resourceSafety.diskGuardWriteMiB }} --interval {{ resourceSafety.diskGuardIntervalSeconds }}
+      Restart=always
+      RestartSec=5
+      OOMScoreAdjust=-1000
+      MemoryMax=64M
+      MemorySwapMax=0
+      CPUQuota=10%
+      TasksMax=4
+      NoNewPrivileges=true
+      PrivateTmp=true
+      PrivateNetwork=true
+      ProtectSystem=strict
+      ProtectHome=read-only
+      ReadWritePaths=/sys/fs/cgroup
+      ProtectKernelModules=true
+      LogRateLimitIntervalSec=60
+      LogRateLimitBurst=20
+      [Install]
+      WantedBy=multi-user.target
+  when: development_source is defined and disk_guard_source.stat.exists
+  register: disk_guard_unit
+- name: Run the disk guard
+  ansible.builtin.systemd_service:
+    name: personal-fleet-disk-guard.service
+    daemon_reload: '{{ disk_guard_unit.changed }}'
+    enabled: true
+    state: '{{ "restarted" if disk_guard_unit.changed else "started" }}'
+  when: development_source is defined and disk_guard_source.stat.exists
+
+- name: Read the tracked disk GC
+  ansible.builtin.stat:
+    path: '{{ development_source }}/ansible/ubuntu/disk-gc.py'
+    checksum_algorithm: sha256
+  register: disk_gc_source
+  when: development_source is defined
+- name: Declare the disk GC service and its hourly timer
+  ansible.builtin.copy:
+    dest: '{{ item.dest }}'
+    owner: root
+    group: root
+    mode: '0644'
+    content: '{{ item.content }}'
+  loop:
+    - dest: /etc/systemd/system/personal-fleet-disk-gc.service
+      content: |
+        # Managed by personal fleet-sync: ansible/ubuntu/disk-gc.py
+        # Source sha256 {{ disk_gc_source.stat.checksum }}
+        [Unit]
+        Description=Personal Fleet disk GC (deletes build output and caches a build recreates)
+        After=docker.service
+        [Service]
+        Type=oneshot
+        ExecStart=/usr/bin/python3 -I -B {{ disk_gc_source.stat.path }} --home {{ development_home }} --path / --path {{ development_home }} --idle-days {{ resourceSafety.diskGcIdleDays }} --pressure-mib {{ resourceSafety.diskGcPressureMiB }} --pressure-idle-hours {{ resourceSafety.diskGcPressureIdleHours }} --docker-age-hours {{ resourceSafety.diskGcDockerAgeHours }} --pressure-docker-age-hours {{ resourceSafety.diskGcPressureDockerAgeHours }}{% for cache in resourceSafety.diskGcPressureCaches %} --pressure-cache {{ cache }}{% endfor %}
+
+        TimeoutStartSec=2h
+        Nice=19
+        IOSchedulingClass=idle
+        CPUQuota=50%
+        MemoryMax=512M
+        NoNewPrivileges=true
+        PrivateTmp=true
+        LogRateLimitIntervalSec=60
+        LogRateLimitBurst=200
+    - dest: /etc/systemd/system/personal-fleet-disk-gc.timer
+      content: |
+        # Managed by personal fleet-sync: ansible/ubuntu/disk-gc.py
+        [Unit]
+        Description=Run the Personal Fleet disk GC hourly
+        [Timer]
+        OnBootSec=10min
+        OnCalendar=hourly
+        RandomizedDelaySec=5min
+        Persistent=true
+        [Install]
+        WantedBy=timers.target
+  loop_control:
+    label: '{{ item.dest }}'
+  when: development_source is defined and disk_gc_source.stat.exists
+  register: disk_gc_units
+- name: Schedule the disk GC
+  ansible.builtin.systemd_service:
+    name: personal-fleet-disk-gc.timer
+    daemon_reload: '{{ disk_gc_units.changed }}'
+    enabled: true
+    state: started
+  when: development_source is defined and disk_gc_source.stat.exists

package/ansible/ubuntu/tasks/services.yml

@@ -0,0 +1,31 @@
+---
+# Always-on recovery for the system services every worker depends on. Drop-ins
+# change restart policy only; running services are never restarted here.
+- name: Create recovery drop-in directories for required system services
+  ansible.builtin.file:
+    path: '/etc/systemd/system/{{ item }}.service.d'
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+  loop: '{{ alwaysOnServices }}'
+- name: Declare unlimited automatic restart for required system services
+  ansible.builtin.copy:
+    dest: '/etc/systemd/system/{{ item }}.service.d/60-personal-fleet.conf'
+    owner: root
+    group: root
+    mode: '0644'
+    backup: true
+    content: |
+      # Managed by personal fleet-sync: keep this worker service always on.
+      [Unit]
+      StartLimitIntervalSec=0
+      [Service]
+      Restart=always
+      RestartSec=5
+  loop: '{{ alwaysOnServices }}'
+  register: recovery_dropins
+- name: Reload unit definitions without restarting running services
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+  when: recovery_dropins.changed

package/ansible/ubuntu/tasks/ssh.yml

@@ -0,0 +1,71 @@
+---
+- name: Inspect effective SSH configuration before changes
+  ansible.builtin.command: /usr/sbin/sshd -T
+  changed_when: false
+  register: effective_ssh
+- name: Inspect any existing managed SSH fragment
+  ansible.builtin.stat:
+    path: /etc/ssh/sshd_config.d/60-personal-fleet.conf
+  register: ssh_owned_file
+- name: Read previous fragment for validation rollback
+  ansible.builtin.slurp:
+    src: /etc/ssh/sshd_config.d/60-personal-fleet.conf
+  when: ssh_owned_file.stat.exists
+  register: ssh_previous
+  no_log: true
+- name: Refuse to overwrite unmanaged SSH configuration
+  ansible.builtin.assert:
+    that:
+      - not ssh_owned_file.stat.exists or (ssh_previous.content | b64decode).startswith('# Managed by personal Fleet')
+    fail_msg: SSH_UNMANAGED_CONFIG_CONFLICT; review the existing fragment before adoption.
+- name: Stage and validate forwarding policy while preserving listeners and authentication
+  block:
+    - name: Write forwarding-only owned fragment
+      ansible.builtin.copy:
+        dest: /etc/ssh/sshd_config.d/60-personal-fleet.conf
+        owner: root
+        group: root
+        mode: '0644'
+        backup: true
+        content: |
+          # Managed by personal Fleet; listeners and authentication remain explicit policy.
+          AllowTcpForwarding yes
+          ClientAliveInterval 30
+          ClientAliveCountMax 3
+      register: ssh_fragment
+    - name: Validate complete SSH configuration
+      ansible.builtin.command: /usr/sbin/sshd -t
+      changed_when: false
+  rescue:
+    - name: Restore previous fragment after validation failure
+      ansible.builtin.copy:
+        content: '{{ ssh_previous.content | b64decode }}'
+        dest: /etc/ssh/sshd_config.d/60-personal-fleet.conf
+        owner: root
+        group: root
+        mode: '0644'
+      when: ssh_owned_file.stat.exists
+      no_log: true
+    - name: Remove rejected new fragment
+      ansible.builtin.file:
+        path: /etc/ssh/sshd_config.d/60-personal-fleet.conf
+        state: absent
+      when: not ssh_owned_file.stat.exists
+    - name: Report validation failure without reloading sshd
+      ansible.builtin.fail:
+        msg: SSH_CONFIG_REJECTED; previous configuration restored, active service untouched.
+- name: Enable existing SSH service at boot
+  ansible.builtin.systemd_service:
+    name: ssh.service
+    enabled: true
+    state: started
+- name: Reload validated forwarding policy without terminating sessions
+  ansible.builtin.systemd_service:
+    name: ssh.service
+    state: reloaded
+  when: ssh_fragment.changed
+- name: Verify effective forwarding after reload
+  ansible.builtin.command: /usr/sbin/sshd -T
+  changed_when: false
+  register: ssh_verified
+  failed_when: "'allowtcpforwarding yes' not in ssh_verified.stdout"

package/ansible/ubuntu/tasks/user.yml

@@ -0,0 +1,120 @@
+---
+- name: Inspect existing designated-user sudo policy ownership
+  ansible.builtin.stat:
+    path: '/etc/sudoers.d/personal-fleet-{{ development_user }}'
+  register: sudo_owned_file
+- name: Read existing designated-user sudo policy for ownership validation
+  ansible.builtin.slurp:
+    src: '/etc/sudoers.d/personal-fleet-{{ development_user }}'
+  when: sudo_owned_file.stat.exists
+  register: sudo_previous
+  no_log: true
+- name: Preserve conflicting sudo policy until a reviewed migration
+  ansible.builtin.assert:
+    that:
+      - not sudo_owned_file.stat.islnk | default(false)
+      - "not sudo_owned_file.stat.exists or (sudo_previous.content | b64decode).startswith('# Managed by personal Fleet: designated development user')"
+    fail_msg: SUDOERS_OWNERSHIP_CONFLICT; preserve and review the existing fragment before adoption.
+- name: Validate and install designated-user sudo policy
+  ansible.builtin.copy:
+    dest: '/etc/sudoers.d/personal-fleet-{{ development_user }}'
+    content: |
+      # Managed by personal Fleet: designated development user
+      {{ development_user }} ALL=(ALL:ALL) NOPASSWD: ALL
+    owner: root
+    group: root
+    mode: '0440'
+    validate: /usr/sbin/visudo -cf %s
+    backup: true
+- name: Enable lingering without requiring an open desktop session
+  ansible.builtin.command:
+    argv: [loginctl, enable-linger, '{{ development_user }}']
+    creates: '/var/lib/systemd/linger/{{ development_user }}'
+- name: Check whether GNOME settings schemas are installed
+  ansible.builtin.stat:
+    path: /usr/share/glib-2.0/schemas/org.gnome.desktop.screensaver.gschema.xml
+  register: gnome_schema
+- name: Check whether the GNOME power plugin schema is installed
+  ansible.builtin.stat:
+    path: /usr/share/glib-2.0/schemas/org.gnome.settings-daemon.plugins.power.gschema.xml
+  register: gnome_power_schema
+- name: Ensure user configuration directory exists
+  ansible.builtin.file:
+    path: '{{ development_home }}/.config'
+    state: directory
+    owner: '{{ development_user }}'
+    mode: '0700'
+  when: gnome_schema.stat.exists
+- name: Inspect designated-user GNOME lock and idle policy
+  become: true
+  become_user: '{{ development_user }}'
+  ansible.builtin.command:
+    argv:
+      [
+        dbus-run-session,
+        '--',
+        gsettings,
+        get,
+        '{{ item.schema }}',
+        '{{ item.key }}',
+      ]
+  environment:
+    HOME: '{{ development_home }}'
+    XDG_CONFIG_HOME: '{{ development_home }}/.config'
+  loop: '{{ gnome_session_settings + (gnome_power_settings if gnome_power_schema.stat.exists else []) }}'
+  vars:
+    gnome_session_settings:
+      - {
+          schema: org.gnome.desktop.screensaver,
+          key: lock-enabled,
+          value: 'false',
+          observed: 'false',
+        }
+      - {
+          schema: org.gnome.desktop.session,
+          key: idle-delay,
+          value: '0',
+          observed: 'uint32 0',
+        }
+    gnome_power_settings:
+      - {
+          schema: org.gnome.settings-daemon.plugins.power,
+          key: sleep-inactive-ac-type,
+          value: 'nothing',
+          observed: "'nothing'",
+        }
+      - {
+          schema: org.gnome.settings-daemon.plugins.power,
+          key: sleep-inactive-battery-type,
+          value: 'nothing',
+          observed: "'nothing'",
+        }
+      - {
+          schema: org.gnome.settings-daemon.plugins.power,
+          key: idle-dim,
+          value: 'false',
+          observed: 'false',
+        }
+  when: gnome_schema.stat.exists
+  changed_when: false
+  register: gnome_settings
+- name: Repair only changed designated-user settings
+  become: true
+  become_user: '{{ development_user }}'
+  ansible.builtin.command:
+    argv:
+      [
+        dbus-run-session,
+        '--',
+        gsettings,
+        set,
+        '{{ item.item.schema }}',
+        '{{ item.item.key }}',
+        '{{ item.item.value }}',
+      ]
+  environment:
+    HOME: '{{ development_home }}'
+    XDG_CONFIG_HOME: '{{ development_home }}/.config'
+  loop: '{{ gnome_settings.results }}'
+  when: gnome_schema.stat.exists and item.stdout | default('') != item.item.observed
+  changed_when: true