@crvouga/dev-fleet 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +88 -7
  2. package/agents/audit.md +18 -18
  3. package/ansible/agents/agents.yml +134 -0
  4. package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
  5. package/ansible/agents/templates/claude.settings.json.j2 +3 -0
  6. package/ansible/agents/templates/codex.config.toml.j2 +11 -0
  7. package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
  8. package/ansible/agents/templates/grok.config.toml.j2 +19 -0
  9. package/ansible/agents/templates/opencode.json.j2 +24 -0
  10. package/ansible/agents/templates/receipt.json.j2 +10 -0
  11. package/ansible/ansible.cfg +14 -0
  12. package/ansible/github/git.yml +82 -0
  13. package/ansible/macos/manager-privilege.yml +83 -0
  14. package/ansible/macos/manager.yml +164 -9
  15. package/ansible/macos/policy.json +13 -0
  16. package/ansible/macos/t3-policy.yml +5 -3
  17. package/ansible/macos/vault-renew.yml +85 -0
  18. package/ansible/macos/worker-host-key.yml +33 -0
  19. package/ansible/monitoring/agent-darwin.yml +3 -0
  20. package/ansible/monitoring/agent.yml +2 -0
  21. package/ansible/monitoring/hub-darwin.yml +3 -0
  22. package/ansible/monitoring/hub-route-darwin.yml +1 -0
  23. package/ansible/net/remote-probe.sh +4 -1
  24. package/ansible/platforms.json +2 -2
  25. package/ansible/pyproject.toml +5 -0
  26. package/ansible/ubuntu/baseline.yml +39 -13
  27. package/ansible/ubuntu/disk-gc.py +6 -2
  28. package/ansible/ubuntu/firewall.nft.j2 +50 -0
  29. package/ansible/ubuntu/policy.json +125 -6
  30. package/ansible/ubuntu/reboot.yml +89 -0
  31. package/ansible/ubuntu/recovery.yml +13 -3
  32. package/ansible/ubuntu/t3/policy.json +3 -0
  33. package/ansible/ubuntu/t3-desktop.yml +23 -2
  34. package/ansible/ubuntu/t3-headless.yml +128 -0
  35. package/ansible/ubuntu/t3-health.sh +24 -10
  36. package/ansible/ubuntu/t3-idle.py +37 -2
  37. package/ansible/ubuntu/t3-policy.yml +4 -1
  38. package/ansible/ubuntu/t3-route.yml +1 -0
  39. package/ansible/ubuntu/tailscale.yml +54 -8
  40. package/ansible/ubuntu/tasks/browser.yml +3 -0
  41. package/ansible/ubuntu/tasks/containers-off.yml +43 -0
  42. package/ansible/ubuntu/tasks/docker.yml +53 -6
  43. package/ansible/ubuntu/tasks/dpkg.yml +4 -0
  44. package/ansible/ubuntu/tasks/features.yml +22 -0
  45. package/ansible/ubuntu/tasks/firewall.yml +172 -0
  46. package/ansible/ubuntu/tasks/install.yml +2 -1
  47. package/ansible/ubuntu/tasks/packages.yml +4 -2
  48. package/ansible/ubuntu/tasks/release.yml +6 -0
  49. package/ansible/ubuntu/tasks/resources.yml +6 -4
  50. package/ansible/ubuntu/tasks/services.yml +2 -2
  51. package/ansible/ubuntu/tasks/sizing.yml +49 -0
  52. package/ansible/ubuntu/tasks/snapshot.yml +14 -3
  53. package/ansible/ubuntu/tasks/ssh.yml +61 -4
  54. package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
  55. package/ansible/ubuntu/tasks/user.yml +1 -0
  56. package/ansible/ubuntu/worker-account.yml +117 -0
  57. package/ansible/uv.lock +265 -0
  58. package/bin/devfleet.mjs +42 -4
  59. package/bootstrap/fleet-remote-bootstrap.sh +91 -25
  60. package/bootstrap/global-config.ts +29 -3
  61. package/bootstrap/package-smoke.ts +127 -0
  62. package/bootstrap/pins.ts +189 -0
  63. package/bootstrap/release-notes.ts +88 -0
  64. package/bootstrap/release-version.ts +41 -7
  65. package/bootstrap/worker-features.ts +114 -0
  66. package/bootstrap/worker-seed.yaml +65 -0
  67. package/bootstrap/worker-tools.ts +39 -9
  68. package/cliproxy/README.md +1 -1
  69. package/cliproxy/docs/health-workflow.example.yml +20 -143
  70. package/docs/authentication.md +21 -11
  71. package/docs/consumer-setup.md +74 -58
  72. package/docs/convergence.md +67 -0
  73. package/docs/declared.md +61 -34
  74. package/docs/machine-setup.md +73 -7
  75. package/docs/manager-setup.md +34 -0
  76. package/docs/operations.md +64 -9
  77. package/docs/releasing.md +49 -1
  78. package/docs/runbooks.md +112 -7
  79. package/docs/state.md +23 -11
  80. package/docs/support-matrix.md +40 -4
  81. package/docs/topology.md +82 -12
  82. package/docs/ubuntu-sync.md +98 -42
  83. package/docs/worker-contract.md +122 -19
  84. package/docs/worker-requirements.md +107 -0
  85. package/examples/fleet.json +1 -0
  86. package/examples/workers-from-infrastructure/main.tf +132 -0
  87. package/examples/workers-from-infrastructure/worker-records.json +14 -0
  88. package/llms.txt +3 -1
  89. package/mise.lock +29 -0
  90. package/mise.toml +5 -3
  91. package/package.json +2 -2
  92. package/schema/check-ids.json +1991 -162
  93. package/schema/check-ids.schema.json +34 -0
  94. package/schema/fleet.schema.json +349 -10
  95. package/schema/output/check.schema.json +1 -1
  96. package/schema/output/fleet-state-definition.schema.json +14 -0
  97. package/schema/output/fleet-sync-schedule.schema.json +26 -0
  98. package/schema/output/fleet-sync.schema.json +18 -0
  99. package/schema/output/report.schema.json +3 -2
  100. package/schema/worker-record.schema.json +76 -0
  101. package/src/agents/converge.ts +140 -24
  102. package/src/agents/files.ts +2 -32
  103. package/src/agents/native-verify.ts +34 -3
  104. package/src/agents/secrets.ts +39 -4
  105. package/src/check-owners.ts +62 -2
  106. package/src/check-registry.ts +630 -0
  107. package/src/cli/agent-guide.ts +3 -3
  108. package/src/cli/cli.ts +4 -0
  109. package/src/cli/fleet-manifest.ts +108 -0
  110. package/src/cli/fleet-node-lifecycle.ts +704 -0
  111. package/src/cli/fleet-node-park.ts +238 -0
  112. package/src/cli/fleet-node-seed.ts +167 -0
  113. package/src/cli/fleet-state.ts +2 -1
  114. package/src/cli/fleet-sync-checkout.ts +7 -0
  115. package/src/cli/fleet-sync-enrollment.ts +193 -32
  116. package/src/cli/fleet-sync-forward.ts +6 -0
  117. package/src/cli/fleet-sync-manager.ts +123 -8
  118. package/src/cli/fleet-sync-output.ts +31 -11
  119. package/src/cli/fleet-sync-plan.ts +65 -9
  120. package/src/cli/fleet-sync-records.ts +113 -0
  121. package/src/cli/fleet-sync-remote.ts +172 -15
  122. package/src/cli/fleet-sync-schedule.ts +14 -4
  123. package/src/cli/fleet-sync-state.ts +31 -10
  124. package/src/cli/fleet-sync-vault.ts +152 -26
  125. package/src/cli/fleet-sync-worker.ts +214 -0
  126. package/src/cli/fleet-sync.ts +172 -21
  127. package/src/cli/fleet-validate.ts +277 -0
  128. package/src/cli/fleet-watch.ts +7 -1
  129. package/src/cli/fleet.ts +87 -5
  130. package/src/cli/managed-health.ts +25 -1
  131. package/src/cli/register-fleet.ts +251 -4
  132. package/src/cli/registry.ts +47 -0
  133. package/src/cli/tailnet.ts +43 -10
  134. package/src/cli/ui-data.ts +3 -1
  135. package/src/cli/worker-usage-report.ts +63 -0
  136. package/src/clock.ts +42 -0
  137. package/src/definition.ts +762 -232
  138. package/src/diagnostics.ts +5 -2
  139. package/src/exposure.ts +102 -0
  140. package/src/fleet-config.ts +3 -1
  141. package/src/git-identity.ts +76 -3
  142. package/src/ide/ide.ts +5 -0
  143. package/src/ide/t3.ts +12 -2
  144. package/src/input-hashes.ts +6 -0
  145. package/src/machine/machine-class.ts +59 -0
  146. package/src/machine/platform.ts +90 -11
  147. package/src/machine/state.ts +70 -16
  148. package/src/machine/support-matrix.ts +11 -9
  149. package/src/machine-health.ts +28 -5
  150. package/src/manager-health.ts +148 -2
  151. package/src/manager-sync.ts +89 -8
  152. package/src/monitoring/beszel.ts +8 -8
  153. package/src/monitoring/enroll.ts +8 -1
  154. package/src/monitoring/hub-access.ts +10 -4
  155. package/src/monitoring/systems.ts +138 -0
  156. package/src/network-definition.ts +13 -4
  157. package/src/node-converge-linux.ts +136 -27
  158. package/src/node-converge.ts +13 -3
  159. package/src/platform/ansible-receipt.ts +17 -2
  160. package/src/platform/ansible.ts +40 -4
  161. package/src/platform/manager-host.ts +25 -8
  162. package/src/platform/resource-verify.ts +18 -7
  163. package/src/platform/t3-policy.ts +24 -10
  164. package/src/platform/t3-service.ts +109 -34
  165. package/src/platform/ubuntu-release.ts +24 -0
  166. package/src/platform/ubuntu-verify.ts +31 -3
  167. package/src/platform/ubuntu-worker.ts +58 -0
  168. package/src/platform/worker-platform.ts +72 -0
  169. package/src/protocol.ts +1 -1
  170. package/src/public-reachability.ts +186 -0
  171. package/src/remote-bootstrap.ts +50 -12
  172. package/src/remote-operation.ts +18 -4
  173. package/src/repair-plan.ts +23 -12
  174. package/src/report.ts +2 -1
  175. package/src/schedule-check.ts +21 -2
  176. package/src/secrets.ts +7 -0
  177. package/src/service-auth.ts +55 -48
  178. package/src/ssh-connection.ts +141 -0
  179. package/src/ssh.ts +63 -24
  180. package/src/state/store.ts +4 -2
  181. package/src/state/sync.ts +35 -6
  182. package/src/state/workers.ts +140 -0
  183. package/src/t3/pairing.ts +59 -0
  184. package/src/t3/peer-link.ts +333 -0
  185. package/src/t3/version.ts +1 -1
  186. package/src/tailnet-cleanup.ts +185 -0
  187. package/src/tailnet-control.ts +346 -0
  188. package/src/tailscale.ts +1 -1
  189. package/src/toolchain-checks.ts +11 -1
  190. package/src/types.ts +4 -2
  191. package/src/worker-account.ts +63 -0
  192. package/src/worker-activity.ts +303 -0
  193. package/src/worker-capacity.ts +31 -0
  194. package/src/worker-credentials.ts +201 -0
  195. package/src/worker-exposure.ts +388 -0
  196. package/src/worker-features.ts +201 -0
  197. package/src/worker-host-checks.ts +71 -20
  198. package/src/worker-manifest.ts +224 -0
  199. package/src/worker-parking.ts +70 -0
  200. package/src/worker-policy.ts +38 -36
  201. package/src/worker-readiness.ts +15 -3
  202. package/src/worker-reboot.ts +129 -0
  203. package/src/worker-records.ts +305 -0
  204. package/src/worker-seed.ts +143 -0
  205. package/src/worker-sizing.ts +76 -0
  206. package/src/cli/schedule-logs.ts +0 -36
package/bin/devfleet.mjs CHANGED
@@ -3,21 +3,59 @@
3
3
  // run by Bun, and Node refuses to run TypeScript from node_modules, so this
4
4
  // plain-JavaScript launcher hands the arguments to Bun and nothing else.
5
5
  import { spawnSync } from 'node:child_process';
6
+ import { closeSync, openSync, readSync } from 'node:fs';
6
7
  import { createRequire } from 'node:module';
8
+ import { Buffer } from 'node:buffer';
7
9
  import process from 'node:process';
8
10
  import { fileURLToPath, URL } from 'node:url';
9
11
 
10
12
  const cli = fileURLToPath(new URL('../src/cli/cli.ts', import.meta.url));
11
13
 
14
+ /** The first bytes of an ELF, Mach-O (thin or universal) or PE executable. */
15
+ const MAGIC = [
16
+ [0x7f, 0x45, 0x4c, 0x46],
17
+ [0xfe, 0xed, 0xfa, 0xce],
18
+ [0xfe, 0xed, 0xfa, 0xcf],
19
+ [0xce, 0xfa, 0xed, 0xfe],
20
+ [0xcf, 0xfa, 0xed, 0xfe],
21
+ [0xca, 0xfe, 0xba, 0xbe],
22
+ [0xca, 0xfe, 0xba, 0xbf],
23
+ [0x4d, 0x5a],
24
+ ];
25
+
26
+ /** Whether the file is a compiled executable rather than a text stub. Linux
27
+ * runs a stub without a shebang through /bin/sh instead of failing to spawn
28
+ * it, so the spawn error cannot tell the two apart.
29
+ */
30
+ function isExecutable(path) {
31
+ const head = Buffer.alloc(4);
32
+ let read = 0;
33
+ try {
34
+ const fd = openSync(path, 'r');
35
+ try {
36
+ read = readSync(fd, head, 0, head.length, 0);
37
+ } finally {
38
+ closeSync(fd);
39
+ }
40
+ } catch {
41
+ return false;
42
+ }
43
+ return MAGIC.some(
44
+ (magic) =>
45
+ magic.length <= read && magic.every((byte, at) => head[at] === byte)
46
+ );
47
+ }
48
+
12
49
  /** The Bun this release pins (a dependency), then any Bun on PATH. The
13
- * dependency's binary is a stub when its install script was skipped.
50
+ * dependency's binary is a stub when its install script was skipped, and is
51
+ * passed over then. The one on PATH may be a version manager's script shim,
52
+ * so it is only tried.
14
53
  */
15
54
  function bunCandidates() {
16
55
  const candidates = ['bun'];
17
56
  try {
18
- candidates.unshift(
19
- createRequire(import.meta.url).resolve('bun/bin/bun.exe')
20
- );
57
+ const pinned = createRequire(import.meta.url).resolve('bun/bin/bun.exe');
58
+ if (isExecutable(pinned)) candidates.unshift(pinned);
21
59
  } catch {
22
60
  // Not installed beside this package.
23
61
  }
@@ -1,8 +1,9 @@
1
1
  #!/bin/sh
2
2
  # Executed over strict SSH from the selected verified Git revision. stdin is
3
3
  # one line holding the base64 fleet configuration (empty when there is none),
4
- # then the base64 Git bundle (nothing in cached mode). All executable content
5
- # remains in this tracked source. The Ubuntu releases workers may run are
4
+ # one line holding a single-use tailnet enrolment key (empty when there is
5
+ # none), then the base64 Git bundle (nothing in cached mode). All executable
6
+ # content remains in this tracked source. The Ubuntu releases workers may run are
6
7
  # named once, in ansible/ubuntu/policy.json; the manager passes them in.
7
8
  set -eu
8
9
  umask 077
@@ -11,7 +12,11 @@ bundle_sha=$2
11
12
  node=$3
12
13
  source_mode=${4:-transfer}
13
14
  releases=${5:-}
15
+ # Whether a worker's T3 restarts while agent turns run (fleet-sync
16
+ # --restart-busy); an older manager passes nothing and turns are kept.
17
+ t3_restart=${6:-keep-turns}
14
18
  case "$source_mode" in cached|transfer) ;; *) exit 2;; esac
19
+ case "$t3_restart" in keep-turns|restart-busy) ;; *) exit 2;; esac
15
20
  case "$releases" in *[!0-9.\ ]*|'') exit 2;; esac
16
21
  case "$revision" in *[!a-f0-9]*|'') exit 2;; esac
17
22
  [ "${#revision}" = 40 ] || exit 2
@@ -23,12 +28,17 @@ bundle=''
23
28
  code=0
24
29
  apt_log=''
25
30
  failure_code=REMOTE_BOOTSTRAP_INCOMPLETE
31
+ # How long a machine's own first boot (cloud-init) may still run, in seconds.
32
+ first_boot_wait=900
26
33
  die() { failure_code=$1; exit 1; }
27
34
  # One remedy per failure: what the operator does next. Never raw output.
28
35
  remedy() {
29
36
  case "$1" in
30
37
  FLEET_REMOTE_LOCKED) echo 'Another fleet operation holds this machine. Wait for it to finish; rerun fleet-sync afterwards. Never remove the lock while its holder runs.';;
31
- ROOT_DEVELOPMENT_USER_REJECTED) echo 'Connect as the non-root developer account the fleet configuration names.';;
38
+ ROOT_DEVELOPMENT_USER_REJECTED) echo 'Connect as the non-root developer account the fleet configuration names (workers[].network.user). With that account declared, fleet-sync creates it once over the root login (ansible/ubuntu/worker-account.yml) and continues as it.';;
39
+ FIRST_BOOT_PENDING) echo 'The machine is still finishing its first boot (cloud-init). Nothing was changed; rerun fleet-sync in a few minutes.';;
40
+ FIRST_BOOT_FAILED) echo 'The first boot of the machine (cloud-init) ended in an error, so it is half-provisioned and was not converged. Read cloud-init status --long on the worker, fix the image or its user data, and recreate the machine.';;
41
+ TAILNET_KEY_INVALID|TAILNET_KEY_DELIVERY_FAILED) echo 'The single-use tailnet key could not be handed to the worker. Rerun fleet-sync; it mints a new one. The worker needs passwordless sudo to receive it.';;
32
42
  UBUNTU_BASELINE_MISMATCH|UBUNTU_ARCHITECTURE_UNSUPPORTED|PLATFORM_UNSUPPORTED) echo 'Workers run an Ubuntu release docs/support-matrix.md lists, on x86_64 or aarch64. Reinstall the machine; fleet-sync never upgrades an operating system.';;
33
43
  WORKER_ENVIRONMENT_UNSUPPORTED) echo 'WSL2 is not a worker. Run the worker on a physical or virtual machine of its own (docs/support-matrix.md).';;
34
44
  FLEET_STATE_PATH_UNSAFE) echo 'A fleet state directory under ~/.local/share/personal is a symbolic link. Replace it with a real directory.';;
@@ -39,7 +49,7 @@ remedy() {
39
49
  APT_DNS_UNAVAILABLE) echo 'The worker cannot resolve the Ubuntu archive. Restore its DNS, then rerun fleet-sync.';;
40
50
  APT_REPOSITORY_UNAVAILABLE) echo 'The Ubuntu archive could not be reached. Check the network or archive status, then rerun fleet-sync.';;
41
51
  APT_DISK_FULL) echo 'Package installation ran out of disk space. Free space under /var and /usr, then rerun fleet-sync.';;
42
- DPKG_INTERRUPTED) echo 'An earlier package installation that fleet-sync did not start was interrupted. Run sudo dpkg --configure -a once no package process is running, then rerun fleet-sync.';;
52
+ DPKG_INTERRUPTED) echo 'An interrupted package installation could not be finished: dpkg --configure -a, run once and bounded, left the package database inconsistent. Read sudo dpkg --audit on the worker, repair the packages it names, then rerun fleet-sync.';;
43
53
  APT_UPDATE_FAILED|APT_INSTALL_FAILED) echo 'Run sudo apt-get update and install the bootstrap packages by hand to see the error, then rerun fleet-sync.';;
44
54
  SOURCE_CACHE_CHANGED|SOURCE_STATUS_FAILED|SOURCE_CHECKOUT_CONFLICT) echo 'The fleet-owned checkout of this revision holds changes fleet-sync did not make. Inspect and remove that revisions directory, then rerun fleet-sync.';;
45
55
  CONFIGURATION_INSTALL_FAILED) echo 'The fleet configuration could not be written under ~/.config/dev-fleet. Check that directory, then rerun fleet-sync.';;
@@ -47,9 +57,12 @@ remedy() {
47
57
  *) echo 'Inspect the remote operation lock and rerun the approved revision. No success receipt advanced.';;
48
58
  esac
49
59
  }
60
+ # apt-failure: begin
50
61
  # Names an APT failure by the fixed messages of the pinned Ubuntu release.
51
62
  apt_failure() {
52
- if grep -q 'Could not get lock' "$apt_log" 2>/dev/null; then die APT_LOCK_CONTENDED
63
+ # APT names a held lock in several ways: with the process that holds it,
64
+ # without one (it could not read the holder), or in dpkg's own words.
65
+ if grep -qE 'Could not get lock|Unable to acquire the dpkg frontend lock|Unable to lock the administration directory|locked by another process' "$apt_log" 2>/dev/null; then die APT_LOCK_CONTENDED
53
66
  elif grep -q 'No space left on device' "$apt_log" 2>/dev/null; then die APT_DISK_FULL
54
67
  elif grep -qE 'Temporary failure resolving|Could not resolve' "$apt_log" 2>/dev/null; then die APT_DNS_UNAVAILABLE
55
68
  elif grep -qE 'Failed to fetch|Unable to connect|Connection failed|Unable to locate package' "$apt_log" 2>/dev/null; then die APT_REPOSITORY_UNAVAILABLE
@@ -57,6 +70,7 @@ apt_failure() {
57
70
  else die "$1"
58
71
  fi
59
72
  }
73
+ # apt-failure: end
60
74
  fail() {
61
75
  printf '{"version":1,"checks":[{"id":"bootstrap:%s","status":"fail","summary":"%s","remedy":{"action":"%s"}}]}\n' "$phase" "$failure_code" "$(remedy "$failure_code")"
62
76
  }
@@ -101,12 +115,32 @@ operation lock
101
115
  # runs never write it at once. It holds no secret.
102
116
  phase='configuration-install'
103
117
  IFS= read -r configuration || configuration=''
118
+ # Held in this shell only (never exported, never an argument) until it is
119
+ # written to a root-only file for the Tailscale play.
120
+ IFS= read -r tailnet_key || tailnet_key=''
104
121
  if [ -n "$configuration" ]; then
105
122
  config_dir="$HOME/.config/dev-fleet"
106
123
  mkdir -p "$config_dir" || die CONFIGURATION_INSTALL_FAILED
107
124
  printf '%s' "$configuration" | base64 -d >"$config_dir/fleet.json.new" || die CONFIGURATION_INSTALL_FAILED
108
125
  mv "$config_dir/fleet.json.new" "$config_dir/fleet.json" || die CONFIGURATION_INSTALL_FAILED
109
126
  fi
127
+ phase=first-boot
128
+ # first-boot: begin
129
+ # A new machine answers SSH before its own first boot has finished: cloud-init
130
+ # may still be growing the disk and installing packages. Wait for it, within a
131
+ # bound, before any package work. A machine without cloud-init has nothing to
132
+ # wait for. Exit 2 is a first boot that finished with recoverable errors.
133
+ if command -v cloud-init >/dev/null 2>&1; then
134
+ operation first-boot
135
+ first_boot=0
136
+ timeout "$first_boot_wait" cloud-init status --wait >/dev/null 2>&1 || first_boot=$?
137
+ case "$first_boot" in
138
+ 0|2) ;;
139
+ 124) die FIRST_BOOT_PENDING;;
140
+ *) die FIRST_BOOT_FAILED;;
141
+ esac
142
+ fi
143
+ # first-boot: end
110
144
  phase=resources
111
145
  operation resources
112
146
  # A bootstrap floor protects source transfer and minimal prerequisites. The
@@ -123,37 +157,60 @@ base64 -d >"$bundle"
123
157
  printf '%s %s\n' "$bundle_sha" "$bundle" | sha256sum -c - >/dev/null 2>&1 || die SOURCE_DIGEST_MISMATCH
124
158
  fi
125
159
  phase=prerequisites
126
- missing=''
127
- for package in git curl ca-certificates unzip xz-utils build-essential python3 ansible-core; do
128
- dpkg-query -W -f='${db:Status-Status}' "$package" 2>/dev/null | grep -qx installed || missing="$missing $package"
129
- done
160
+ # prerequisites: begin
161
+ missing_prerequisites() {
162
+ missing=''
163
+ for package in git curl ca-certificates unzip xz-utils build-essential python3 ansible-core; do
164
+ dpkg-query -W -f='${db:Status-Status}' "$package" 2>/dev/null | grep -qx installed || missing="$missing $package"
165
+ done
166
+ }
167
+ missing_prerequisites
130
168
  if [ -n "$missing" ]; then
131
169
  sudo -n true >/dev/null 2>&1 || die SUDO_AUTHORIZATION_REQUIRED
132
170
  # APT's own messages, in a private file read only for fixed patterns and
133
171
  # removed on exit: they tell a held lock, DNS, an unreachable archive and a
134
172
  # full disk apart. Nothing of them is reported.
135
173
  apt_log=$(mktemp "$root/apt.XXXXXX")
136
- pending="$root/prerequisites-pending"
137
- # An installation this script started and that was interrupted (the shell
138
- # killed, the machine powered off) leaves dpkg half-configured. Finishing
139
- # our own transaction is safe; one fleet-sync did not start is reported.
174
+ # A package run that was cut off (the shell killed, the machine powered
175
+ # off) leaves dpkg half-configured, whoever started it: this script, the
176
+ # Ubuntu baseline upgrading one of these very packages, or the release's
177
+ # own updates. Finishing it is the standard safe repair, the one the
178
+ # baseline makes (ansible/ubuntu/tasks/dpkg.yml): tried once, bounded,
179
+ # waiting while another package process holds the lock (never killing it).
180
+ # Only a database still inconsistent afterwards stops the run.
140
181
  # dpkg reads its database lock even to audit, which needs root.
141
182
  if [ -n "$(sudo -n dpkg --audit 2>/dev/null)" ]; then
142
- [ -f "$pending" ] || die DPKG_INTERRUPTED
183
+ lock_waits=0
143
184
  # shellcheck disable=SC2024
144
- sudo -n timeout 600 env DEBIAN_FRONTEND=noninteractive LC_ALL=C dpkg --configure -a >"$apt_log" 2>&1 || apt_failure APT_INSTALL_FAILED
185
+ until sudo -n timeout 600 env DEBIAN_FRONTEND=noninteractive LC_ALL=C dpkg --configure -a >"$apt_log" 2>&1; do
186
+ grep -q 'locked by another process' "$apt_log" 2>/dev/null && [ "$lock_waits" -lt 24 ] || break
187
+ lock_waits=$((lock_waits + 1))
188
+ sleep 5
189
+ done
190
+ if [ -n "$(sudo -n dpkg --audit 2>/dev/null)" ]; then
191
+ ! grep -q 'locked by another process' "$apt_log" 2>/dev/null || die APT_LOCK_CONTENDED
192
+ ! grep -q 'No space left on device' "$apt_log" 2>/dev/null || die APT_DISK_FULL
193
+ die DPKG_INTERRUPTED
194
+ fi
195
+ # The repair finishes the prerequisite a cut-off run was upgrading.
196
+ missing_prerequisites
145
197
  fi
146
- : >"$pending"
147
- # The log belongs to the developer account; sudo only runs the command.
148
- # shellcheck disable=SC2024
149
- # Never remove APT locks or kill package processes. Timeout keeps genuine
150
- # transactions bounded; Ubuntu archive signatures/TLS remain enabled.
151
- sudo -n timeout 300 env LC_ALL=C apt-get -o DPkg::Lock::Timeout=120 update >"$apt_log" 2>&1 || apt_failure APT_UPDATE_FAILED
152
- # shellcheck disable=SC2086,SC2024
153
- sudo -n timeout 600 env DEBIAN_FRONTEND=noninteractive LC_ALL=C apt-get -o DPkg::Lock::Timeout=120 --no-install-recommends install -y $missing >>"$apt_log" 2>&1 || apt_failure APT_INSTALL_FAILED
154
- rm -f -- "$pending" "$apt_log"
198
+ # Earlier revisions marked their own installation here; the repair above
199
+ # no longer asks who started a run, so the marker is only removed.
200
+ rm -f -- "$root/prerequisites-pending"
201
+ if [ -n "$missing" ]; then
202
+ # The log belongs to the developer account; sudo only runs the command.
203
+ # Never remove APT locks or kill package processes. Timeout keeps genuine
204
+ # transactions bounded; Ubuntu archive signatures/TLS remain enabled.
205
+ # shellcheck disable=SC2024
206
+ sudo -n timeout 300 env LC_ALL=C apt-get -o DPkg::Lock::Timeout=120 update >"$apt_log" 2>&1 || apt_failure APT_UPDATE_FAILED
207
+ # shellcheck disable=SC2086,SC2024
208
+ sudo -n timeout 600 env DEBIAN_FRONTEND=noninteractive LC_ALL=C apt-get -o DPkg::Lock::Timeout=120 --no-install-recommends install -y $missing >>"$apt_log" 2>&1 || apt_failure APT_INSTALL_FAILED
209
+ fi
210
+ rm -f -- "$apt_log"
155
211
  apt_log=''
156
212
  fi
213
+ # prerequisites: end
157
214
  phase=source-verification
158
215
  deployment="$root/revisions/$revision"
159
216
  [ ! -L "$root/revisions" ] && [ ! -L "$deployment" ] || exit 1
@@ -248,6 +305,15 @@ for old in $(ls -1t "$root/revisions"); do
248
305
  [ "$kept" -gt 2 ] || continue
249
306
  rm -rf -- "${root:?}/revisions/$old" || true
250
307
  done
308
+ phase=tailnet-key
309
+ # The key the manager minted for a worker that is not on the tailnet yet: a
310
+ # root-only file tailscale.yml joins with and deletes. sudo reads it from
311
+ # stdin, so it never appears in a process's arguments.
312
+ if [ -n "$tailnet_key" ]; then
313
+ case "$tailnet_key" in *[!A-Za-z0-9_-]*) die TAILNET_KEY_INVALID;; esac
314
+ printf '%s\n' "$tailnet_key" | sudo -n sh -c 'umask 077 && mkdir -p /run/devfleet && cat >/run/devfleet/tailnet-auth-key' || die TAILNET_KEY_DELIVERY_FAILED
315
+ fi
316
+ tailnet_key=''
251
317
  phase=configuration
252
318
  operation configuration
253
319
  trap - EXIT INT TERM
@@ -255,7 +321,7 @@ trap - EXIT INT TERM
255
321
  # session: the session may be gone by the time it ends. The evidence is then
256
322
  # printed for the manager that is still connected, under the same lock.
257
323
  code=0
258
- mise exec --locked bun -- bun src/node-converge.ts "$node" "$revision" >"$result.partial" 2>"$root/remote-operation.log" || code=$?
324
+ mise exec --locked bun -- bun src/node-converge.ts "$node" "$revision" "$t3_restart" >"$result.partial" 2>"$root/remote-operation.log" || code=$?
259
325
  mv -f -- "$result.partial" "$result"
260
326
  operation finished ",\"finished\":$(date +%s),\"exit\":$code"
261
327
  cat -- "$result" || true
@@ -53,9 +53,11 @@ export function assertManagedConfig(dir: string): void {
53
53
  function publishedConfig(root: string, previous: string | undefined): string {
54
54
  return (
55
55
  MARKER +
56
- withTrustedPaths(
57
- readFileSync(join(root, 'mise.toml'), 'utf8'),
58
- trustedPaths(previous)
56
+ withProjectLocks(
57
+ withTrustedPaths(
58
+ readFileSync(join(root, 'mise.toml'), 'utf8'),
59
+ trustedPaths(previous)
60
+ )
59
61
  )
60
62
  );
61
63
  }
@@ -162,6 +164,30 @@ export function withTrustedPaths(config: string, paths: string[]): string {
162
164
  : `${config.replace(/\n*$/, '\n')}\n[settings]\n${line}\n`;
163
165
  }
164
166
 
167
+ /**
168
+ * A project adds the tools dev-fleet does not ship in its own mise.toml. On a
169
+ * fleet machine mise installs those only from the project's mise.lock and
170
+ * refuses a tool the lock does not name; the fleet's own tools are installed
171
+ * with --locked anyway. mise honours `locked_scopes` only in the global
172
+ * config, so both settings are added at publish time.
173
+ */
174
+ export const PROJECT_LOCK_SETTINGS = [
175
+ 'locked = true',
176
+ 'locked_scopes = ["project"]',
177
+ ];
178
+
179
+ /** Adds the project lock settings to the [settings] table withTrustedPaths made. */
180
+ export function withProjectLocks(config: string): string {
181
+ if (/^locked(_scopes)?\s*=/m.test(config))
182
+ throw new Error(
183
+ 'mise.toml sets locked or locked_scopes; the global config adds them for projects.'
184
+ );
185
+ return config.replace(
186
+ /^\[settings\][^\S\n]*\n/m,
187
+ (match) => `${match}${PROJECT_LOCK_SETTINGS.join('\n')}\n`
188
+ );
189
+ }
190
+
165
191
  function changedFile(source: string, destination: string): boolean {
166
192
  return (
167
193
  statSync(source).isDirectory() ||
@@ -0,0 +1,127 @@
1
+ /**
2
+ * Installs a packed release the way a fleet project does and runs it, before
3
+ * the Release workflow publishes that same tarball. A package whose `devfleet`
4
+ * cannot start (a broken bin path, a file left out of `files`, a launcher that
5
+ * fails on this Node) fails here, never on npm.
6
+ *
7
+ * bun bootstrap/package-smoke.ts <tarball> <version> <commit>
8
+ *
9
+ * It installs twice: as npm does by default, and with `--ignore-scripts` as
10
+ * pnpm and cautious projects do, which leaves the pinned Bun a text stub and
11
+ * makes the launcher fall back to the Bun on PATH. Each runs in an empty
12
+ * directory, outside any project, so it reads no fleet.json and contacts no
13
+ * machine.
14
+ */
15
+ import { spawnSync } from 'node:child_process';
16
+ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
17
+ import { tmpdir } from 'node:os';
18
+ import { join, resolve } from 'node:path';
19
+
20
+ export interface Ran {
21
+ status: number | null;
22
+ stdout: string;
23
+ stderr: string;
24
+ }
25
+
26
+ export interface Expected {
27
+ version: string;
28
+ /** The commit the release workflow stamped as `gitHead`. */
29
+ commit: string;
30
+ }
31
+
32
+ function run(command: string[], cwd: string): Ran {
33
+ const result = spawnSync(command[0]!, command.slice(1), {
34
+ cwd,
35
+ encoding: 'utf8',
36
+ env: { ...process.env, DEVFLEET_CONFIG: '' },
37
+ });
38
+ return {
39
+ status: result.error ? null : result.status,
40
+ stdout: result.stdout ?? '',
41
+ stderr: result.stderr ?? '',
42
+ };
43
+ }
44
+
45
+ function must(ran: Ran, what: string): string {
46
+ if (ran.status !== 0)
47
+ throw new Error(
48
+ `${what} failed (exit ${ran.status ?? 'not started'}): ${ran.stderr.trim().split('\n').slice(-3).join(' ')}`
49
+ );
50
+ return ran.stdout;
51
+ }
52
+
53
+ /** Install the tarball in a new directory, with these npm flags, and run its
54
+ * `devfleet`.
55
+ */
56
+ function installAndRun(
57
+ tarball: string,
58
+ expected: Expected,
59
+ flags: string[]
60
+ ): void {
61
+ const directory = mkdtempSync(join(tmpdir(), 'devfleet-smoke-'));
62
+ const how = flags.length ? ` (installed with ${flags.join(' ')})` : '';
63
+ try {
64
+ writeFileSync(
65
+ join(directory, 'package.json'),
66
+ JSON.stringify({ name: 'devfleet-smoke', private: true })
67
+ );
68
+ must(
69
+ run(
70
+ // Into the throwaway project only: nothing on the machine changes.
71
+ [
72
+ 'npm',
73
+ '--prefix',
74
+ directory,
75
+ 'install',
76
+ '--no-audit',
77
+ '--no-fund',
78
+ '--save-exact',
79
+ ...flags,
80
+ resolve(tarball),
81
+ ],
82
+ directory
83
+ ),
84
+ `npm install${how}`
85
+ );
86
+ const devfleet = join(directory, 'node_modules', '.bin', 'devfleet');
87
+ const version = must(
88
+ run([devfleet, '--version'], directory),
89
+ `devfleet --version${how}`
90
+ );
91
+ if (!version.includes(expected.version))
92
+ throw new Error(
93
+ `devfleet --version printed "${version.trim()}", not ${expected.version}${how}.`
94
+ );
95
+ if (!version.includes(expected.commit.slice(0, 7)))
96
+ throw new Error(
97
+ `devfleet --version names no commit ${expected.commit.slice(0, 7)}${how}.`
98
+ );
99
+ const example = must(
100
+ run([devfleet, 'fleet-state', '--example'], directory),
101
+ `devfleet fleet-state --example${how}`
102
+ );
103
+ const parsed = JSON.parse(example) as { $schema?: unknown };
104
+ if (typeof parsed.$schema !== 'string')
105
+ throw new Error(`The example fleet.json names no schema${how}.`);
106
+ } finally {
107
+ rmSync(directory, { recursive: true, force: true });
108
+ }
109
+ }
110
+
111
+ /** Install the tarball with and without install scripts and run it each time. */
112
+ export function smoke(tarball: string, expected: Expected): void {
113
+ installAndRun(tarball, expected, []);
114
+ installAndRun(tarball, expected, ['--ignore-scripts']);
115
+ }
116
+
117
+ if (import.meta.main) {
118
+ const [tarball, version, commit] = process.argv.slice(2);
119
+ if (!tarball || !version || !commit) {
120
+ console.error(
121
+ 'Usage: bun bootstrap/package-smoke.ts <tarball> <version> <commit>'
122
+ );
123
+ process.exit(2);
124
+ }
125
+ smoke(tarball, { version, commit });
126
+ console.log(`${tarball} installs and runs: devfleet ${version}`);
127
+ }
@@ -0,0 +1,189 @@
1
+ /**
2
+ * Moves a pin that carries digests to its newest upstream release, digests
3
+ * included, so an update is one reviewed pull request and never a manual
4
+ * batch. Renovate cannot do this: it bumps a version but cannot compute the
5
+ * checksums beside it. The Pins workflow runs this on a schedule and opens a
6
+ * pull request when the file changed.
7
+ *
8
+ * bun bootstrap/pins.ts mise # min_version and mise_*_sha256 in mise.toml
9
+ * bun bootstrap/pins.ts t3 # t3_version and t3_*_sha256 in mise.toml
10
+ *
11
+ * Prints `<pin> <from> -> <to>`, or `<pin> <version> is current`.
12
+ */
13
+ import { createHash } from 'node:crypto';
14
+ import { readFileSync, writeFileSync } from 'node:fs';
15
+ import { join } from 'node:path';
16
+
17
+ /** Where upstream releases are read. Injected by the features. */
18
+ export interface Upstream {
19
+ /** Release tags of a GitHub repository, newest first. */
20
+ tags(repository: string): Promise<string[]>;
21
+ /** The body of a file at a URL. */
22
+ text(url: string): Promise<string>;
23
+ /** The SHA-256 of a file at a URL. */
24
+ sha256(url: string): Promise<string>;
25
+ }
26
+
27
+ export interface Change {
28
+ pin: string;
29
+ from: string;
30
+ to: string;
31
+ /** The new text of mise.toml. */
32
+ text: string;
33
+ }
34
+
35
+ const MISE_PLATFORMS = [
36
+ 'linux_x64',
37
+ 'linux_arm64',
38
+ 'macos_x64',
39
+ 'macos_arm64',
40
+ ] as const;
41
+ const T3_NIGHTLY = /^v(\d+\.\d+\.\d+-nightly\.\d{8}\.\d+)$/;
42
+ const MISE_RELEASE = /^v(\d{4}\.\d+\.\d+)$/;
43
+
44
+ function value(text: string, name: string): string {
45
+ const found = new RegExp(`^${name} = "(.*)"$`, 'm').exec(text)?.[1];
46
+ if (found === undefined) throw new Error(`mise.toml has no ${name}.`);
47
+ return found;
48
+ }
49
+
50
+ function replace(text: string, name: string, next: string): string {
51
+ value(text, name);
52
+ return text.replace(
53
+ new RegExp(`^${name} = ".*"$`, 'm'),
54
+ `${name} = "${next}"`
55
+ );
56
+ }
57
+
58
+ function digest(hex: string, what: string): string {
59
+ if (!/^[a-f0-9]{64}$/.test(hex))
60
+ throw new Error(`Upstream gave no SHA-256 for ${what}.`);
61
+ return hex;
62
+ }
63
+
64
+ async function mise(text: string, upstream: Upstream): Promise<Change> {
65
+ const from = value(text, 'min_version');
66
+ const tag = (await upstream.tags('jdx/mise')).find((candidate) =>
67
+ MISE_RELEASE.test(candidate)
68
+ );
69
+ if (!tag) throw new Error('jdx/mise has no release.');
70
+ const to = MISE_RELEASE.exec(tag)![1]!;
71
+ if (to === from) return { pin: 'mise', from, to, text };
72
+ // The digests mise publishes for its own archives, the ones
73
+ // bootstrap/bootstrap.sh downloads.
74
+ const sums = await upstream.text(
75
+ `https://github.com/jdx/mise/releases/download/${tag}/SHASUMS256.txt`
76
+ );
77
+ let next = replace(text, 'min_version', to);
78
+ for (const platform of MISE_PLATFORMS) {
79
+ const asset = `mise-v${to}-${platform.replace('_', '-')}.tar.gz`;
80
+ const line = sums
81
+ .split('\n')
82
+ .map((entry) => entry.trim().split(/\s+/))
83
+ .find(([, name]) => name?.replace(/^\.\//, '') === asset);
84
+ next = replace(
85
+ next,
86
+ `mise_${platform}_sha256`,
87
+ digest(line?.[0] ?? '', asset)
88
+ );
89
+ }
90
+ return { pin: 'mise', from, to, text: next };
91
+ }
92
+
93
+ async function t3(text: string, upstream: Upstream): Promise<Change> {
94
+ const from = value(text, 't3_version');
95
+ const tag = (await upstream.tags('pingdotgg/t3code')).find((candidate) =>
96
+ T3_NIGHTLY.test(candidate)
97
+ );
98
+ if (!tag) throw new Error('pingdotgg/t3code has no nightly release.');
99
+ const to = T3_NIGHTLY.exec(tag)![1]!;
100
+ if (to === from) return { pin: 't3', from, to, text };
101
+ const base = `https://github.com/pingdotgg/t3code/releases/download/${tag}`;
102
+ // Upstream's SHA256SUMS omits the desktop packages: every artifact is
103
+ // hashed from the release itself, the way bootstrap/native-t3.ts and
104
+ // ansible/ubuntu/t3-desktop.yml check it.
105
+ const artifacts: [string, string][] = [
106
+ ['t3_linux_x64_sha256', `t3-${to}-linux-x64.tar.gz`],
107
+ ['t3_linux_arm64_sha256', `t3-${to}-linux-arm64.tar.gz`],
108
+ // The manager's own T3: without it the manager refuses the new release.
109
+ ['t3_darwin_arm64_sha256', `t3-${to}-darwin-arm64.tar.gz`],
110
+ ['t3_desktop_amd64_sha256', `T3-Code-${to}-amd64.deb`],
111
+ ['t3_desktop_arm64_sha256', `T3-Code-${to}-arm64.deb`],
112
+ ];
113
+ let next = replace(text, 't3_version', to);
114
+ for (const [name, asset] of artifacts)
115
+ next = replace(
116
+ next,
117
+ name,
118
+ digest(await upstream.sha256(`${base}/${asset}`), asset)
119
+ );
120
+ return { pin: 't3', from, to, text: next };
121
+ }
122
+
123
+ export const PINS: Record<
124
+ string,
125
+ (text: string, upstream: Upstream) => Promise<Change>
126
+ > = { mise, t3 };
127
+
128
+ /** The newest release of one pin, as the new text of mise.toml. */
129
+ export async function update(
130
+ pin: string,
131
+ text: string,
132
+ upstream: Upstream
133
+ ): Promise<Change> {
134
+ const updater = PINS[pin];
135
+ if (!updater)
136
+ throw new Error(
137
+ `Unknown pin ${pin}: one of ${Object.keys(PINS).join(', ')}.`
138
+ );
139
+ return updater(text, upstream);
140
+ }
141
+
142
+ /** GitHub and HTTPS, as the Pins workflow reaches them. */
143
+ export function githubUpstream(token = process.env['GH_TOKEN']): Upstream {
144
+ const fetchOk = async (url: string, headers: Record<string, string> = {}) => {
145
+ const response = await fetch(url, { headers, redirect: 'follow' });
146
+ if (!response.ok) throw new Error(`${url} answered ${response.status}.`);
147
+ return response;
148
+ };
149
+ return {
150
+ async tags(repository) {
151
+ const response = await fetchOk(
152
+ `https://api.github.com/repos/${repository}/releases?per_page=100`,
153
+ {
154
+ accept: 'application/vnd.github+json',
155
+ ...(token ? { authorization: `Bearer ${token}` } : {}),
156
+ }
157
+ );
158
+ const releases = (await response.json()) as {
159
+ tag_name: string;
160
+ draft: boolean;
161
+ }[];
162
+ return releases
163
+ .filter((release) => !release.draft)
164
+ .map((release) => release.tag_name);
165
+ },
166
+ async text(url) {
167
+ return (await fetchOk(url)).text();
168
+ },
169
+ async sha256(url) {
170
+ const bytes = await (await fetchOk(url)).arrayBuffer();
171
+ return createHash('sha256').update(new Uint8Array(bytes)).digest('hex');
172
+ },
173
+ };
174
+ }
175
+
176
+ if (import.meta.main) {
177
+ const pin = process.argv[2] ?? '';
178
+ const path = join(import.meta.dir, '..', 'mise.toml');
179
+ const change = await update(
180
+ pin,
181
+ readFileSync(path, 'utf8'),
182
+ githubUpstream()
183
+ );
184
+ if (change.from === change.to) console.log(`${pin} ${change.to} is current`);
185
+ else {
186
+ writeFileSync(path, change.text);
187
+ console.log(`${pin} ${change.from} -> ${change.to}`);
188
+ }
189
+ }