@crvouga/dev-fleet 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/README.md +265 -37
  2. package/agents/setup.md +42 -12
  3. package/ansible/macos/manager.yml +10 -7
  4. package/ansible/macos/policy.json +2 -10
  5. package/ansible/macos/t3-policy.yml +49 -0
  6. package/ansible/monitoring/README.md +86 -25
  7. package/ansible/monitoring/agent.yml +40 -2
  8. package/ansible/monitoring/hub-route-darwin.yml +51 -0
  9. package/ansible/monitoring/lock.json +5 -0
  10. package/ansible/pyproject.toml +12 -0
  11. package/ansible/ubuntu/baseline.yml +1 -1
  12. package/ansible/ubuntu/disk-gc.py +336 -0
  13. package/ansible/ubuntu/policy.json +13 -3
  14. package/ansible/ubuntu/t3-desktop.yml +19 -0
  15. package/ansible/ubuntu/t3-health.sh +93 -22
  16. package/ansible/ubuntu/t3-idle.py +42 -0
  17. package/ansible/ubuntu/t3-policy.yml +4 -11
  18. package/ansible/ubuntu/tailscale.yml +8 -5
  19. package/ansible/ubuntu/tasks/recovery.yml +60 -1
  20. package/ansible/ubuntu/tasks/resources.yml +61 -1
  21. package/ansible/ubuntu/tasks/t3-restart.yml +37 -0
  22. package/ansible/ubuntu/wip-guard.py +121 -0
  23. package/ansible/uv.lock +190 -0
  24. package/bootstrap/bootstrap.sh +25 -18
  25. package/bootstrap/doctor.ts +46 -1
  26. package/bootstrap/fleet-manager-run.sh +42 -15
  27. package/bootstrap/fleet-remote-bootstrap.sh +85 -12
  28. package/bootstrap/global-config.ts +64 -6
  29. package/bootstrap/install-launcher.sh +3 -0
  30. package/bootstrap/native-t3.ts +22 -11
  31. package/bootstrap/npm.ts +10 -2
  32. package/bootstrap/release-version.ts +157 -0
  33. package/cliproxy/README.md +3 -3
  34. package/cliproxy/scripts/accounts.py +216 -0
  35. package/cliproxy/scripts/checkpoint.py +29 -1
  36. package/cliproxy/scripts/cli.py +1 -1
  37. package/cliproxy/scripts/clientkeys.py +99 -0
  38. package/cliproxy/scripts/lease.py +80 -0
  39. package/cliproxy/scripts/ops +137 -25
  40. package/cliproxy/scripts/restore-verify +15 -0
  41. package/docs/accounts.md +154 -0
  42. package/docs/consumer-setup.md +408 -0
  43. package/docs/declared.md +91 -0
  44. package/docs/durability.md +26 -0
  45. package/docs/machine-setup.md +400 -0
  46. package/docs/operations.md +276 -0
  47. package/docs/releasing.md +103 -0
  48. package/docs/state.md +254 -0
  49. package/docs/topology.md +69 -0
  50. package/docs/ubuntu-sync.md +357 -0
  51. package/docs/worker-contract.md +67 -0
  52. package/examples/fleet.json +5 -0
  53. package/llms.txt +36 -0
  54. package/mise.lock +29 -0
  55. package/mise.toml +15 -2
  56. package/package.json +11 -2
  57. package/schema/fleet.schema.json +137 -15
  58. package/src/accounts/checks.ts +220 -0
  59. package/src/accounts/context.ts +76 -0
  60. package/src/accounts/propagate.ts +62 -0
  61. package/src/accounts/roster.ts +85 -0
  62. package/src/accounts/selection.ts +67 -0
  63. package/src/accounts/sessions.ts +120 -0
  64. package/src/agents/account-relay.ts +176 -26
  65. package/src/agents/fleet-verify.ts +9 -0
  66. package/src/agents/launch.ts +84 -5
  67. package/src/agents/native-verify.ts +293 -22
  68. package/src/agents/provider-errors.ts +9 -2
  69. package/src/agents/registry.ts +7 -1
  70. package/src/cli/agent-guide.ts +39 -5
  71. package/src/cli/app.ts +11 -4
  72. package/src/cli/cli.ts +7 -0
  73. package/src/cli/command-index.ts +7 -0
  74. package/src/cli/fleet-accounts.ts +253 -0
  75. package/src/cli/fleet-state.ts +499 -0
  76. package/src/cli/fleet-sync-accounts.ts +58 -0
  77. package/src/cli/fleet-sync-checkout.ts +56 -17
  78. package/src/cli/fleet-sync-enrollment.ts +19 -23
  79. package/src/cli/fleet-sync-forward.ts +1 -1
  80. package/src/cli/fleet-sync-manager.ts +84 -12
  81. package/src/cli/fleet-sync-monitoring.ts +5 -5
  82. package/src/cli/fleet-sync-output.ts +7 -3
  83. package/src/cli/fleet-sync-remote.ts +31 -18
  84. package/src/cli/fleet-sync-state.ts +86 -0
  85. package/src/cli/fleet-sync.ts +159 -57
  86. package/src/cli/fleet.ts +4 -2
  87. package/src/cli/managed-health.ts +4 -4
  88. package/src/cli/node-sync.ts +7 -0
  89. package/src/cli/program.ts +20 -4
  90. package/src/cli/project.ts +51 -0
  91. package/src/cli/register-fleet.ts +86 -1
  92. package/src/cli/registry.ts +41 -1
  93. package/src/cli/roster.ts +46 -30
  94. package/src/cli/settings.ts +15 -12
  95. package/src/cli/source-operations.ts +27 -3
  96. package/src/cli/tailnet.ts +2 -2
  97. package/src/definition.ts +116 -4
  98. package/src/diagnostics.ts +1 -1
  99. package/src/fleet-config.ts +105 -19
  100. package/src/ide/definition.ts +86 -0
  101. package/src/ide/ide.ts +43 -0
  102. package/src/ide/index.ts +5 -0
  103. package/src/ide/orchestrate.ts +50 -0
  104. package/src/ide/t3.ts +111 -0
  105. package/src/machine/source.ts +3 -3
  106. package/src/machine/state.ts +15 -27
  107. package/src/manager-sync.ts +9 -18
  108. package/src/monitoring/agent-evidence.ts +146 -0
  109. package/src/monitoring/apply.ts +15 -3
  110. package/src/monitoring/beszel.ts +11 -4
  111. package/src/monitoring/enroll.ts +2 -1
  112. package/src/monitoring/hub-access.ts +4 -3
  113. package/src/monitoring/hub-backup.ts +121 -0
  114. package/src/monitoring/hub-viewer.ts +8 -8
  115. package/src/monitoring/hub.ts +3 -3
  116. package/src/monitoring/manager-hub.ts +30 -14
  117. package/src/node-converge-linux.ts +30 -7
  118. package/src/package-info.ts +45 -0
  119. package/src/platform/ansible.ts +19 -15
  120. package/src/platform/darwin.ts +0 -13
  121. package/src/platform/fallback.ts +0 -8
  122. package/src/platform/linux.ts +0 -10
  123. package/src/platform/resource-verify.ts +8 -4
  124. package/src/platform/t3-policy.ts +33 -6
  125. package/src/platform/t3-service.ts +104 -0
  126. package/src/platform/types.ts +0 -4
  127. package/src/platform/ubuntu-audit.ts +5 -2
  128. package/src/platform/windows.ts +0 -14
  129. package/src/remote-bootstrap.ts +17 -19
  130. package/src/remote-entry.ts +15 -1
  131. package/src/repair-catalog.ts +1 -1
  132. package/src/runner.ts +1 -1
  133. package/src/service-auth.ts +15 -5
  134. package/src/state/bucket.ts +208 -0
  135. package/src/state/check.ts +74 -0
  136. package/src/state/connection.ts +92 -0
  137. package/src/state/enrollment.ts +45 -0
  138. package/src/state/legacy.ts +89 -0
  139. package/src/state/litestream.ts +143 -0
  140. package/src/state/operator.ts +20 -0
  141. package/src/state/paths.ts +40 -0
  142. package/src/state/roster.ts +62 -0
  143. package/src/state/store.ts +169 -0
  144. package/src/state/sync.ts +352 -0
  145. package/src/t3/accounts.ts +7 -1
  146. package/src/t3/models.ts +133 -0
  147. package/src/t3/pairing.ts +31 -0
  148. package/src/t3/profile.ts +203 -0
  149. package/src/t3/version.ts +119 -0
  150. package/src/t3-endpoint.ts +64 -8
  151. package/src/t3-linked-check.ts +24 -2
  152. package/src/toolchain-checks.ts +25 -0
  153. package/src/transport.ts +3 -2
  154. package/src/types.ts +4 -0
  155. package/src/worker-access-checks.ts +56 -3
  156. package/src/worker-host-checks.ts +41 -13
  157. package/src/workstation-checks.ts +1 -1
  158. package/src/index.ts +0 -6
  159. package/src/test-fixtures.ts +0 -36
package/mise.toml CHANGED
@@ -1,5 +1,7 @@
1
1
  # Shared workstation toolchain. Bootstrap installs this globally; projects may override it.
2
- # See bootstrap/README.md. Commit mise.lock whenever changing these pins.
2
+ # Commit mise.lock whenever changing these pins.
3
+ # min_version is also the exact mise bootstrap/bootstrap.sh installs; bump the
4
+ # mise_*_sha256 digests under [vars] with it.
3
5
  min_version = "2026.10.24"
4
6
 
5
7
  [settings]
@@ -17,6 +19,8 @@ atlas = { version = "1.3.0", os = ["linux", "macos"], lazy = true }
17
19
  azure-cli = { version = "2.91.0", lazy = true }
18
20
  # CLI proxy backups on the manager (cliproxy/scripts/install-dependencies).
19
21
  restic = { version = "0.19.1", os = ["linux", "macos"], lazy = true }
22
+ # Replicates the fleet state database to the storage bucket (src/state).
23
+ litestream = { version = "0.5.18", os = ["linux", "macos"], lazy = true }
20
24
  pnpm = "12.16.1"
21
25
  "npm:@yarnpkg/cli-dist" = "4.18.1"
22
26
  deno = "2.9.8"
@@ -77,10 +81,19 @@ GROK_DISABLE_AUTOUPDATER = "1"
77
81
  AGY_CLI_DISABLE_AUTO_UPDATE = "true"
78
82
 
79
83
  [vars]
80
- # Official native Ubuntu server distribution; bootstrap/native-t3.ts owns installation.
84
+ # mise itself, installed by bootstrap/bootstrap.sh before mise can read this
85
+ # file: the version is min_version above, these are its release archives
86
+ # (SHASUMS256.txt of that release).
87
+ mise_linux_x64_sha256 = "9a6c9c55f5c6372d0af70a9979fe6d742aea3793267a929e1738022143c466cc"
88
+ mise_linux_arm64_sha256 = "8c8f5d7f748bf3aef431dc50f0ad776270f36025c788aa21c74e2fabdbbd6d3e"
89
+ mise_macos_x64_sha256 = "83a36355bf6ee7c1660300e12dd94464de5c672dab1b8cc33ec4d19356e7ddac"
90
+ mise_macos_arm64_sha256 = "aa2fc59242db108f97ea9fb2d2e2cb42ff1467fcafe3c91e48d51c841ba67a65"
91
+ # Official native server distribution (Ubuntu workers, and the manager's
92
+ # pinned CLI); bootstrap/native-t3.ts owns installation. From the release's SHA256SUMS.
81
93
  t3_version = "0.1.8-nightly.20261106.4262"
82
94
  t3_linux_x64_sha256 = "5df94ca15c4f68f749fa5540fbb9d5ca879ca8750ac43dd2704f6a18899ed6d5"
83
95
  t3_linux_arm64_sha256 = "0649615c4a294eee40bc753c8668e4a83c5c3620794f2163758f8dc48a39b880"
96
+ t3_darwin_arm64_sha256 = "40c81f1d696e7056c0c43e29bdfd2a154ea6f6bd9129826034256751418de633"
84
97
  # Official T3 Code desktop .deb (same release). Workers run this GUI app as their
85
98
  # only T3 process; it embeds the server. Upstream SHA256SUMS omits .deb files,
86
99
  # so these digests were computed from the release assets.
package/package.json CHANGED
@@ -1,16 +1,21 @@
1
1
  {
2
2
  "name": "@crvouga/dev-fleet",
3
- "version": "0.1.0",
3
+ "version": "0.3.0",
4
4
  "description": "Keeps a set of development machines in sync: one manager and many workers",
5
5
  "license": "MIT",
6
6
  "repository": "github:crvouga/dev-fleet",
7
7
  "homepage": "https://github.com/crvouga/dev-fleet#readme",
8
+ "bugs": {
9
+ "url": "https://github.com/crvouga/dev-fleet/issues"
10
+ },
8
11
  "type": "module",
9
12
  "bin": {
10
13
  "devfleet": "./bin/devfleet.mjs"
11
14
  },
12
15
  "files": [
13
16
  "bin",
17
+ "llms.txt",
18
+ "docs",
14
19
  "src",
15
20
  "agents",
16
21
  "ansible",
@@ -52,6 +57,9 @@
52
57
  "yaml": "^2.8.0"
53
58
  },
54
59
  "devDependencies": {
60
+ "@cucumber/cucumber-expressions": "20.1.0",
61
+ "@cucumber/gherkin": "42.0.1",
62
+ "@cucumber/messages": "34.2.1",
55
63
  "@eslint/js": "^9.25.0",
56
64
  "@types/bun": "^1.3.9",
57
65
  "@types/node": "^22.15.29",
@@ -63,5 +71,6 @@
63
71
  "packageManager": "bun@1.4.4",
64
72
  "engines": {
65
73
  "node": ">=22"
66
- }
74
+ },
75
+ "gitHead": "0c583e40a0c32c0db355e9c0bca6903894945310"
67
76
  }
@@ -27,7 +27,8 @@
27
27
  "properties": {
28
28
  "id": {
29
29
  "type": "string",
30
- "pattern": "^[a-z][a-z0-9-]{0,62}$"
30
+ "pattern": "^[a-z][a-z0-9-]{0,62}$",
31
+ "description": "A short stable id for the manager, used in reports."
31
32
  },
32
33
  "os": {
33
34
  "const": "macOS"
@@ -39,11 +40,13 @@
39
40
  "properties": {
40
41
  "tailscaleName": {
41
42
  "type": "string",
42
- "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$"
43
+ "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$",
44
+ "description": "The manager's Tailscale MagicDNS name, without the trailing dot (tailscale status --json: .Self.DNSName)."
43
45
  },
44
46
  "user": {
45
47
  "type": "string",
46
- "pattern": "^[a-z_][a-z0-9_.-]{0,31}$"
48
+ "pattern": "^[a-z_][a-z0-9_.-]{0,31}$",
49
+ "description": "The manager's login name (id -un)."
47
50
  }
48
51
  }
49
52
  },
@@ -64,11 +67,13 @@
64
67
  "properties": {
65
68
  "id": {
66
69
  "type": "string",
67
- "pattern": "^[a-z][a-z0-9-]{0,62}$"
70
+ "pattern": "^[a-z][a-z0-9-]{0,62}$",
71
+ "description": "A short stable id for the worker. The first worker listed is the canary: a revision new to the fleet must pass there before the others receive it."
68
72
  },
69
73
  "ssh": {
70
74
  "type": "string",
71
- "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$"
75
+ "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$",
76
+ "description": "The worker's Host alias in the manager's ~/.ssh/config. It must work with BatchMode=yes and a host key already in known_hosts."
72
77
  },
73
78
  "os": {
74
79
  "const": "linux",
@@ -81,21 +86,25 @@
81
86
  "properties": {
82
87
  "tailscaleName": {
83
88
  "type": "string",
84
- "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$"
89
+ "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$",
90
+ "description": "The worker's Tailscale MagicDNS name, without the trailing dot."
85
91
  },
86
92
  "user": {
87
93
  "type": "string",
88
- "pattern": "^[a-z_][a-z0-9_-]{0,31}$"
94
+ "pattern": "^[a-z_][a-z0-9_-]{0,31}$",
95
+ "description": "The non-root worker account that can use sudo."
89
96
  },
90
97
  "sshPort": {
91
98
  "type": "integer",
92
99
  "minimum": 1,
93
- "maximum": 65535
100
+ "maximum": 65535,
101
+ "description": "The port sshd listens on, normally 22."
94
102
  },
95
103
  "t3Port": {
96
104
  "type": "integer",
97
105
  "minimum": 1,
98
- "maximum": 65535
106
+ "maximum": 65535,
107
+ "description": "The port the worker T3 server listens on, normally 3773."
99
108
  }
100
109
  }
101
110
  }
@@ -117,6 +126,41 @@
117
126
  "type": "string",
118
127
  "pattern": "^https://[A-Za-z0-9.-]+$",
119
128
  "description": "Public HTTPS origin of the CLI proxy every node uses."
129
+ },
130
+ "accounts": {
131
+ "type": "object",
132
+ "additionalProperties": false,
133
+ "description": "How provider accounts signed in on the proxy are watched and chosen. See docs/accounts.md.",
134
+ "properties": {
135
+ "minimumUsable": {
136
+ "type": "integer",
137
+ "minimum": 1,
138
+ "description": "proxy:account:<provider> warns when a provider has fewer usable accounts than this (default 2) and fails when it has none."
139
+ },
140
+ "selection": {
141
+ "type": "object",
142
+ "description": "Per provider (claude, codex, antigravity): which account an unpinned request goes to first. Accounts cooling down are always skipped.",
143
+ "additionalProperties": {
144
+ "type": "object",
145
+ "additionalProperties": false,
146
+ "required": ["strategy"],
147
+ "properties": {
148
+ "strategy": {
149
+ "enum": [
150
+ "fill-first",
151
+ "round-robin",
152
+ "least-recently-limited"
153
+ ]
154
+ },
155
+ "order": {
156
+ "type": "array",
157
+ "items": { "type": "string" },
158
+ "description": "Account labels (sign-in emails) first, in this order; the others follow in the order of their routing prefix."
159
+ }
160
+ }
161
+ }
162
+ }
163
+ }
120
164
  }
121
165
  }
122
166
  },
@@ -163,7 +207,8 @@
163
207
  "architectures": {
164
208
  "const": ["amd64", "arm64"]
165
209
  }
166
- }
210
+ },
211
+ "description": "The supported worker release and architectures (fixed)."
167
212
  },
168
213
  "worker": {
169
214
  "type": "object",
@@ -191,15 +236,18 @@
191
236
  "workstation",
192
237
  "worker"
193
238
  ]
194
- }
239
+ },
240
+ "description": "Readiness checks every worker must pass; normally all six."
195
241
  },
196
242
  "githubLogin": {
197
243
  "type": "string",
198
- "pattern": "^[a-zA-Z0-9-]{1,39}$"
244
+ "pattern": "^[a-zA-Z0-9-]{1,39}$",
245
+ "description": "The GitHub login every worker must be signed in as."
199
246
  },
200
247
  "vaultUser": {
201
248
  "type": "string",
202
- "pattern": "^[a-zA-Z0-9_.-]{1,64}$"
249
+ "pattern": "^[a-zA-Z0-9_.-]{1,64}$",
250
+ "description": "The Vault user each worker's own token is issued for."
203
251
  },
204
252
  "toolchain": {
205
253
  "const": "mise.lock"
@@ -216,13 +264,14 @@
216
264
  "neverSleep": {
217
265
  "const": true
218
266
  }
219
- }
267
+ },
268
+ "description": "Policy every worker must meet. All but the two logins and the capabilities are fixed."
220
269
  },
221
270
  "source": {
222
271
  "type": "object",
223
272
  "additionalProperties": false,
224
273
  "required": ["repository"],
225
- "description": "The Git repository that holds this fleet's configuration and pins. Nodes only accept source from it.",
274
+ "description": "The Git repository machines take dev-fleet's code from: the one the package you run was published from, or your fork. Nodes only accept source from it.",
226
275
  "properties": {
227
276
  "repository": {
228
277
  "type": "string",
@@ -230,6 +279,79 @@
230
279
  }
231
280
  }
232
281
  },
282
+ "ides": {
283
+ "type": "array",
284
+ "minItems": 1,
285
+ "uniqueItems": true,
286
+ "items": { "enum": ["t3"] },
287
+ "description": "The IDEs every machine runs. Defaults to [\"t3\"]; the README's support matrix lists the supported ones."
288
+ },
289
+ "t3": {
290
+ "type": "object",
291
+ "additionalProperties": false,
292
+ "description": "What every machine's T3 Code holds beyond the provider instances generated from the proxy's accounts.",
293
+ "properties": {
294
+ "profile": {
295
+ "type": "object",
296
+ "description": "Top-level T3 settings every machine holds the same (tool servers, projects, selected preferences). Keys not named here stay local; a key edited on a machine since dev-fleet last wrote it is reported as a conflict, never overwritten.",
297
+ "propertyNames": {
298
+ "pattern": "^[a-zA-Z][a-zA-Z0-9_-]{0,63}$",
299
+ "not": { "const": "providerInstances" }
300
+ }
301
+ },
302
+ "requiredModels": {
303
+ "type": "object",
304
+ "description": "Models T3 must offer, and the proxy must serve, for every account of a tool, by tool (claude, codex, opencode).",
305
+ "propertyNames": { "pattern": "^[a-zA-Z][a-zA-Z0-9_-]{0,63}$" },
306
+ "additionalProperties": {
307
+ "type": "array",
308
+ "items": { "type": "string", "pattern": "^[A-Za-z0-9._:-]{1,128}$" }
309
+ }
310
+ }
311
+ }
312
+ },
313
+ "state": {
314
+ "description": "The bucket that holds the fleet state (the roster, the approved revision and the configuration last applied). Any S3-compatible service, or a directory. Never credentials: a devfleet command reads those from DEVFLEET_BUCKET_ACCESS_KEY_ID and DEVFLEET_BUCKET_SECRET_ACCESS_KEY.",
315
+ "oneOf": [
316
+ {
317
+ "type": "object",
318
+ "additionalProperties": false,
319
+ "required": ["bucket"],
320
+ "properties": {
321
+ "bucket": {
322
+ "type": "string",
323
+ "pattern": "^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"
324
+ },
325
+ "endpoint": {
326
+ "type": "string",
327
+ "description": "The https origin of the S3 service; omit for AWS S3.",
328
+ "pattern": "^https://[A-Za-z0-9.-]{1,253}(:[0-9]{1,5})?$"
329
+ },
330
+ "region": {
331
+ "type": "string",
332
+ "pattern": "^[a-z0-9-]{1,32}$"
333
+ },
334
+ "prefix": {
335
+ "type": "string",
336
+ "description": "Key prefix, so one bucket can hold several fleets. Default: dev-fleet.",
337
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]{0,199}$"
338
+ }
339
+ }
340
+ },
341
+ {
342
+ "type": "object",
343
+ "additionalProperties": false,
344
+ "required": ["directory"],
345
+ "properties": {
346
+ "directory": {
347
+ "type": "string",
348
+ "description": "An absolute path on an attached disk or a mounted share.",
349
+ "pattern": "^/"
350
+ }
351
+ }
352
+ }
353
+ ]
354
+ },
233
355
  "secrets": {
234
356
  "description": "Where this fleet keeps its secrets. env: DEVFLEET_SECRET_<NAME>; file: one file per secret; sops: one encrypted document; vault: a KV v2 location per secret.",
235
357
  "oneOf": [
@@ -0,0 +1,220 @@
1
+ import { existsSync } from 'node:fs';
2
+ import { homedir } from 'node:os';
3
+ import { join } from 'node:path';
4
+ import type { Check } from '../types';
5
+ import type { ProxyRoster, RosterAccount } from './roster';
6
+
7
+ /** Fewer usable accounts than this warns unless the fleet says otherwise. */
8
+ export const MINIMUM_USABLE = 2;
9
+
10
+ /** `proxy:account:<provider>`: fails when a provider has no usable account,
11
+ * warns below the fleet's minimum. Accounts are named by their opaque index
12
+ * and counts only, never by an identifier.
13
+ */
14
+ export function providerAccountChecks(
15
+ roster: ProxyRoster,
16
+ minimumUsable = MINIMUM_USABLE
17
+ ): Check[] {
18
+ const providers = new Map<string, RosterAccount[]>();
19
+ for (const account of roster.accounts)
20
+ if (account.state !== 'removed')
21
+ providers.set(account.provider, [
22
+ ...(providers.get(account.provider) ?? []),
23
+ account,
24
+ ]);
25
+ return [...providers.entries()]
26
+ .sort(([a], [b]) => a.localeCompare(b))
27
+ .map(([provider, accounts]) =>
28
+ providerCheck(provider, accounts, minimumUsable)
29
+ );
30
+ }
31
+
32
+ /** "account 2 needs-login, account 3 cooling-down until …" */
33
+ const describe = (accounts: RosterAccount[]): string[] =>
34
+ accounts
35
+ .filter((account) => account.state !== 'usable')
36
+ .map(
37
+ (account) =>
38
+ `account ${account.index} ${account.state}${account.until ? ` until ${account.until}` : ''}`
39
+ );
40
+
41
+ function reloginRemedy(provider: string, accounts: RosterAccount[]) {
42
+ const login = accounts.find((account) => account.state === 'needs-login');
43
+ return login
44
+ ? {
45
+ action: `Sign ${provider} account ${login.index} in again. Signing in is the only manual step.`,
46
+ command: `devfleet fleet-accounts --relogin ${provider} ${login.index}`,
47
+ }
48
+ : undefined;
49
+ }
50
+
51
+ function providerCheck(
52
+ provider: string,
53
+ accounts: RosterAccount[],
54
+ minimumUsable: number
55
+ ): Check {
56
+ const usable = accounts.filter((account) => account.state === 'usable');
57
+ const id = `proxy:account:${provider}`;
58
+ const detail = describe(accounts);
59
+ const counted = `${usable.length} of ${accounts.length} ${provider} accounts usable${detail.length ? ` (${detail.join(', ')})` : ''}`;
60
+ const remedy = reloginRemedy(provider, accounts);
61
+ if (usable.length === 0) return noneUsable(provider, accounts);
62
+ if (usable.length < minimumUsable)
63
+ return {
64
+ id,
65
+ status: 'warn',
66
+ summary: `${counted}; one limit or expired login away from none.`,
67
+ remedy: remedy ?? {
68
+ action: `Add a ${provider} account: devfleet fleet-accounts --add ${provider}.`,
69
+ },
70
+ };
71
+ return {
72
+ id,
73
+ status: 'pass',
74
+ summary: `${counted}.`,
75
+ ...(remedy ? { remedy } : {}),
76
+ };
77
+ }
78
+
79
+ /** No usable account: say why, and when the first cooldown clears. */
80
+ function noneUsable(provider: string, accounts: RosterAccount[]): Check {
81
+ const clears = accounts
82
+ .map((account) => account.until)
83
+ .filter((until): until is string => until !== null)
84
+ .sort()[0];
85
+ const cooling = accounts.every((account) => account.state === 'cooling-down');
86
+ const wait = `Wait until ${clears ?? 'the provider lifts the limit'}, or add an account: devfleet fleet-accounts --add ${provider}.`;
87
+ return {
88
+ id: `proxy:account:${provider}`,
89
+ status: 'fail',
90
+ summary: cooling
91
+ ? `No usable ${provider} account: every one is cooling down${clears ? `; the first clears at ${clears}` : ''}.`
92
+ : `No usable ${provider} account (${describe(accounts).join(', ')}).`,
93
+ remedy: reloginRemedy(provider, accounts) ?? {
94
+ action: cooling
95
+ ? wait
96
+ : `Inspect the ${provider} accounts with devfleet fleet-accounts.`,
97
+ },
98
+ };
99
+ }
100
+
101
+ /** `accounts:roster-current`: this node offers exactly the accounts the
102
+ * proxy holds. `proxy` is the proxy's roster version (null: unreadable),
103
+ * `node` the version of the account instances the node exposes.
104
+ */
105
+ export function rosterCurrentCheck(proxy: string | null, node: string): Check {
106
+ if (proxy === null)
107
+ return {
108
+ id: 'accounts:roster-current',
109
+ status: 'warn',
110
+ summary: `The proxy's account roster could not be read; this node offers roster version ${node}.`,
111
+ remedy: {
112
+ action: 'Check proxy:reachable-from, then run fleet-status again.',
113
+ },
114
+ };
115
+ return proxy === node
116
+ ? {
117
+ id: 'accounts:roster-current',
118
+ status: 'pass',
119
+ summary: `This node offers the proxy's account roster (version ${node}).`,
120
+ }
121
+ : {
122
+ id: 'accounts:roster-current',
123
+ status: 'fail',
124
+ summary: `This node offers account roster version ${node}; the proxy holds ${proxy}.`,
125
+ remedy: {
126
+ action: 'Propagate the accounts to every node without a full sync.',
127
+ command: 'devfleet fleet-sync --only accounts',
128
+ },
129
+ };
130
+ }
131
+
132
+ /** Where agent CLIs keep a provider login of their own, relative to home. */
133
+ export const NATIVE_LOGINS: Record<string, string[]> = {
134
+ claude: ['.claude/.credentials.json'],
135
+ codex: ['.codex/auth.json'],
136
+ gemini: ['.gemini/oauth_creds.json'],
137
+ opencode: ['.local/share/opencode/auth.json'],
138
+ grok: ['.grok/auth.json'],
139
+ };
140
+
141
+ /** The provider logins found on this node outside the proxy path. */
142
+ export function nativeLogins(
143
+ home = homedir(),
144
+ exists: (path: string) => boolean = existsSync
145
+ ): { cli: string; path: string }[] {
146
+ return Object.entries(NATIVE_LOGINS).flatMap(([cli, paths]) =>
147
+ paths
148
+ .map((path) => join(home, path))
149
+ .filter((path) => exists(path))
150
+ .map((path) => ({ cli, path }))
151
+ );
152
+ }
153
+
154
+ /** `accounts:native-login`: a node holds no provider login of its own, so it
155
+ * can use no account the proxy does not know. Names the CLIs only.
156
+ */
157
+ export function nativeLoginCheck(found: { cli: string }[]): Check {
158
+ const clis = [...new Set(found.map((login) => login.cli))].sort();
159
+ return clis.length === 0
160
+ ? {
161
+ id: 'accounts:native-login',
162
+ status: 'pass',
163
+ summary:
164
+ 'No agent CLI on this node holds a provider login of its own; every account comes from the proxy.',
165
+ }
166
+ : {
167
+ id: 'accounts:native-login',
168
+ status: 'warn',
169
+ summary: `Provider logins outside the proxy path: ${clis.join(', ')}.`,
170
+ remedy: {
171
+ action:
172
+ 'Provider accounts belong on the proxy. Remove these logins (opt-in), and sign the account in on the proxy if it is needed.',
173
+ command: 'devfleet fleet-accounts --remove-native-logins --yes',
174
+ },
175
+ };
176
+ }
177
+
178
+ /** `proxy:reachable-from`: this node reaches the proxy, and how fast. */
179
+ export async function proxyReachableCheck(
180
+ endpoint: string,
181
+ send: typeof fetch = fetch,
182
+ clock: () => number = () => performance.now()
183
+ ): Promise<Check> {
184
+ const started = clock();
185
+ try {
186
+ // Unauthenticated: the proxy refuses it, which proves it answered.
187
+ const response = await send(endpoint + '/v1/models', {
188
+ redirect: 'error',
189
+ signal: AbortSignal.timeout(10_000),
190
+ });
191
+ await response.body?.cancel().catch(() => undefined);
192
+ const latency = Math.round(clock() - started);
193
+ const answered = response.status < 500;
194
+ return {
195
+ id: 'proxy:reachable-from',
196
+ status: !answered ? 'fail' : latency > 2000 ? 'warn' : 'pass',
197
+ summary: answered
198
+ ? `The proxy answered in ${latency} ms.`
199
+ : 'PROXY_UNREACHABLE: the route to the proxy answered, the proxy did not.',
200
+ ...(answered
201
+ ? {}
202
+ : {
203
+ remedy: {
204
+ action:
205
+ 'Agents retry for about a minute and then report "proxy unreachable". Check the proxy on the manager: devfleet cliproxy status.',
206
+ },
207
+ }),
208
+ };
209
+ } catch {
210
+ return {
211
+ id: 'proxy:reachable-from',
212
+ status: 'fail',
213
+ summary: 'PROXY_UNREACHABLE: no answer from the proxy.',
214
+ remedy: {
215
+ action:
216
+ 'Agents retry for about a minute and then report "proxy unreachable". Check this node’s tailnet and the proxy on the manager: devfleet cliproxy status.',
217
+ },
218
+ };
219
+ }
220
+ }
@@ -0,0 +1,76 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { homedir } from 'node:os';
3
+ import { join, resolve } from 'node:path';
4
+ import { accountPrefix, discoverAccounts } from '../agents/accounts';
5
+ import { proxyKey } from '../agents/secrets';
6
+ import { registry } from '../agents/registry';
7
+ import type { AccountContext } from '../agents/native-verify';
8
+ import type { Check } from '../types';
9
+ import {
10
+ nativeLoginCheck,
11
+ nativeLogins,
12
+ proxyReachableCheck,
13
+ rosterCurrentCheck,
14
+ } from './checks';
15
+ import { liveVersion, rosterVersion } from './roster';
16
+ import { exposedInstances, type Instance } from './sessions';
17
+
18
+ /** This machine's T3 settings, where its account instances are written. */
19
+ export const t3Settings = (): string =>
20
+ join(
21
+ resolve(process.env['T3CODE_HOME'] ?? join(homedir(), '.t3')),
22
+ 'userdata/settings.json'
23
+ );
24
+
25
+ /** A short digest of the client key: its version, never its value. */
26
+ export const keyVersion = (key: string): string =>
27
+ createHash('sha256').update(key).digest('hex').slice(0, 12);
28
+
29
+ /** The roster version of the instances a node offers. */
30
+ export function exposedVersion(instances: Instance[]): string {
31
+ return rosterVersion(
32
+ instances.flatMap((instance) => {
33
+ try {
34
+ return [accountPrefix(instance.tool, instance.account)];
35
+ } catch {
36
+ return [];
37
+ }
38
+ })
39
+ );
40
+ }
41
+
42
+ /** The proxy's accounts and what this node offers, read with this node's
43
+ * own key; undefined when the proxy or the key is unavailable (the provider
44
+ * checks report that).
45
+ */
46
+ export async function accountContext(
47
+ settings = t3Settings()
48
+ ): Promise<AccountContext | undefined> {
49
+ try {
50
+ const key = await proxyKey();
51
+ const live = await discoverAccounts(key);
52
+ return {
53
+ live,
54
+ exposed: await exposedInstances(settings),
55
+ rosterVersion: liveVersion(live),
56
+ keyVersion: keyVersion(key),
57
+ };
58
+ } catch {
59
+ return undefined;
60
+ }
61
+ }
62
+
63
+ /** What a node measures about accounts: it reaches the proxy, it offers
64
+ * the proxy's roster, and it holds no provider login of its own.
65
+ */
66
+ export async function nodeAccountChecks(
67
+ context: AccountContext | undefined,
68
+ settings = t3Settings()
69
+ ): Promise<Check[]> {
70
+ const exposed = context?.exposed ?? (await exposedInstances(settings));
71
+ return [
72
+ await proxyReachableCheck(registry.proxy.endpoint),
73
+ rosterCurrentCheck(context?.rosterVersion ?? null, exposedVersion(exposed)),
74
+ nativeLoginCheck(nativeLogins()),
75
+ ];
76
+ }
@@ -0,0 +1,62 @@
1
+ import { blocking, type Check, type FleetNode, type Runner } from '../types';
2
+ import { inspectRemote } from '../transport';
3
+ import type { CommandOutcome } from '../cli/contracts';
4
+
5
+ export interface Propagation {
6
+ /** The enrolled workers. */
7
+ nodes: FleetNode[];
8
+ /** The approved revision every worker runs. */
9
+ revision: string;
10
+ /** The bundled remote entry sent to each worker. */
11
+ source: string;
12
+ /** The manager's own account instances, applied locally. */
13
+ manager: () => Promise<Check[]>;
14
+ run: Runner;
15
+ }
16
+
17
+ /** `fleet-sync --only accounts`: the account-derived configuration and
18
+ * nothing else, on the manager and on every worker at once. Each node
19
+ * re-reads the proxy's roster and rewrites only its per-account T3
20
+ * instances; T3 watches its settings, so nothing restarts and no running
21
+ * turn ends. Instance ids derive from the account's routing prefix, so they
22
+ * stay the same and existing threads keep working. A node that cannot be
23
+ * reached is reported; it catches up on the next propagation or sync, and
24
+ * reports the roster version it has (`accounts:roster-current`) until then.
25
+ */
26
+ export async function propagateAccounts(
27
+ input: Propagation
28
+ ): Promise<CommandOutcome> {
29
+ const started = Date.now();
30
+ const [manager, ...workers] = await Promise.all([
31
+ input.manager().then((checks) => ({ node: 'manager', checks })),
32
+ ...input.nodes.map((node) =>
33
+ inspectRemote(
34
+ node,
35
+ {
36
+ source: input.source,
37
+ timeoutMs: 180_000,
38
+ action: 'accounts',
39
+ expected: { head: input.revision, files: {} },
40
+ },
41
+ input.run
42
+ )
43
+ ),
44
+ ]);
45
+ const nodes = [manager!, ...workers].map((report) => ({
46
+ node: report.node,
47
+ checks: report.checks,
48
+ }));
49
+ const ok = nodes.every((node) => !node.checks.some(blocking));
50
+ return {
51
+ exitCode: ok ? 0 : 1,
52
+ data: {
53
+ version: 1,
54
+ command: 'fleet-sync',
55
+ only: 'accounts',
56
+ ok,
57
+ revision: input.revision,
58
+ durationMs: Date.now() - started,
59
+ nodes,
60
+ },
61
+ };
62
+ }