@crvouga/dev-fleet 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/README.md +65 -9
  2. package/agents/audit.md +18 -18
  3. package/ansible/agents/agents.yml +134 -0
  4. package/ansible/agents/templates/antigravity.settings.json.j2 +3 -0
  5. package/ansible/agents/templates/claude.settings.json.j2 +3 -0
  6. package/ansible/agents/templates/codex.config.toml.j2 +11 -0
  7. package/ansible/agents/templates/gemini.settings.json.j2 +14 -0
  8. package/ansible/agents/templates/grok.config.toml.j2 +19 -0
  9. package/ansible/agents/templates/opencode.json.j2 +24 -0
  10. package/ansible/agents/templates/receipt.json.j2 +10 -0
  11. package/ansible/ansible.cfg +14 -0
  12. package/ansible/github/git.yml +82 -0
  13. package/ansible/macos/manager-privilege.yml +55 -9
  14. package/ansible/macos/manager.yml +154 -7
  15. package/ansible/macos/policy.json +13 -0
  16. package/ansible/macos/t3-policy.yml +5 -3
  17. package/ansible/macos/vault-renew.yml +85 -0
  18. package/ansible/monitoring/agent-darwin.yml +3 -0
  19. package/ansible/monitoring/agent.yml +2 -0
  20. package/ansible/monitoring/hub-darwin.yml +3 -0
  21. package/ansible/monitoring/hub-route-darwin.yml +1 -0
  22. package/ansible/net/remote-probe.sh +4 -1
  23. package/ansible/platforms.json +2 -2
  24. package/ansible/pyproject.toml +5 -0
  25. package/ansible/ubuntu/baseline.yml +39 -13
  26. package/ansible/ubuntu/disk-gc.py +6 -2
  27. package/ansible/ubuntu/firewall.nft.j2 +50 -0
  28. package/ansible/ubuntu/policy.json +125 -6
  29. package/ansible/ubuntu/reboot.yml +89 -0
  30. package/ansible/ubuntu/recovery.yml +13 -3
  31. package/ansible/ubuntu/t3/policy.json +3 -0
  32. package/ansible/ubuntu/t3-desktop.yml +23 -2
  33. package/ansible/ubuntu/t3-headless.yml +128 -0
  34. package/ansible/ubuntu/t3-health.sh +24 -10
  35. package/ansible/ubuntu/t3-idle.py +37 -2
  36. package/ansible/ubuntu/t3-policy.yml +4 -1
  37. package/ansible/ubuntu/t3-route.yml +1 -0
  38. package/ansible/ubuntu/tailscale.yml +54 -8
  39. package/ansible/ubuntu/tasks/browser.yml +3 -0
  40. package/ansible/ubuntu/tasks/containers-off.yml +43 -0
  41. package/ansible/ubuntu/tasks/docker.yml +53 -6
  42. package/ansible/ubuntu/tasks/dpkg.yml +4 -0
  43. package/ansible/ubuntu/tasks/features.yml +22 -0
  44. package/ansible/ubuntu/tasks/firewall.yml +172 -0
  45. package/ansible/ubuntu/tasks/install.yml +2 -1
  46. package/ansible/ubuntu/tasks/packages.yml +4 -2
  47. package/ansible/ubuntu/tasks/release.yml +6 -0
  48. package/ansible/ubuntu/tasks/resources.yml +6 -4
  49. package/ansible/ubuntu/tasks/services.yml +2 -2
  50. package/ansible/ubuntu/tasks/sizing.yml +49 -0
  51. package/ansible/ubuntu/tasks/snapshot.yml +14 -3
  52. package/ansible/ubuntu/tasks/ssh.yml +61 -4
  53. package/ansible/ubuntu/tasks/t3-restart.yml +3 -1
  54. package/ansible/ubuntu/tasks/user.yml +1 -0
  55. package/ansible/ubuntu/worker-account.yml +117 -0
  56. package/ansible/uv.lock +265 -0
  57. package/bin/devfleet.mjs +42 -4
  58. package/bootstrap/fleet-remote-bootstrap.sh +91 -25
  59. package/bootstrap/global-config.ts +29 -3
  60. package/bootstrap/package-smoke.ts +127 -0
  61. package/bootstrap/pins.ts +189 -0
  62. package/bootstrap/release-notes.ts +88 -0
  63. package/bootstrap/release-version.ts +41 -7
  64. package/bootstrap/worker-features.ts +114 -0
  65. package/bootstrap/worker-seed.yaml +7 -3
  66. package/bootstrap/worker-tools.ts +39 -9
  67. package/cliproxy/README.md +1 -1
  68. package/cliproxy/docs/health-workflow.example.yml +20 -143
  69. package/docs/authentication.md +19 -11
  70. package/docs/consumer-setup.md +42 -27
  71. package/docs/convergence.md +67 -0
  72. package/docs/declared.md +61 -34
  73. package/docs/machine-setup.md +73 -7
  74. package/docs/manager-setup.md +10 -3
  75. package/docs/operations.md +64 -9
  76. package/docs/releasing.md +49 -1
  77. package/docs/runbooks.md +98 -7
  78. package/docs/state.md +23 -11
  79. package/docs/support-matrix.md +40 -4
  80. package/docs/topology.md +67 -6
  81. package/docs/ubuntu-sync.md +78 -28
  82. package/docs/worker-contract.md +122 -19
  83. package/docs/worker-requirements.md +107 -0
  84. package/examples/fleet.json +1 -0
  85. package/examples/workers-from-infrastructure/main.tf +132 -0
  86. package/examples/workers-from-infrastructure/worker-records.json +14 -0
  87. package/llms.txt +3 -1
  88. package/mise.lock +29 -0
  89. package/mise.toml +5 -3
  90. package/package.json +2 -2
  91. package/schema/check-ids.json +1967 -170
  92. package/schema/check-ids.schema.json +34 -0
  93. package/schema/fleet.schema.json +327 -8
  94. package/schema/output/check.schema.json +1 -1
  95. package/schema/output/fleet-state-definition.schema.json +14 -0
  96. package/schema/output/fleet-sync-schedule.schema.json +26 -0
  97. package/schema/output/fleet-sync.schema.json +18 -0
  98. package/schema/output/report.schema.json +3 -2
  99. package/schema/worker-record.schema.json +76 -0
  100. package/src/agents/converge.ts +140 -24
  101. package/src/agents/files.ts +2 -32
  102. package/src/agents/native-verify.ts +34 -3
  103. package/src/agents/secrets.ts +39 -4
  104. package/src/check-owners.ts +61 -2
  105. package/src/check-registry.ts +630 -0
  106. package/src/cli/agent-guide.ts +3 -3
  107. package/src/cli/cli.ts +4 -0
  108. package/src/cli/fleet-manifest.ts +9 -3
  109. package/src/cli/fleet-node-lifecycle.ts +313 -64
  110. package/src/cli/fleet-node-park.ts +238 -0
  111. package/src/cli/fleet-node-seed.ts +51 -35
  112. package/src/cli/fleet-state.ts +2 -1
  113. package/src/cli/fleet-sync-checkout.ts +7 -0
  114. package/src/cli/fleet-sync-enrollment.ts +193 -32
  115. package/src/cli/fleet-sync-forward.ts +4 -0
  116. package/src/cli/fleet-sync-manager.ts +103 -33
  117. package/src/cli/fleet-sync-output.ts +31 -11
  118. package/src/cli/fleet-sync-plan.ts +65 -9
  119. package/src/cli/fleet-sync-records.ts +113 -0
  120. package/src/cli/fleet-sync-remote.ts +151 -9
  121. package/src/cli/fleet-sync-schedule.ts +14 -4
  122. package/src/cli/fleet-sync-state.ts +28 -16
  123. package/src/cli/fleet-sync-vault.ts +77 -0
  124. package/src/cli/fleet-sync-worker.ts +214 -0
  125. package/src/cli/fleet-sync.ts +150 -19
  126. package/src/cli/fleet-validate.ts +59 -21
  127. package/src/cli/fleet-watch.ts +7 -1
  128. package/src/cli/fleet.ts +87 -5
  129. package/src/cli/managed-health.ts +25 -1
  130. package/src/cli/register-fleet.ts +115 -4
  131. package/src/cli/registry.ts +14 -0
  132. package/src/cli/tailnet.ts +43 -10
  133. package/src/cli/ui-data.ts +3 -1
  134. package/src/cli/worker-usage-report.ts +63 -0
  135. package/src/clock.ts +42 -0
  136. package/src/definition.ts +332 -9
  137. package/src/diagnostics.ts +5 -2
  138. package/src/exposure.ts +102 -0
  139. package/src/fleet-config.ts +3 -1
  140. package/src/git-identity.ts +76 -3
  141. package/src/ide/ide.ts +5 -0
  142. package/src/ide/t3.ts +12 -2
  143. package/src/input-hashes.ts +6 -0
  144. package/src/machine/machine-class.ts +59 -0
  145. package/src/machine/platform.ts +90 -11
  146. package/src/machine/state.ts +37 -15
  147. package/src/machine/support-matrix.ts +11 -9
  148. package/src/machine-health.ts +28 -5
  149. package/src/manager-health.ts +148 -2
  150. package/src/manager-sync.ts +65 -4
  151. package/src/monitoring/beszel.ts +8 -8
  152. package/src/monitoring/enroll.ts +8 -1
  153. package/src/monitoring/hub-access.ts +10 -4
  154. package/src/monitoring/systems.ts +138 -0
  155. package/src/network-definition.ts +13 -4
  156. package/src/node-converge-linux.ts +136 -27
  157. package/src/node-converge.ts +13 -3
  158. package/src/platform/ansible-receipt.ts +17 -2
  159. package/src/platform/ansible.ts +40 -4
  160. package/src/platform/manager-host.ts +25 -8
  161. package/src/platform/resource-verify.ts +18 -7
  162. package/src/platform/t3-policy.ts +24 -10
  163. package/src/platform/t3-service.ts +109 -34
  164. package/src/platform/ubuntu-release.ts +24 -0
  165. package/src/platform/ubuntu-verify.ts +31 -3
  166. package/src/platform/ubuntu-worker.ts +58 -0
  167. package/src/platform/worker-platform.ts +72 -0
  168. package/src/protocol.ts +1 -1
  169. package/src/public-reachability.ts +186 -0
  170. package/src/remote-bootstrap.ts +50 -12
  171. package/src/remote-operation.ts +18 -4
  172. package/src/repair-plan.ts +23 -12
  173. package/src/report.ts +2 -1
  174. package/src/schedule-check.ts +21 -2
  175. package/src/secrets.ts +4 -0
  176. package/src/service-auth.ts +55 -48
  177. package/src/ssh-connection.ts +141 -0
  178. package/src/ssh.ts +63 -24
  179. package/src/state/store.ts +4 -2
  180. package/src/state/sync.ts +35 -6
  181. package/src/state/workers.ts +140 -0
  182. package/src/t3/pairing.ts +59 -0
  183. package/src/t3/peer-link.ts +333 -0
  184. package/src/t3/version.ts +1 -1
  185. package/src/tailnet-cleanup.ts +185 -0
  186. package/src/tailnet-control.ts +346 -0
  187. package/src/tailscale.ts +1 -1
  188. package/src/toolchain-checks.ts +11 -1
  189. package/src/types.ts +4 -2
  190. package/src/worker-account.ts +63 -0
  191. package/src/worker-activity.ts +303 -0
  192. package/src/worker-capacity.ts +31 -0
  193. package/src/worker-exposure.ts +388 -0
  194. package/src/worker-features.ts +201 -0
  195. package/src/worker-host-checks.ts +71 -20
  196. package/src/worker-manifest.ts +27 -4
  197. package/src/worker-parking.ts +70 -0
  198. package/src/worker-policy.ts +17 -1
  199. package/src/worker-readiness.ts +15 -3
  200. package/src/worker-reboot.ts +129 -0
  201. package/src/worker-records.ts +305 -0
  202. package/src/worker-seed.ts +16 -11
  203. package/src/worker-sizing.ts +76 -0
  204. package/src/cli/schedule-logs.ts +0 -36
@@ -0,0 +1,88 @@
1
+ /**
2
+ * The notes of a release, from the Conventional Commits since the last
3
+ * release tag. A commit whose change restarts T3 on the workers, and so ends
4
+ * the agent turns running there, says so in a footer:
5
+ *
6
+ * Worker-Restart: the T3 desktop app is replaced
7
+ *
8
+ * and the notes put it first, so whoever installs the release knows to wait
9
+ * for the agents to finish.
10
+ *
11
+ * bun bootstrap/release-notes.ts > notes.md
12
+ */
13
+ import { spawnSync } from 'node:child_process';
14
+ import type { Git } from './release-version';
15
+
16
+ const SUBJECT = /^(\w+)(?:\(([^)]*)\))?(!)?:\s*(.+)$/;
17
+ const BREAKING_FOOTER = /^BREAKING[ -]CHANGE:\s*(.+)$/m;
18
+ const RESTART_FOOTER = /^Worker-Restart:\s*(.+)$/m;
19
+
20
+ const SECTIONS: [string, (type: string) => boolean][] = [
21
+ ['Features', (type) => type === 'feat'],
22
+ ['Fixes', (type) => type === 'fix'],
23
+ ['Other changes', () => true],
24
+ ];
25
+
26
+ export function releaseNotes(messages: string[]): string {
27
+ const restarts: string[] = [];
28
+ const breaking: string[] = [];
29
+ const sections = new Map<string, string[]>(
30
+ SECTIONS.map(([title]) => [title, []])
31
+ );
32
+ for (const message of messages) {
33
+ const subject = message.split('\n', 1)[0]!.trim();
34
+ // A merge commit only joins commits already listed.
35
+ if (/^Merge (pull request|branch) /.test(subject)) continue;
36
+ const parsed = SUBJECT.exec(subject);
37
+ const type = parsed?.[1] ?? '';
38
+ const scope = parsed?.[2];
39
+ const text = parsed?.[4] ?? subject;
40
+ const entry = scope ? `**${scope}:** ${text}` : text;
41
+ const restart = RESTART_FOOTER.exec(message)?.[1];
42
+ if (restart) restarts.push(`${entry} (${restart.trim()})`);
43
+ const footer = BREAKING_FOOTER.exec(message)?.[1];
44
+ if (parsed?.[3] || footer)
45
+ breaking.push(footer ? `${entry}: ${footer.trim()}` : entry);
46
+ const [title] = SECTIONS.find(([, matches]) => matches(type))!;
47
+ sections.get(title)!.push(entry);
48
+ }
49
+ const blocks: string[] = [];
50
+ const block = (title: string, entries: string[]) => {
51
+ if (entries.length)
52
+ blocks.push(
53
+ `## ${title}\n\n${entries.map((entry) => `- ${entry}`).join('\n')}`
54
+ );
55
+ };
56
+ block('Requires a worker restart (ends agent turns)', restarts);
57
+ block('Breaking changes', breaking);
58
+ for (const [title] of SECTIONS) block(title, sections.get(title)!);
59
+ return blocks.length ? `${blocks.join('\n\n')}\n` : 'No changes.\n';
60
+ }
61
+
62
+ /** The messages of the commits since the newest release tag, newest first. */
63
+ export function messagesSinceRelease(git: Git): string[] {
64
+ // A hand-pushed tag releases HEAD itself: the last release is the one before.
65
+ const tags = (git(['tag', '--list', 'v*', '--sort=-v:refname']) ?? '')
66
+ .split('\n')
67
+ .map((tag) => tag.trim())
68
+ .filter((tag) => /^v\d+\.\d+\.\d+$/.test(tag));
69
+ const tag =
70
+ tags.find(
71
+ (candidate) =>
72
+ git(['merge-base', '--is-ancestor', 'HEAD', candidate]) === null
73
+ ) ?? null;
74
+ const log =
75
+ git(['log', '--format=%B%x00', tag ? `${tag}..HEAD` : 'HEAD']) ?? '';
76
+ return log
77
+ .split('\0')
78
+ .map((message) => message.trim())
79
+ .filter(Boolean);
80
+ }
81
+
82
+ if (import.meta.main) {
83
+ const git: Git = (args) => {
84
+ const result = spawnSync('git', args, { encoding: 'utf8' });
85
+ return result.status === 0 ? (result.stdout ?? '') : null;
86
+ };
87
+ process.stdout.write(releaseNotes(messagesSinceRelease(git)));
88
+ }
@@ -12,6 +12,10 @@
12
12
  * `feat:` minor
13
13
  * anything else patch
14
14
  *
15
+ * A `HEAD` that already carries its release tag is released again when that
16
+ * tag has no GitHub Release: a run that pushed the tag and then failed is
17
+ * finished by any re-run of it, not only by re-running its failed jobs.
18
+ *
15
19
  * Exits 1 when nothing can be released because publishing was never set up
16
20
  * (no tag and nothing on npm), so the Release workflow does not report a
17
21
  * skipped release as a success.
@@ -62,13 +66,16 @@ export function bumpVersion(tag: string, bump: Bump): string {
62
66
  return `${major}.${minor + 1}.0`;
63
67
  }
64
68
 
69
+ function releaseTags(listed: string | null): string[] {
70
+ return (listed ?? '')
71
+ .split('\n')
72
+ .map((tag) => tag.trim())
73
+ .filter((tag) => TAG.test(tag));
74
+ }
75
+
65
76
  export function latestTag(git: Git): string | null {
66
- const tags = git(['tag', '--list', 'v*', '--sort=-v:refname']) ?? '';
67
77
  return (
68
- tags
69
- .split('\n')
70
- .map((tag) => tag.trim())
71
- .find((tag) => TAG.test(tag)) ?? null
78
+ releaseTags(git(['tag', '--list', 'v*', '--sort=-v:refname']))[0] ?? null
72
79
  );
73
80
  }
74
81
 
@@ -81,11 +88,30 @@ export function newestPublished(versions: string[]): string | null {
81
88
  return stable[0]?.join('.') ?? null;
82
89
  }
83
90
 
91
+ /** Whether the release tag has its GitHub Release. */
92
+ export type Finished = (tag: string) => boolean;
93
+
84
94
  /**
85
95
  * `published` is the newest stable version on npm, or null when the package
86
96
  * is not there.
87
97
  */
88
- export function decide(git: Git, published: string | null = null): Decision {
98
+ export function decide(
99
+ git: Git,
100
+ published: string | null = null,
101
+ finished: Finished = () => true
102
+ ): Decision {
103
+ const unfinished = releaseTags(
104
+ git(['tag', '--list', 'v*', '--points-at', 'HEAD', '--sort=-v:refname'])
105
+ ).find((tag) => !finished(tag));
106
+ if (unfinished)
107
+ return {
108
+ release: true,
109
+ version: unfinished.slice(1),
110
+ previous:
111
+ releaseTags(git(['tag', '--list', 'v*', '--sort=-v:refname'])).find(
112
+ (tag) => tag !== unfinished
113
+ ) ?? unfinished,
114
+ };
89
115
  const tag = latestTag(git);
90
116
  if (!tag && !published)
91
117
  return {
@@ -136,8 +162,16 @@ function npmVersions(): string[] {
136
162
  : [];
137
163
  }
138
164
 
165
+ /** Whether the tag has a GitHub Release, as `gh` sees it with the run's token. */
166
+ const hasGitHubRelease: Finished = (tag) =>
167
+ spawnSync('gh', ['release', 'view', tag], { stdio: 'ignore' }).status === 0;
168
+
139
169
  if (import.meta.main) {
140
- const decision = decide(system, newestPublished(npmVersions()));
170
+ const decision = decide(
171
+ system,
172
+ newestPublished(npmVersions()),
173
+ hasGitHubRelease
174
+ );
141
175
  const summary = process.env['GITHUB_STEP_SUMMARY'];
142
176
  const note = (line: string) => {
143
177
  if (summary) appendFileSync(summary, `${line}\n`);
@@ -0,0 +1,114 @@
1
+ import { existsSync, readFileSync } from 'node:fs';
2
+ import { homedir } from 'node:os';
3
+ import { join } from 'node:path';
4
+
5
+ /** How a worker runs T3: `desktop`, the pinned desktop app in an auto-login
6
+ * graphical session, or `headless`, the pinned native server as a lingering
7
+ * user service with no display manager (docs/worker-contract.md).
8
+ */
9
+ export const SESSIONS = ['desktop', 'headless'] as const;
10
+ export type Session = (typeof SESSIONS)[number];
11
+
12
+ /** What a worker is built with beyond the base that every worker has. */
13
+ export interface WorkerFeatures {
14
+ /** Docker: its packages, service, group membership, CLI and clean-up. */
15
+ containers: boolean;
16
+ session: Session;
17
+ }
18
+
19
+ /** A fleet that names no feature gets exactly what every worker had before
20
+ * features existed. */
21
+ export const DEFAULT_FEATURES: WorkerFeatures = {
22
+ containers: true,
23
+ session: 'desktop',
24
+ };
25
+
26
+ /** Named sets of features. `editing` is a worker for remote code editing by
27
+ * agents: no containers, no desktop, and only the base tools (it adds no
28
+ * tool profile).
29
+ */
30
+ export const PRESETS = {
31
+ editing: { containers: false, session: 'headless' },
32
+ } as const satisfies Record<string, WorkerFeatures>;
33
+ export type Preset = keyof typeof PRESETS;
34
+
35
+ const record = (value: unknown): value is Record<string, unknown> =>
36
+ typeof value === 'object' && value !== null && !Array.isArray(value);
37
+
38
+ /** The first field of a `features` block that breaks its rule, with what
39
+ * it expects; null when the block is valid. */
40
+ export function featuresFault(
41
+ value: unknown
42
+ ): { field: string; expected: string } | null {
43
+ if (!record(value)) return { field: '', expected: 'an object' };
44
+ const expected: Record<string, string> = {
45
+ preset: `one of ${Object.keys(PRESETS).join(', ')}`,
46
+ containers: 'true or false',
47
+ session: `one of ${SESSIONS.join(', ')}`,
48
+ };
49
+ const unknown = Object.keys(value).find((key) => !(key in expected));
50
+ if (unknown !== undefined)
51
+ return { field: unknown, expected: 'preset, containers or session' };
52
+ const valid: Record<string, (item: unknown) => boolean> = {
53
+ preset: (item) => typeof item === 'string' && Object.hasOwn(PRESETS, item),
54
+ containers: (item) => typeof item === 'boolean',
55
+ session: (item) => SESSIONS.includes(item as Session),
56
+ };
57
+ const field = Object.keys(value).find((key) => !valid[key]!(value[key]));
58
+ return field === undefined ? null : { field, expected: expected[field]! };
59
+ }
60
+
61
+ /** One block over the features before it: its preset, then its own fields. */
62
+ function over(base: WorkerFeatures, block: unknown): WorkerFeatures {
63
+ if (!record(block)) return base;
64
+ const preset =
65
+ typeof block['preset'] === 'string' &&
66
+ Object.hasOwn(PRESETS, block['preset'])
67
+ ? PRESETS[block['preset'] as Preset]
68
+ : {};
69
+ return {
70
+ ...base,
71
+ ...preset,
72
+ ...(typeof block['containers'] === 'boolean'
73
+ ? { containers: block['containers'] }
74
+ : {}),
75
+ ...(SESSIONS.includes(block['session'] as Session)
76
+ ? { session: block['session'] as Session }
77
+ : {}),
78
+ };
79
+ }
80
+
81
+ /** The features of one worker: the defaults, then the fleet-wide
82
+ * `worker.features`, then the worker's own `features`, each with its preset
83
+ * first. Read from the plain document, so the bootstrap needs no fleet code.
84
+ */
85
+ export function workerFeatures(
86
+ definition: unknown,
87
+ node?: string
88
+ ): WorkerFeatures {
89
+ if (!record(definition)) return DEFAULT_FEATURES;
90
+ const policy = record(definition['worker']) ? definition['worker'] : {};
91
+ const workers: unknown[] = Array.isArray(definition['workers'])
92
+ ? definition['workers']
93
+ : [];
94
+ const own = workers
95
+ .filter(record)
96
+ .find((entry) => node !== undefined && entry['id'] === node);
97
+ return over(over(DEFAULT_FEATURES, policy['features']), own?.['features']);
98
+ }
99
+
100
+ /** The features of this worker in the fleet configuration fleet-sync
101
+ * installed; the defaults when there is none yet.
102
+ */
103
+ export function installedFeatures(
104
+ node: string | undefined,
105
+ home: string = homedir()
106
+ ): WorkerFeatures {
107
+ const path = join(home, '.config/dev-fleet/fleet.json');
108
+ if (!existsSync(path)) return DEFAULT_FEATURES;
109
+ try {
110
+ return workerFeatures(JSON.parse(readFileSync(path, 'utf8')), node);
111
+ } catch {
112
+ return DEFAULT_FEATURES;
113
+ }
114
+ }
@@ -2,9 +2,12 @@
2
2
  # fleet-node-seed fills each ${NAME} with a value of the worker it seeds:
3
3
  # values from fleet.json, the manager's public key, a host key generated for
4
4
  # the machine, the Tailscale version and key digest ansible/ubuntu/policy.json
5
- # pins, and a short-lived single-use auth key that is deleted once the machine
6
- # has joined. The sudoers fragment is the one ansible/ubuntu/authorize-sudo.sh
7
- # writes: its path and marker are already on running machines.
5
+ # pins, the login server of the fleet's control plane, and a short-lived
6
+ # single-use auth key that is deleted once the machine has joined: one minted
7
+ # for this worker through the tailnet control plane fleet.json declares, or
8
+ # else the secret tailnet-auth-key. The sudoers fragment is the one
9
+ # ansible/ubuntu/authorize-sudo.sh writes: its path and marker are already on
10
+ # running machines.
8
11
  hostname: '${HOSTNAME}'
9
12
  users:
10
13
  - name: '${USER}'
@@ -56,6 +59,7 @@ runcmd:
56
59
  up,
57
60
  '--auth-key=file:/run/devfleet/tailnet-auth-key',
58
61
  '--hostname=${HOSTNAME}',
62
+ '--login-server=${LOGIN_SERVER}',
59
63
  '--ssh=false',
60
64
  ]
61
65
  - [rm, -f, /run/devfleet/tailnet-auth-key]
@@ -1,11 +1,15 @@
1
1
  import { existsSync, readFileSync } from 'node:fs';
2
2
  import { homedir } from 'node:os';
3
3
  import { join } from 'node:path';
4
+ import policy from '../ansible/ubuntu/policy.json';
5
+ import { workerFeatures } from './worker-features';
4
6
 
5
7
  /** A worker installs only this declared subset; every version still comes from mise.lock. */
6
8
  export function workerTools<T>(
7
9
  tools: Record<string, T>,
8
- selected: unknown
10
+ selected: unknown,
11
+ /** False on a worker without containers: docker-cli is then not required. */
12
+ containers = true
9
13
  ): Record<string, T> {
10
14
  if (typeof selected === 'string') selected = selected.split(',');
11
15
  if (
@@ -15,9 +19,15 @@ export function workerTools<T>(
15
19
  !selected.every(
16
20
  (key) => typeof key === 'string' && Object.hasOwn(tools, key)
17
21
  ) ||
18
- !['bun', 'node', 'gh', 'python', 'uv', 'docker-cli', 'vault'].every((key) =>
19
- selected.includes(key)
20
- )
22
+ ![
23
+ 'bun',
24
+ 'node',
25
+ 'gh',
26
+ 'python',
27
+ 'uv',
28
+ ...(containers ? ['docker-cli'] : []),
29
+ 'vault',
30
+ ].every((key) => selected.includes(key))
21
31
  )
22
32
  throw new Error(
23
33
  'Invalid worker tool profile: all required tools need authoritative mise pins.'
@@ -40,6 +50,9 @@ export type ToolProfile = (typeof TOOL_PROFILES)[number];
40
50
  export interface ToolSelection {
41
51
  profiles: string[];
42
52
  tools: string[];
53
+ /** False when the worker's `containers` feature is off: the base tools
54
+ * then leave out the Docker CLI. Absent means on. */
55
+ containers?: boolean;
43
56
  }
44
57
  export const NO_SELECTION: ToolSelection = { profiles: [], tools: [] };
45
58
 
@@ -68,9 +81,11 @@ export function workerSelection(
68
81
  .filter(record)
69
82
  .filter((entry) => node !== undefined && entry['id'] === node)
70
83
  .flatMap((entry) => list(entry['toolProfiles']));
84
+ const containers = workerFeatures(definition, node).containers;
71
85
  return {
72
86
  profiles: [...new Set([...list(policy['toolProfiles']), ...own])],
73
87
  tools: [...new Set(list(policy['tools']))],
88
+ ...(containers ? {} : { containers }),
74
89
  };
75
90
  }
76
91
 
@@ -101,7 +116,11 @@ export function workerToolset<T>(
101
116
  ): Record<string, T> {
102
117
  const names = (value: unknown) =>
103
118
  typeof value === 'string' ? value.split(',').filter(Boolean) : [];
104
- const base = names(vars['worker_tools']);
119
+ const containers = selection.containers !== false;
120
+ // Without containers the Docker CLI is not a base tool (CONTAINER_TOOLS).
121
+ const base = names(vars['worker_tools']).filter(
122
+ (tool) => containers || !CONTAINER_TOOLS.includes(tool)
123
+ );
105
124
  const added = selection.profiles.flatMap((profile) => {
106
125
  const declared = vars[`worker_profile_${profile}`];
107
126
  if (typeof declared !== 'string' || !declared)
@@ -115,11 +134,17 @@ export function workerToolset<T>(
115
134
  throw new Error(
116
135
  `The worker tool "${tool}" has no pin in dev-fleet's mise.toml. Ask for it in an issue.`
117
136
  );
118
- return workerTools(tools, [
119
- ...new Set([...base, ...added, ...selection.tools]),
120
- ]);
137
+ return workerTools(
138
+ tools,
139
+ [...new Set([...base, ...added, ...selection.tools])],
140
+ containers
141
+ );
121
142
  }
122
143
 
144
+ /** The base tools that belong to the `containers` feature
145
+ * (`features.containers.tools` in ansible/ubuntu/policy.json). */
146
+ export const CONTAINER_TOOLS: string[] = policy.features.containers.tools;
147
+
123
148
  /** `--profiles web,go --tools fd` for the doctor and the setup. */
124
149
  export function selectionArguments(selection: ToolSelection): string[] {
125
150
  return [
@@ -127,6 +152,7 @@ export function selectionArguments(selection: ToolSelection): string[] {
127
152
  ? ['--profiles', selection.profiles.join(',')]
128
153
  : []),
129
154
  ...(selection.tools.length ? ['--tools', selection.tools.join(',')] : []),
155
+ ...(selection.containers === false ? ['--no-containers'] : []),
130
156
  ];
131
157
  }
132
158
 
@@ -138,5 +164,9 @@ export function parseSelectionArguments(argv: string[]): ToolSelection {
138
164
  ? []
139
165
  : (argv[index + 1] ?? '').split(',').filter((item) => item.length > 0);
140
166
  };
141
- return { profiles: value('--profiles'), tools: value('--tools') };
167
+ return {
168
+ profiles: value('--profiles'),
169
+ tools: value('--tools'),
170
+ ...(argv.includes('--no-containers') ? { containers: false } : {}),
171
+ };
142
172
  }
@@ -199,7 +199,7 @@ On the manager (or a replacement manager), prepare the tracked installation, log
199
199
 
200
200
  ## Off-machine checks and offline recovery
201
201
 
202
- The `CLI proxy health` GitHub Actions workflow runs at minutes 7, 22, 37 and 52 of each hour and supports manual dispatch. It uses the existing read-only GitHub OIDC Vault role, with a short-lived token rather than a stored workstation/admin credential. From a GitHub-hosted runner it checks public endpoint authentication, available models, dashboard availability, a cloud checkpoint no more than 45 minutes old, and the last successful GC no more than nine days old. GC status is included in subsequent encrypted checkpoints. A failed workflow is visible independently of this workstation and uses GitHub's existing notification settings; it is not a guaranteed paging service, and schedules can be delayed by GitHub.
202
+ The `CLI proxy health` GitHub Actions workflow is the reusable workflow `.github/workflows/cliproxy-health.yml`. The repository that holds your `fleet.json` enables it with one `uses:` line: copy [`docs/health-workflow.example.yml`](docs/health-workflow.example.yml) into its `.github/workflows/` and name the dev-fleet release tag your `package.json` pins. It needs no stored secret: the Vault address and the endpoint come from your `fleet.json`, and the checks from that exact release. The example runs it at minutes 7, 22, 37 and 52 of each hour and supports manual dispatch. It uses the existing read-only GitHub OIDC Vault role (`vault-role`, `github-actions` by default), with a short-lived token rather than a stored workstation/admin credential. From a GitHub-hosted runner it checks public endpoint authentication, available models, dashboard availability, a cloud checkpoint no more than 45 minutes old, and the last successful GC no more than nine days old. GC status is included in subsequent encrypted checkpoints. A failed workflow is visible independently of this workstation and uses GitHub's existing notification settings; it is not a guaranteed paging service, and schedules can be delayed by GitHub.
203
203
 
204
204
  The `CLI proxy health` workflow also runs the declarative model checks described in [Model validation and health checks](docs/model-health.md): a free readiness check every 15 minutes, a light two-request-per-model inference check hourly, a comprehensive nightly suite and an offline pull-request contract job, with per-model status, capability coverage, JSON/Markdown reports, an agent brief and a tracking issue. The local five-minute monitor does not run inference, attempt management logins, access provider credential files, prune storage, change cloud infrastructure, or restart the workstation. Management authorization remains covered by explicit verification; repeated failed management logins can trigger upstream IP bans, so they are excluded from scheduled monitoring. Its Restic client permits only read operations and uses no remote locks, so a killed runner cannot leave a lock blocking backups. Concurrent pruning can cause a transient read failure; subsequent scheduled runs retry. HTTPS redirects are rejected to prevent forwarding credentials to another origin.
205
205
 
@@ -1,19 +1,21 @@
1
1
  name: CLI proxy health
2
- # One workflow, five entry points, deliberately cheap by default:
2
+ # Copy this file into .github/workflows/ of the repository that holds your
3
+ # fleet.json, and name the dev-fleet release your package.json pins in the
4
+ # `uses:` line (RELEASE_TAG, such as v1.4.0). The checks are the reusable workflow
5
+ # .github/workflows/cliproxy-health.yml of dev-fleet; this file only says
6
+ # when they run. One workflow, five entry points, deliberately cheap by
7
+ # default:
3
8
  # every 15 min readiness: free (no inference) + cloud checkpoint freshness
4
9
  # hourly light: 2 tiny requests per model (about 200 requests a day)
5
10
  # 03:41 nightly comprehensive suite (bounded by models.json budgets)
6
- # pull request contract: offline, no credentials, no inference
7
11
  # manual any profile, one model or one check, optional force-probe
12
+ # The offline contract (no credentials, no inference) runs in dev-fleet's own
13
+ # CI, not here.
8
14
  on:
9
15
  schedule:
10
16
  - cron: '7,22,37,52 * * * *'
11
17
  - cron: '17 * * * *'
12
18
  - cron: '41 3 * * *'
13
- pull_request:
14
- paths:
15
- - cliproxy/**
16
- - .github/workflows/cliproxy-health.yml
17
19
  workflow_dispatch:
18
20
  inputs:
19
21
  profile:
@@ -42,144 +44,19 @@ on:
42
44
  description: Probe models even if they were rate limited recently (normally paused to protect quota).
43
45
  type: boolean
44
46
  default: false
47
+ # A short-lived Vault token from GitHub's OIDC token, and one tracking issue.
45
48
  permissions:
46
49
  contents: read
47
- concurrency:
48
- group: cliproxy-model-health-${{ github.event_name == 'pull_request' && format('pr-{0}', github.ref) || github.event_name == 'workflow_dispatch' && inputs.profile || github.event.schedule }}
49
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
50
- env:
51
- PROFILE: ${{ github.event_name == 'workflow_dispatch' && inputs.profile || github.event.schedule == '41 3 * * *' && 'nightly' || github.event.schedule == '17 * * * *' && 'light' || 'readiness' }}
50
+ id-token: write
51
+ issues: write
52
52
  jobs:
53
- contract:
54
- # Pull requests: deterministic protocol, routing and report contracts. No credentials, no inference.
55
- if: github.event_name == 'pull_request'
56
- runs-on: ubuntu-latest
57
- timeout-minutes: 10
58
- steps:
59
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
60
- - name: Install PyYAML for the validate-config contract
61
- run: |
62
- sudo apt-get update
63
- sudo apt-get install -y --no-install-recommends python3-yaml
64
- - name: Lint models.json and run the offline contract tests
65
- run: |
66
- python3 -B cliproxy/scripts/check.py
67
- python3 -B -m unittest discover -s cliproxy/tests -p "test_model_*.py"
68
- - name: Run the full declarative suite against the local emulator
69
- run: |
70
- set +e
71
- python3 -B cliproxy/scripts/model_health.py model-suite --target mock --profile manual \
72
- --fail-on degraded --output model-health-report --seed 1 > /dev/null
73
- code=$?
74
- if [ -f model-health-report/model-health.md ]; then cat model-health-report/model-health.md >> "$GITHUB_STEP_SUMMARY"; fi
75
- exit $code
76
- - name: Keep structured report
77
- if: ${{ always() && hashFiles('model-health-report/model-health.json') != '' }}
78
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
79
- with:
80
- name: model-health-contract-${{ github.run_id }}
81
- path: model-health-report
82
- retention-days: 14
83
53
  health:
84
- if: github.event_name != 'pull_request' && github.repository == 'crvouga/dev-fleet' && github.ref == 'refs/heads/main'
85
- runs-on: ubuntu-latest
86
- timeout-minutes: 60
87
- permissions:
88
- contents: read
89
- id-token: write
90
- issues: write
91
- steps:
92
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
93
- - name: Install read-only backup client
94
- if: env.PROFILE == 'readiness'
95
- run: |
96
- sudo apt-get update
97
- sudo apt-get install -y --no-install-recommends restic
98
- - name: Read-only Vault login using GitHub OIDC
99
- uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0
100
- with:
101
- url: https://vault.example.com
102
- method: jwt
103
- path: jwt
104
- role: github-actions
105
- jwtGithubAudience: https://vault.example.com
106
- exportToken: true
107
- - name: Verify endpoint and cloud recovery freshness
108
- if: env.PROFILE == 'readiness'
109
- run: python3 -B cliproxy/scripts/external-health.py
110
- - name: Restore last-success and rate-limit state
111
- uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
112
- with:
113
- path: model-health-state.json
114
- key: cliproxy-model-health-state-${{ github.run_id }}
115
- restore-keys: cliproxy-model-health-state-
116
- - name: Validate every required model
117
- # Bounded spend: the profile's request/token budget, concurrency and timeouts come from models.json;
118
- # inference is never retried; a model rate limited recently is not probed again until its pause ends.
119
- # The client key is read in-process from Vault and never printed.
120
- if: ${{ !cancelled() }}
121
- env:
122
- MODEL: ${{ inputs.model }}
123
- CHECK: ${{ inputs.check }}
124
- MAX_REQUESTS: ${{ inputs.max_requests }}
125
- SUITE: ${{ inputs.suite || 'all' }}
126
- FORCE_PROBE: ${{ inputs.force_probe }}
127
- run: |
128
- command=model-health
129
- fail_on=unhealthy
130
- case "$PROFILE" in nightly | manual) command=model-suite; fail_on=degraded ;; esac
131
- args=(--profile "$PROFILE" --suite "$SUITE" --fail-on "$fail_on"
132
- --output model-health-report --state model-health-state.json)
133
- if [ -n "$MODEL" ]; then args+=(--model "$MODEL"); fi
134
- if [ -n "$CHECK" ]; then args+=(--check "$CHECK"); fi
135
- if [ -n "$MAX_REQUESTS" ]; then args+=(--max-requests "$MAX_REQUESTS"); fi
136
- if [ "$FORCE_PROBE" = "true" ]; then args+=(--force-probe); fi
137
- set +e
138
- python3 -B cliproxy/scripts/model_health.py "$command" "${args[@]}" > /dev/null
139
- code=$?
140
- if [ -f model-health-report/agent-brief.md ]; then cat model-health-report/agent-brief.md >> "$GITHUB_STEP_SUMMARY"; fi
141
- if [ -f model-health-report/model-health.md ]; then cat model-health-report/model-health.md >> "$GITHUB_STEP_SUMMARY"; fi
142
- exit $code
143
- - name: Save last-success and rate-limit state
144
- if: ${{ always() && hashFiles('model-health-state.json') != '' }}
145
- uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
146
- with:
147
- path: model-health-state.json
148
- key: cliproxy-model-health-state-${{ github.run_id }}
149
- - name: Keep structured report
150
- if: ${{ always() && hashFiles('model-health-report/model-health.json') != '' }}
151
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
152
- with:
153
- name: model-health-${{ env.PROFILE }}-${{ github.run_id }}
154
- path: model-health-report
155
- retention-days: 30
156
- - name: Track findings in one issue per scope
157
- # Opens or updates a single tracking issue when the findings change, and closes it when a full
158
- # run is healthy. The body is the redacted agent brief: structure and numbers, no prompts or keys.
159
- if: ${{ always() && hashFiles('model-health-report/model-health.json') != '' && inputs.model == '' && inputs.check == '' }}
160
- env:
161
- GH_TOKEN: ${{ github.token }}
162
- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
163
- run: |
164
- scope=live
165
- case "$PROFILE" in nightly | manual) scope=nightly ;; esac
166
- title="[ops] CLI proxy model health ($scope)"
167
- report=model-health-report/model-health.json
168
- status=$(jq -r .summary.status "$report")
169
- fingerprint=$(jq -r .summary.findings.fingerprint "$report")
170
- number=$(gh issue list --state open --search "\"$title\" in:title" --json number,title \
171
- --jq "map(select(.title==\"$title\"))[0].number // empty")
172
- if [ "$status" = unhealthy ] || [ "$status" = degraded ]; then
173
- {
174
- head -c 58000 model-health-report/agent-brief.md
175
- printf '\n\nRun: %s (artifact model-health-%s-%s)\n<!-- fingerprint: %s -->\n' "$RUN_URL" "$PROFILE" "$GITHUB_RUN_ID" "$fingerprint"
176
- } > model-health-report/issue.md
177
- if [ -z "$number" ]; then
178
- gh issue create --title "$title" --body-file model-health-report/issue.md
179
- elif ! gh issue view "$number" --json body --jq .body | grep -q "fingerprint: $fingerprint"; then
180
- gh issue edit "$number" --body-file model-health-report/issue.md
181
- gh issue comment "$number" --body "Findings changed: $RUN_URL"
182
- fi
183
- elif [ "$status" = healthy ] && [ -n "$number" ]; then
184
- gh issue close "$number" --comment "Healthy again: $RUN_URL"
185
- fi
54
+ uses: crvouga/dev-fleet/.github/workflows/cliproxy-health.yml@RELEASE_TAG
55
+ with:
56
+ # Inference only on the hourly schedule; the 15-minute one is free.
57
+ profile: ${{ github.event_name == 'workflow_dispatch' && inputs.profile || github.event.schedule == '41 3 * * *' && 'nightly' || github.event.schedule == '17 * * * *' && 'light' || 'readiness' }}
58
+ suite: ${{ inputs.suite || 'all' }}
59
+ model: ${{ inputs.model || '' }}
60
+ check: ${{ inputs.check || '' }}
61
+ max-requests: ${{ inputs.max_requests || '' }}
62
+ force-probe: ${{ inputs.force_probe == true }}