@crvouga/dev-fleet 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +265 -37
- package/agents/setup.md +42 -12
- package/ansible/macos/manager.yml +10 -7
- package/ansible/macos/policy.json +2 -10
- package/ansible/macos/t3-policy.yml +49 -0
- package/ansible/monitoring/README.md +86 -25
- package/ansible/monitoring/agent.yml +40 -2
- package/ansible/monitoring/hub-route-darwin.yml +51 -0
- package/ansible/monitoring/lock.json +5 -0
- package/ansible/pyproject.toml +12 -0
- package/ansible/ubuntu/baseline.yml +1 -1
- package/ansible/ubuntu/disk-gc.py +336 -0
- package/ansible/ubuntu/policy.json +13 -3
- package/ansible/ubuntu/t3-desktop.yml +19 -0
- package/ansible/ubuntu/t3-health.sh +93 -22
- package/ansible/ubuntu/t3-idle.py +42 -0
- package/ansible/ubuntu/t3-policy.yml +4 -11
- package/ansible/ubuntu/tailscale.yml +8 -5
- package/ansible/ubuntu/tasks/recovery.yml +60 -1
- package/ansible/ubuntu/tasks/resources.yml +61 -1
- package/ansible/ubuntu/tasks/t3-restart.yml +37 -0
- package/ansible/ubuntu/wip-guard.py +121 -0
- package/ansible/uv.lock +190 -0
- package/bootstrap/bootstrap.sh +25 -18
- package/bootstrap/doctor.ts +46 -1
- package/bootstrap/fleet-manager-run.sh +42 -15
- package/bootstrap/fleet-remote-bootstrap.sh +85 -12
- package/bootstrap/global-config.ts +64 -6
- package/bootstrap/install-launcher.sh +3 -0
- package/bootstrap/native-t3.ts +22 -11
- package/bootstrap/npm.ts +10 -2
- package/bootstrap/release-version.ts +157 -0
- package/cliproxy/README.md +3 -3
- package/cliproxy/scripts/accounts.py +216 -0
- package/cliproxy/scripts/checkpoint.py +29 -1
- package/cliproxy/scripts/cli.py +1 -1
- package/cliproxy/scripts/clientkeys.py +99 -0
- package/cliproxy/scripts/lease.py +80 -0
- package/cliproxy/scripts/ops +137 -25
- package/cliproxy/scripts/restore-verify +15 -0
- package/docs/accounts.md +154 -0
- package/docs/consumer-setup.md +408 -0
- package/docs/declared.md +91 -0
- package/docs/durability.md +26 -0
- package/docs/machine-setup.md +400 -0
- package/docs/operations.md +276 -0
- package/docs/releasing.md +103 -0
- package/docs/state.md +254 -0
- package/docs/topology.md +69 -0
- package/docs/ubuntu-sync.md +357 -0
- package/docs/worker-contract.md +67 -0
- package/examples/fleet.json +5 -0
- package/llms.txt +36 -0
- package/mise.lock +29 -0
- package/mise.toml +15 -2
- package/package.json +11 -2
- package/schema/fleet.schema.json +137 -15
- package/src/accounts/checks.ts +220 -0
- package/src/accounts/context.ts +76 -0
- package/src/accounts/propagate.ts +62 -0
- package/src/accounts/roster.ts +85 -0
- package/src/accounts/selection.ts +67 -0
- package/src/accounts/sessions.ts +120 -0
- package/src/agents/account-relay.ts +176 -26
- package/src/agents/fleet-verify.ts +9 -0
- package/src/agents/launch.ts +84 -5
- package/src/agents/native-verify.ts +293 -22
- package/src/agents/provider-errors.ts +9 -2
- package/src/agents/registry.ts +7 -1
- package/src/cli/agent-guide.ts +39 -5
- package/src/cli/app.ts +11 -4
- package/src/cli/cli.ts +7 -0
- package/src/cli/command-index.ts +7 -0
- package/src/cli/fleet-accounts.ts +253 -0
- package/src/cli/fleet-state.ts +499 -0
- package/src/cli/fleet-sync-accounts.ts +58 -0
- package/src/cli/fleet-sync-checkout.ts +56 -17
- package/src/cli/fleet-sync-enrollment.ts +19 -23
- package/src/cli/fleet-sync-forward.ts +1 -1
- package/src/cli/fleet-sync-manager.ts +84 -12
- package/src/cli/fleet-sync-monitoring.ts +5 -5
- package/src/cli/fleet-sync-output.ts +7 -3
- package/src/cli/fleet-sync-remote.ts +31 -18
- package/src/cli/fleet-sync-state.ts +86 -0
- package/src/cli/fleet-sync.ts +159 -57
- package/src/cli/fleet.ts +4 -2
- package/src/cli/managed-health.ts +4 -4
- package/src/cli/node-sync.ts +7 -0
- package/src/cli/program.ts +20 -4
- package/src/cli/project.ts +51 -0
- package/src/cli/register-fleet.ts +86 -1
- package/src/cli/registry.ts +41 -1
- package/src/cli/roster.ts +46 -30
- package/src/cli/settings.ts +15 -12
- package/src/cli/source-operations.ts +27 -3
- package/src/cli/tailnet.ts +2 -2
- package/src/definition.ts +116 -4
- package/src/diagnostics.ts +1 -1
- package/src/fleet-config.ts +105 -19
- package/src/ide/definition.ts +86 -0
- package/src/ide/ide.ts +43 -0
- package/src/ide/index.ts +5 -0
- package/src/ide/orchestrate.ts +50 -0
- package/src/ide/t3.ts +111 -0
- package/src/machine/source.ts +3 -3
- package/src/machine/state.ts +15 -27
- package/src/manager-sync.ts +9 -18
- package/src/monitoring/agent-evidence.ts +146 -0
- package/src/monitoring/apply.ts +15 -3
- package/src/monitoring/beszel.ts +11 -4
- package/src/monitoring/enroll.ts +2 -1
- package/src/monitoring/hub-access.ts +4 -3
- package/src/monitoring/hub-backup.ts +121 -0
- package/src/monitoring/hub-viewer.ts +8 -8
- package/src/monitoring/hub.ts +3 -3
- package/src/monitoring/manager-hub.ts +30 -14
- package/src/node-converge-linux.ts +30 -7
- package/src/package-info.ts +45 -0
- package/src/platform/ansible.ts +19 -15
- package/src/platform/darwin.ts +0 -13
- package/src/platform/fallback.ts +0 -8
- package/src/platform/linux.ts +0 -10
- package/src/platform/resource-verify.ts +8 -4
- package/src/platform/t3-policy.ts +33 -6
- package/src/platform/t3-service.ts +104 -0
- package/src/platform/types.ts +0 -4
- package/src/platform/ubuntu-audit.ts +5 -2
- package/src/platform/windows.ts +0 -14
- package/src/remote-bootstrap.ts +17 -19
- package/src/remote-entry.ts +15 -1
- package/src/repair-catalog.ts +1 -1
- package/src/runner.ts +1 -1
- package/src/service-auth.ts +15 -5
- package/src/state/bucket.ts +208 -0
- package/src/state/check.ts +74 -0
- package/src/state/connection.ts +92 -0
- package/src/state/enrollment.ts +45 -0
- package/src/state/legacy.ts +89 -0
- package/src/state/litestream.ts +143 -0
- package/src/state/operator.ts +20 -0
- package/src/state/paths.ts +40 -0
- package/src/state/roster.ts +62 -0
- package/src/state/store.ts +169 -0
- package/src/state/sync.ts +352 -0
- package/src/t3/accounts.ts +7 -1
- package/src/t3/models.ts +133 -0
- package/src/t3/pairing.ts +31 -0
- package/src/t3/profile.ts +203 -0
- package/src/t3/version.ts +119 -0
- package/src/t3-endpoint.ts +64 -8
- package/src/t3-linked-check.ts +24 -2
- package/src/toolchain-checks.ts +25 -0
- package/src/transport.ts +3 -2
- package/src/types.ts +4 -0
- package/src/worker-access-checks.ts +56 -3
- package/src/worker-host-checks.ts +41 -13
- package/src/workstation-checks.ts +1 -1
- package/src/index.ts +0 -6
- package/src/test-fixtures.ts +0 -36
|
@@ -3,7 +3,11 @@ import type { Check, Runner } from '../types';
|
|
|
3
3
|
|
|
4
4
|
const AGENT_DROP_IN =
|
|
5
5
|
'/etc/systemd/user/t3code-.scope.d/60-personal-fleet.conf';
|
|
6
|
-
const GUARDS = [
|
|
6
|
+
const GUARDS = [
|
|
7
|
+
'systemd-oomd.service',
|
|
8
|
+
'personal-fleet-disk-guard.service',
|
|
9
|
+
'personal-fleet-disk-gc.timer',
|
|
10
|
+
];
|
|
7
11
|
const MIB = 1024 * 1024;
|
|
8
12
|
const FLOOR_MIB = policy.resourceSafety.diskGuardFloorMiB;
|
|
9
13
|
|
|
@@ -67,7 +71,7 @@ const diskAboveFloor =
|
|
|
67
71
|
if (free >= FLOOR_MIB * MIB) return undefined;
|
|
68
72
|
return [
|
|
69
73
|
'DISK_FREE_BELOW_FLOOR',
|
|
70
|
-
`${path} has ${Math.floor(free / MIB)} MiB free, under the ${FLOOR_MIB} MiB floor. The
|
|
74
|
+
`${path} has ${Math.floor(free / MIB)} MiB free, under the ${FLOOR_MIB} MiB floor. The disk GC already removed idle build output and caches; remove stale worktrees or data, or run "sudo systemctl start personal-fleet-disk-gc.service" and read its journal.`,
|
|
71
75
|
];
|
|
72
76
|
};
|
|
73
77
|
|
|
@@ -90,7 +94,7 @@ const tmpWritable: Probe = async (run) =>
|
|
|
90
94
|
|
|
91
95
|
/** Measured memory and disk safety on an Ubuntu worker: the running managers,
|
|
92
96
|
* not the files Ansible wrote. Free space below the guard floor fails because
|
|
93
|
-
* the
|
|
97
|
+
* what is left after the disk GC is work a person must decide about.
|
|
94
98
|
*/
|
|
95
99
|
export async function verifyResourceSafety(
|
|
96
100
|
home: string,
|
|
@@ -122,7 +126,7 @@ export async function verifyResourceSafety(
|
|
|
122
126
|
id: 'ubuntu:resource-safety',
|
|
123
127
|
status: 'pass',
|
|
124
128
|
summary:
|
|
125
|
-
'systemd-oomd and the disk
|
|
129
|
+
'systemd-oomd, the disk guard and the disk GC timer are running, user workloads are capped below physical memory, T3 agents have per-agent limits, swap is active, free disk is above the guard floor and /tmp is writable.',
|
|
126
130
|
},
|
|
127
131
|
];
|
|
128
132
|
}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { runAnsible } from './ansible';
|
|
2
2
|
import { desired } from '../t3/desired';
|
|
3
3
|
import policy from '../../ansible/ubuntu/t3/policy.json';
|
|
4
|
+
import ubuntu from '../../ansible/ubuntu/policy.json';
|
|
5
|
+
import { restartDeferral } from './t3-service';
|
|
4
6
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
5
7
|
import { rename, rm, symlink } from 'node:fs/promises';
|
|
6
8
|
import { userInfo } from 'node:os';
|
|
@@ -34,6 +36,7 @@ export async function stageT3Policy(
|
|
|
34
36
|
t3_policy_document: document,
|
|
35
37
|
development_user: user.username,
|
|
36
38
|
development_uid: user.uid,
|
|
39
|
+
development_home: user.homedir,
|
|
37
40
|
}),
|
|
38
41
|
timeoutMs: 120_000,
|
|
39
42
|
});
|
|
@@ -84,14 +87,21 @@ export const T3_RESTARTED_SUMMARY =
|
|
|
84
87
|
'T3 was restarted to load the new managed policy; the running server confirms it.';
|
|
85
88
|
|
|
86
89
|
/** `t3-policy.yml` restarts a running T3 that has not loaded the staged policy
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
+
* so every node offers the same providers, but only while it runs no agent
|
|
91
|
+
* turn: a busy T3 keeps its turns and the play records the restart as
|
|
92
|
+
* pending, for personal-worker-recovery to apply at the first idle check.
|
|
93
|
+
* This measures the outcome; `restarted` says whether the app restarted
|
|
94
|
+
* during this sync and so needs time to reconnect.
|
|
90
95
|
*/
|
|
91
96
|
export async function loadedT3Policy(
|
|
92
97
|
executable: string,
|
|
93
98
|
run: Runner,
|
|
94
|
-
options: {
|
|
99
|
+
options: {
|
|
100
|
+
restarted: boolean;
|
|
101
|
+
pause?: () => Promise<void>;
|
|
102
|
+
uid?: number;
|
|
103
|
+
now?: number;
|
|
104
|
+
}
|
|
95
105
|
): Promise<Check> {
|
|
96
106
|
// A restarted desktop app takes a few seconds to come back with its server.
|
|
97
107
|
const check = await verifyT3Policy(
|
|
@@ -103,8 +113,25 @@ export async function loadedT3Policy(
|
|
|
103
113
|
setTimeout(done, options.restarted ? 5000 : 1000)
|
|
104
114
|
))
|
|
105
115
|
);
|
|
106
|
-
if (check.status !== 'pass')
|
|
107
|
-
|
|
116
|
+
if (check.status !== 'pass') {
|
|
117
|
+
const deferral = await restartDeferral(
|
|
118
|
+
run,
|
|
119
|
+
options.uid ?? userInfo().uid,
|
|
120
|
+
options.now
|
|
121
|
+
);
|
|
122
|
+
if (!deferral.pending)
|
|
123
|
+
return { ...check, summary: 'T3_POLICY_RESTART_FAILED' };
|
|
124
|
+
return {
|
|
125
|
+
...check,
|
|
126
|
+
status: deferral.overdue ? 'fail' : 'warn',
|
|
127
|
+
summary: 'T3_POLICY_RESTART_PENDING',
|
|
128
|
+
remedy: {
|
|
129
|
+
action: deferral.overdue
|
|
130
|
+
? `Agent turns kept T3 from loading the staged policy for ${ubuntu.t3RestartMaxDeferralHours} hours. Finish or stop them, or restart t3code-desktop.service on the worker (this ends its running turns).`
|
|
131
|
+
: 'Agent turns are running, so T3 keeps them and loads the staged policy at the first check of personal-worker-recovery with no turn running.',
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
108
135
|
return options.restarted
|
|
109
136
|
? { ...check, summary: T3_RESTARTED_SUMMARY }
|
|
110
137
|
: check;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import policy from '../../ansible/ubuntu/policy.json';
|
|
1
2
|
import type { Runner, Check } from '../types';
|
|
2
3
|
|
|
3
4
|
/** The user unit that runs the pinned desktop app (UI and embedded server). */
|
|
@@ -44,6 +45,109 @@ export async function convergeT3Service(
|
|
|
44
45
|
];
|
|
45
46
|
}
|
|
46
47
|
|
|
48
|
+
/** The recovery script, installed outside every revision (`tasks/recovery.yml`). */
|
|
49
|
+
export const T3_HEALTH_SCRIPT = '/usr/local/lib/personal-fleet/t3-health.sh';
|
|
50
|
+
|
|
51
|
+
/** The record of the package the desktop app was unpacked from
|
|
52
|
+
* (`t3-desktop.yml`): `T3-Code-<version>-<arch>.deb`. */
|
|
53
|
+
export const T3_DESKTOP_RELEASE =
|
|
54
|
+
'/opt/T3 Code (Nightly)/.personal-fleet-version';
|
|
55
|
+
|
|
56
|
+
/** Where a play records a T3 restart it deferred because agent turns were
|
|
57
|
+
* running (`tasks/t3-restart.yml`): the second it was first deferred.
|
|
58
|
+
* personal-worker-recovery clears it when it restarts the idle app. */
|
|
59
|
+
export function t3RestartMarker(uid: number): string {
|
|
60
|
+
return `/run/user/${uid}/personal-t3-restart-pending`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** A restart deferred while agent turns run, and whether it has waited longer
|
|
64
|
+
* than the policy allows. */
|
|
65
|
+
export async function restartDeferral(
|
|
66
|
+
run: Runner,
|
|
67
|
+
uid: number,
|
|
68
|
+
now: number = Date.now()
|
|
69
|
+
): Promise<{ pending: boolean; overdue: boolean }> {
|
|
70
|
+
const marker = await run(['cat', t3RestartMarker(uid)]);
|
|
71
|
+
if (!marker.ok) return { pending: false, overdue: false };
|
|
72
|
+
const since = Number(marker.stdout.trim());
|
|
73
|
+
const limit = policy.t3RestartMaxDeferralHours * 3600;
|
|
74
|
+
return {
|
|
75
|
+
pending: true,
|
|
76
|
+
overdue: Number.isFinite(since) && now / 1000 - since > limit,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The release the running server reports, compared with the one installed.
|
|
81
|
+
* Asks the server itself, so it needs no `t3` on the search path. A new
|
|
82
|
+
* release is installed beside a running app, which restarts into it only
|
|
83
|
+
* while it runs no agent turn: until then this is T3_RESTART_PENDING, and a
|
|
84
|
+
* failure once the restart has waited longer than the policy allows. */
|
|
85
|
+
export async function runningVersionCheck(
|
|
86
|
+
run: Runner,
|
|
87
|
+
uid: number,
|
|
88
|
+
now: number = Date.now()
|
|
89
|
+
): Promise<Check> {
|
|
90
|
+
const server = await run(
|
|
91
|
+
[
|
|
92
|
+
'curl',
|
|
93
|
+
'--fail',
|
|
94
|
+
'--silent',
|
|
95
|
+
'--max-time',
|
|
96
|
+
'10',
|
|
97
|
+
'http://127.0.0.1:3773/.well-known/t3/environment',
|
|
98
|
+
],
|
|
99
|
+
{ timeoutMs: 15_000 }
|
|
100
|
+
);
|
|
101
|
+
const installed = await run(['cat', T3_DESKTOP_RELEASE]);
|
|
102
|
+
const running = serverVersion(server.ok ? server.stdout : '');
|
|
103
|
+
const release = /^T3-Code-(.+)-[a-z0-9]+\.deb$/.exec(
|
|
104
|
+
installed.ok ? installed.stdout.trim() : ''
|
|
105
|
+
)?.[1];
|
|
106
|
+
const id = 't3:running-version';
|
|
107
|
+
if (!running || !release)
|
|
108
|
+
return {
|
|
109
|
+
id,
|
|
110
|
+
status: 'fail',
|
|
111
|
+
summary: 'T3_RUNNING_VERSION_UNKNOWN',
|
|
112
|
+
remedy: {
|
|
113
|
+
action:
|
|
114
|
+
'The running server or the installed release could not be read. Check worker:t3-health, then rerun fleet-sync to reinstall the pinned desktop app.',
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
if (running === release)
|
|
118
|
+
return {
|
|
119
|
+
id,
|
|
120
|
+
status: 'pass',
|
|
121
|
+
summary: `The running T3 server is the installed release ${release}.`,
|
|
122
|
+
};
|
|
123
|
+
const deferral = await restartDeferral(run, uid, now);
|
|
124
|
+
return {
|
|
125
|
+
id,
|
|
126
|
+
status: deferral.overdue ? 'fail' : 'warn',
|
|
127
|
+
summary: 'T3_RESTART_PENDING',
|
|
128
|
+
remedy: {
|
|
129
|
+
action: deferral.overdue
|
|
130
|
+
? `T3 still runs ${running} after ${policy.t3RestartMaxDeferralHours} hours because agent turns kept running. Finish or stop them, or restart ${T3_DESKTOP_UNIT} on the worker (this ends its running turns).`
|
|
131
|
+
: `T3 runs ${running}; ${release} is installed. personal-worker-recovery restarts it at the first check with no agent turn running.`,
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function serverVersion(body: string): string | undefined {
|
|
137
|
+
try {
|
|
138
|
+
const value: unknown = JSON.parse(body);
|
|
139
|
+
const version =
|
|
140
|
+
typeof value === 'object' && value !== null
|
|
141
|
+
? (value as Record<string, unknown>)['serverVersion']
|
|
142
|
+
: undefined;
|
|
143
|
+
return typeof version === 'string' && /^[\w.+-]+$/.test(version)
|
|
144
|
+
? version
|
|
145
|
+
: undefined;
|
|
146
|
+
} catch {
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
47
151
|
/** When the desktop app last started; changes exactly when it restarts. */
|
|
48
152
|
export async function t3StartedAt(run: Runner): Promise<string> {
|
|
49
153
|
const shown = await run([
|
package/src/platform/types.ts
CHANGED
|
@@ -110,10 +110,6 @@ export type Platform = {
|
|
|
110
110
|
|
|
111
111
|
/** App-bundled Tailscale CLI binaries that exist (used when `tailscale` is not on PATH). */
|
|
112
112
|
tailscaleAppBinaries(): readonly string[];
|
|
113
|
-
/** VS Code user `settings.json`, or `null` when unknown on this OS. */
|
|
114
|
-
vscodeSettingsPath(): string | null;
|
|
115
|
-
/** VS Code `code` CLI binaries shipped inside the installed app. */
|
|
116
|
-
vscodeCliBinaries(): readonly string[];
|
|
117
113
|
|
|
118
114
|
/** Install (or replace) a periodic job. Never throws — returns a result. */
|
|
119
115
|
installScheduledJob(job: ScheduledJob): NotifierTestResult;
|
|
@@ -31,12 +31,15 @@ export async function auditUbuntu(run: Runner): Promise<UbuntuAudit> {
|
|
|
31
31
|
}
|
|
32
32
|
const version =
|
|
33
33
|
/^VERSION_ID="?([0-9]+\.[0-9]+)"?$/m.exec(release.stdout)?.[1] ?? null;
|
|
34
|
-
const
|
|
34
|
+
const reported = arch.stdout.trim();
|
|
35
|
+
const architecture = policy.architectures.includes(reported)
|
|
36
|
+
? reported
|
|
37
|
+
: null;
|
|
35
38
|
return {
|
|
36
39
|
supported:
|
|
37
40
|
release.ok &&
|
|
38
41
|
/^ID=ubuntu$/m.test(release.stdout) &&
|
|
39
|
-
version ===
|
|
42
|
+
version === policy.release &&
|
|
40
43
|
architecture !== null,
|
|
41
44
|
release: version,
|
|
42
45
|
architecture,
|
package/src/platform/windows.ts
CHANGED
|
@@ -110,20 +110,6 @@ export class WindowsPlatform implements Platform {
|
|
|
110
110
|
);
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
-
vscodeSettingsPath(): string | null {
|
|
114
|
-
const base =
|
|
115
|
-
process.env['APPDATA']?.trim() || join(homedir(), 'AppData/Roaming');
|
|
116
|
-
return join(base, 'Code/User/settings.json');
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
vscodeCliBinaries(): readonly string[] {
|
|
120
|
-
const base =
|
|
121
|
-
process.env['LOCALAPPDATA']?.trim() || join(homedir(), 'AppData/Local');
|
|
122
|
-
return [join(base, 'Programs/Microsoft VS Code/bin/code.cmd')].filter((p) =>
|
|
123
|
-
existsSync(p)
|
|
124
|
-
);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
113
|
installScheduledJob(job: ScheduledJob): NotifierTestResult {
|
|
128
114
|
return {
|
|
129
115
|
ok: false,
|
package/src/remote-bootstrap.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { isRevision } from './machine/state';
|
|
|
6
6
|
import { sshArgs, shellQuote, sshFailure } from './ssh';
|
|
7
7
|
import type { FleetNode, NodeReport, Runner, Check } from './types';
|
|
8
8
|
import { record } from './config';
|
|
9
|
-
import type
|
|
9
|
+
import { installedDefinition, type FleetDefinition } from './definition';
|
|
10
10
|
import { cleanCheck, validCheck } from './protocol';
|
|
11
11
|
|
|
12
12
|
export interface BootstrapPayload {
|
|
@@ -48,16 +48,10 @@ export async function prepareBootstrap(
|
|
|
48
48
|
);
|
|
49
49
|
// The fleet configuration is data, not source: it travels beside the code.
|
|
50
50
|
return payload && options.definition
|
|
51
|
-
? { ...payload, config:
|
|
51
|
+
? { ...payload, config: installedDefinition(options.definition) }
|
|
52
52
|
: payload;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
/** Installs the approved fleet configuration where every launcher and job on
|
|
56
|
-
* the node reads it. It holds no secret.
|
|
57
|
-
*/
|
|
58
|
-
const INSTALL_CONFIG =
|
|
59
|
-
'umask 077; d="$HOME/.config/dev-fleet"; mkdir -p "$d" && cat >"$d/fleet.json.new" && mv "$d/fleet.json.new" "$d/fleet.json"';
|
|
60
|
-
|
|
61
55
|
/** Workers run one revision and never need history: the bundle holds only the
|
|
62
56
|
* approved commit and its files (about a quarter of the full history), and the
|
|
63
57
|
* receiving checkout is marked shallow at that commit. `git bundle create`
|
|
@@ -104,17 +98,9 @@ export async function bootstrapRemote(
|
|
|
104
98
|
): Promise<NodeReport> {
|
|
105
99
|
const cached = await cachedRevision(node, payload.revision, run);
|
|
106
100
|
if (typeof cached !== 'boolean') return { node: node.id, checks: [cached] };
|
|
107
|
-
if (payload.config) {
|
|
108
|
-
const installed = await run([...sshArgs(node), INSTALL_CONFIG], {
|
|
109
|
-
input: payload.config,
|
|
110
|
-
timeoutMs: 20_000,
|
|
111
|
-
});
|
|
112
|
-
if (!installed.ok)
|
|
113
|
-
return { node: node.id, checks: [sshFailure(installed)] };
|
|
114
|
-
}
|
|
115
101
|
const transfer = cached
|
|
116
|
-
? { mode: 'cached',
|
|
117
|
-
: { mode: 'transfer',
|
|
102
|
+
? { mode: 'cached', bundle: '' }
|
|
103
|
+
: { mode: 'transfer', bundle: payload.bundle };
|
|
118
104
|
const command = [
|
|
119
105
|
'sh',
|
|
120
106
|
'-c',
|
|
@@ -128,7 +114,7 @@ export async function bootstrapRemote(
|
|
|
128
114
|
.map(shellQuote)
|
|
129
115
|
.join(' ');
|
|
130
116
|
const result = await run([...sshArgs(node), command], {
|
|
131
|
-
input: transfer.
|
|
117
|
+
input: bootstrapInput(payload.config, transfer.bundle),
|
|
132
118
|
timeoutMs: 5_400_000,
|
|
133
119
|
});
|
|
134
120
|
const evidence = parseEvidence(result);
|
|
@@ -150,6 +136,18 @@ export async function bootstrapRemote(
|
|
|
150
136
|
};
|
|
151
137
|
}
|
|
152
138
|
|
|
139
|
+
/** What the bootstrap reads on stdin: one line with the fleet configuration
|
|
140
|
+
* (installed under the remote lock, so overlapping runs never both write it),
|
|
141
|
+
* then the bundle when the worker lacks the revision.
|
|
142
|
+
*/
|
|
143
|
+
export function bootstrapInput(
|
|
144
|
+
config: string | undefined,
|
|
145
|
+
bundle: string
|
|
146
|
+
): string {
|
|
147
|
+
const line = config ? Buffer.from(config).toString('base64') : '';
|
|
148
|
+
return `${line}\n${bundle}`;
|
|
149
|
+
}
|
|
150
|
+
|
|
153
151
|
async function cachedRevision(
|
|
154
152
|
node: FleetNode,
|
|
155
153
|
revision: string,
|
package/src/remote-entry.ts
CHANGED
|
@@ -6,6 +6,8 @@ import { syncWorkspace } from './sync';
|
|
|
6
6
|
import { repairWorkspace, validRepairs } from './repair-runner';
|
|
7
7
|
import { validSnapshot } from './protocol';
|
|
8
8
|
import type { Fingerprint } from './types';
|
|
9
|
+
import { machineProviders } from './t3/providers';
|
|
10
|
+
import { accountContext, nodeAccountChecks } from './accounts/context';
|
|
9
11
|
|
|
10
12
|
async function main(): Promise<void> {
|
|
11
13
|
const raw: unknown = JSON.parse(
|
|
@@ -36,6 +38,16 @@ async function main(): Promise<void> {
|
|
|
36
38
|
raw['expected'] as Fingerprint
|
|
37
39
|
);
|
|
38
40
|
process.stdout.write(JSON.stringify({ version: 1, checks }));
|
|
41
|
+
} else if (raw['action'] === 'accounts') {
|
|
42
|
+
// The fast account path: only the account-derived T3 instances. T3
|
|
43
|
+
// watches its settings, so no restart and no running turn ends.
|
|
44
|
+
if (raw['apply'] !== true)
|
|
45
|
+
throw new Error('Account propagation requires explicit authorization.');
|
|
46
|
+
const checks = [
|
|
47
|
+
await machineProviders(node.id, node.workspace, true),
|
|
48
|
+
...(await nodeAccountChecks(await accountContext())),
|
|
49
|
+
];
|
|
50
|
+
process.stdout.write(JSON.stringify({ version: 1, checks }));
|
|
39
51
|
} else if (raw['action'] === 'sync') {
|
|
40
52
|
if (raw['apply'] !== true)
|
|
41
53
|
throw new Error('Sync requires explicit authorization.');
|
|
@@ -84,6 +96,8 @@ async function configureRequest(
|
|
|
84
96
|
function validRequest(raw: unknown): raw is Record<string, unknown> {
|
|
85
97
|
return (
|
|
86
98
|
record(raw) &&
|
|
87
|
-
['probe', 'sync', 'repair', 'configure'].includes(
|
|
99
|
+
['probe', 'sync', 'repair', 'configure', 'accounts'].includes(
|
|
100
|
+
String(raw['action'])
|
|
101
|
+
)
|
|
88
102
|
);
|
|
89
103
|
}
|
package/src/repair-catalog.ts
CHANGED
|
@@ -47,7 +47,7 @@ export const REPAIR_RECIPES: RepairRecipe[] = [
|
|
|
47
47
|
owner: 'dev-fleet',
|
|
48
48
|
automatic: true,
|
|
49
49
|
action:
|
|
50
|
-
'Links ~/.local/bin/devfleet to this checkout. Nothing is copied and no configuration is changed.',
|
|
50
|
+
'Links ~/.local/bin/devfleet to this checkout of dev-fleet (refused for an installed package, which is not one). Nothing is copied and no configuration is changed.',
|
|
51
51
|
commands: [['sh', 'bootstrap/install-launcher.sh']],
|
|
52
52
|
},
|
|
53
53
|
{
|
package/src/runner.ts
CHANGED
package/src/service-auth.ts
CHANGED
|
@@ -42,6 +42,18 @@ export async function codingAuth(
|
|
|
42
42
|
];
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
/** CLIs with no sign-in of their own to check, and why. Each says where
|
|
46
|
+
* its access is proven instead; none is a silent skip.
|
|
47
|
+
*/
|
|
48
|
+
export const PROXY_ROUTED: Record<string, string> = {
|
|
49
|
+
gemini:
|
|
50
|
+
'gemini signs in through the CLI proxy and has no T3 launcher; provider:gemini:inference proves its route from this node.',
|
|
51
|
+
copilot:
|
|
52
|
+
'copilot is not routed through the CLI proxy and dev-fleet does not manage its GitHub sign-in; no provider account is checked.',
|
|
53
|
+
opencode:
|
|
54
|
+
'opencode signs in through the CLI proxy; agent:muse:session and the per-account agent:opencode sessions prove it from this node.',
|
|
55
|
+
};
|
|
56
|
+
|
|
45
57
|
export async function serviceAuth(run: Runner): Promise<Check[]> {
|
|
46
58
|
const services: Service[] = [
|
|
47
59
|
{
|
|
@@ -89,14 +101,12 @@ export async function serviceAuth(run: Runner): Promise<Check[]> {
|
|
|
89
101
|
];
|
|
90
102
|
const checks: Check[] = [];
|
|
91
103
|
for (const service of services) checks.push(await serviceCheck(service, run));
|
|
92
|
-
for (const id of
|
|
104
|
+
for (const [id, reason] of Object.entries(PROXY_ROUTED))
|
|
93
105
|
checks.push({
|
|
94
106
|
id: `auth:${id}`,
|
|
95
107
|
status: 'skip',
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
action: `Verify ${id} provider access locally using its supported account diagnostics. No prompt or billable request was made.`,
|
|
99
|
-
},
|
|
108
|
+
required: false,
|
|
109
|
+
summary: `AUTH_NOT_SUPPORTED_ON_ROUTE: ${reason}`,
|
|
100
110
|
});
|
|
101
111
|
return checks;
|
|
102
112
|
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import {
|
|
3
|
+
existsSync,
|
|
4
|
+
mkdirSync,
|
|
5
|
+
readFileSync,
|
|
6
|
+
renameSync,
|
|
7
|
+
rmSync,
|
|
8
|
+
statSync,
|
|
9
|
+
writeFileSync,
|
|
10
|
+
} from 'node:fs';
|
|
11
|
+
import { dirname } from 'node:path';
|
|
12
|
+
import { record } from '../config';
|
|
13
|
+
import { FleetUsageError } from '../errors';
|
|
14
|
+
|
|
15
|
+
/** Where the fleet's state is kept for good. An S3-compatible bucket (AWS S3,
|
|
16
|
+
* Cloudflare R2, Backblaze B2, MinIO) is the usual one; a directory on an
|
|
17
|
+
* attached disk or a mounted share works the same way.
|
|
18
|
+
*/
|
|
19
|
+
export type Bucket =
|
|
20
|
+
| {
|
|
21
|
+
kind: 's3';
|
|
22
|
+
bucket: string;
|
|
23
|
+
/** Omitted for AWS S3 itself. */
|
|
24
|
+
endpoint?: string;
|
|
25
|
+
region?: string;
|
|
26
|
+
/** Key prefix, so one bucket can hold more than this fleet's state. */
|
|
27
|
+
prefix: string;
|
|
28
|
+
/** Omitted to use the standard AWS environment or profile. */
|
|
29
|
+
accessKeyId?: string;
|
|
30
|
+
secretAccessKey?: string;
|
|
31
|
+
}
|
|
32
|
+
| { kind: 'directory'; path: string };
|
|
33
|
+
|
|
34
|
+
export const DEFAULT_PREFIX = 'dev-fleet';
|
|
35
|
+
|
|
36
|
+
const text = (value: unknown, pattern: RegExp): value is string =>
|
|
37
|
+
typeof value === 'string' && pattern.test(value);
|
|
38
|
+
|
|
39
|
+
const ENDPOINT =
|
|
40
|
+
/^(https:\/\/[A-Za-z0-9.-]{1,253}(:\d{1,5})?|http:\/\/(127\.0\.0\.1|localhost)(:\d{1,5})?)$/;
|
|
41
|
+
const KEY = /^[\x21-\x7e]{1,512}$/;
|
|
42
|
+
|
|
43
|
+
function invalid(reason: string): never {
|
|
44
|
+
throw new FleetUsageError(`Invalid bucket settings: ${reason}`);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function directory(value: Record<string, unknown>): Bucket {
|
|
48
|
+
if (Object.keys(value).some((key) => !['version', 'directory'].includes(key)))
|
|
49
|
+
invalid('a directory takes no other setting.');
|
|
50
|
+
const path = value['directory'];
|
|
51
|
+
if (!text(path, /^\/[^\x00-\x1f\x7f]{1,4000}$/))
|
|
52
|
+
invalid('directory must be an absolute path.');
|
|
53
|
+
return { kind: 'directory', path };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function prefix(value: unknown): string {
|
|
57
|
+
if (value === undefined) return DEFAULT_PREFIX;
|
|
58
|
+
if (
|
|
59
|
+
!text(value, /^[A-Za-z0-9][A-Za-z0-9._/-]{0,199}$/) ||
|
|
60
|
+
value.includes('..') ||
|
|
61
|
+
value.endsWith('/')
|
|
62
|
+
)
|
|
63
|
+
invalid('prefix must be a plain key prefix such as dev-fleet.');
|
|
64
|
+
return value;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function credentials(
|
|
68
|
+
value: Record<string, unknown>
|
|
69
|
+
): { accessKeyId: string; secretAccessKey: string } | Record<string, never> {
|
|
70
|
+
const { accessKeyId, secretAccessKey } = value;
|
|
71
|
+
if (accessKeyId === undefined && secretAccessKey === undefined) return {};
|
|
72
|
+
if (!text(accessKeyId, KEY) || !text(secretAccessKey, KEY))
|
|
73
|
+
invalid('accessKeyId and secretAccessKey must be given together.');
|
|
74
|
+
return { accessKeyId, secretAccessKey };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const S3_SETTINGS = [
|
|
78
|
+
'version',
|
|
79
|
+
'bucket',
|
|
80
|
+
'endpoint',
|
|
81
|
+
'region',
|
|
82
|
+
'prefix',
|
|
83
|
+
'accessKeyId',
|
|
84
|
+
'secretAccessKey',
|
|
85
|
+
];
|
|
86
|
+
|
|
87
|
+
function s3(value: Record<string, unknown>): Bucket {
|
|
88
|
+
if (Object.keys(value).some((key) => !S3_SETTINGS.includes(key)))
|
|
89
|
+
invalid('unknown setting.');
|
|
90
|
+
const { bucket, endpoint, region } = value;
|
|
91
|
+
if (!text(bucket, /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/))
|
|
92
|
+
invalid('bucket must be a bucket name.');
|
|
93
|
+
if (endpoint !== undefined && !text(endpoint, ENDPOINT))
|
|
94
|
+
invalid('endpoint must be an https:// origin without a path.');
|
|
95
|
+
if (region !== undefined && !text(region, /^[a-z0-9-]{1,32}$/))
|
|
96
|
+
invalid('region must be a region name such as auto or us-east-1.');
|
|
97
|
+
return {
|
|
98
|
+
kind: 's3',
|
|
99
|
+
bucket,
|
|
100
|
+
...(endpoint ? { endpoint } : {}),
|
|
101
|
+
...(region ? { region } : {}),
|
|
102
|
+
prefix: prefix(value['prefix']),
|
|
103
|
+
...credentials(value),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The contents of bucket.json, or the same fields from `fleet-state --init`. */
|
|
108
|
+
export function parseBucket(value: unknown): Bucket {
|
|
109
|
+
if (!record(value)) return invalid('expected a JSON object.');
|
|
110
|
+
if (value['version'] !== undefined && value['version'] !== 1)
|
|
111
|
+
invalid('unsupported version.');
|
|
112
|
+
return value['directory'] !== undefined ? directory(value) : s3(value);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** The environment a devfleet command takes the bucket's credentials from. */
|
|
116
|
+
export const ACCESS_KEY_ID_ENV = 'DEVFLEET_BUCKET_ACCESS_KEY_ID';
|
|
117
|
+
export const SECRET_ACCESS_KEY_ENV = 'DEVFLEET_BUCKET_SECRET_ACCESS_KEY';
|
|
118
|
+
|
|
119
|
+
/** Where a fleet configuration says its state is kept: a bucket's location
|
|
120
|
+
* without credentials, which never belong in a file kept in Git.
|
|
121
|
+
*/
|
|
122
|
+
export type BucketLocation =
|
|
123
|
+
| { bucket: string; endpoint?: string; region?: string; prefix?: string }
|
|
124
|
+
| { directory: string };
|
|
125
|
+
|
|
126
|
+
const LOCATION = ['bucket', 'endpoint', 'region', 'prefix', 'directory'];
|
|
127
|
+
|
|
128
|
+
/** The `state` block of a fleet configuration. */
|
|
129
|
+
export function parseLocation(value: unknown): Bucket {
|
|
130
|
+
if (!record(value)) return invalid('expected a JSON object.');
|
|
131
|
+
if ('accessKeyId' in value || 'secretAccessKey' in value)
|
|
132
|
+
invalid(
|
|
133
|
+
`credentials do not belong in the fleet configuration. Set ${ACCESS_KEY_ID_ENV} and ${SECRET_ACCESS_KEY_ENV} instead.`
|
|
134
|
+
);
|
|
135
|
+
if (Object.keys(value).some((key) => !LOCATION.includes(key)))
|
|
136
|
+
invalid('unknown setting.');
|
|
137
|
+
return parseBucket(value);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
type S3Bucket = Extract<Bucket, { kind: 's3' }>;
|
|
141
|
+
|
|
142
|
+
/** The same bucket, reached with these credentials instead of its own. */
|
|
143
|
+
export function withCredentials(
|
|
144
|
+
bucket: S3Bucket,
|
|
145
|
+
given: { accessKeyId?: string; secretAccessKey?: string }
|
|
146
|
+
): S3Bucket {
|
|
147
|
+
const { accessKeyId: _id, secretAccessKey: _secret, ...place } = bucket;
|
|
148
|
+
return { ...place, ...credentials(given) };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Null when this machine has no bucket. The file holds credentials, so one
|
|
152
|
+
* that other users can read is refused rather than used.
|
|
153
|
+
*/
|
|
154
|
+
export function readBucket(path: string): Bucket | null {
|
|
155
|
+
if (!existsSync(path)) return null;
|
|
156
|
+
if (process.platform !== 'win32' && (statSync(path).mode & 0o077) !== 0)
|
|
157
|
+
throw new FleetUsageError(
|
|
158
|
+
`${path} holds bucket credentials and must be private. Run chmod 600 on it.`
|
|
159
|
+
);
|
|
160
|
+
let value: unknown;
|
|
161
|
+
try {
|
|
162
|
+
value = JSON.parse(readFileSync(path, 'utf8'));
|
|
163
|
+
} catch {
|
|
164
|
+
return invalid('the file is not JSON.');
|
|
165
|
+
}
|
|
166
|
+
return parseBucket(value);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function writeBucket(path: string, bucket: Bucket): void {
|
|
170
|
+
const { kind, ...settings } = bucket;
|
|
171
|
+
const body =
|
|
172
|
+
kind === 'directory'
|
|
173
|
+
? { version: 1, directory: bucket.path }
|
|
174
|
+
: { version: 1, ...settings };
|
|
175
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
176
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
177
|
+
try {
|
|
178
|
+
writeFileSync(temporary, JSON.stringify(body, null, 2) + '\n', {
|
|
179
|
+
mode: 0o600,
|
|
180
|
+
flag: 'wx',
|
|
181
|
+
});
|
|
182
|
+
renameSync(temporary, path);
|
|
183
|
+
} finally {
|
|
184
|
+
rmSync(temporary, { force: true });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** What a report may say about the bucket: never a credential. */
|
|
189
|
+
export function describeBucket(bucket: Bucket | null): Record<string, unknown> {
|
|
190
|
+
if (!bucket) return { configured: false };
|
|
191
|
+
if (bucket.kind === 'directory')
|
|
192
|
+
return { configured: true, kind: 'directory', directory: bucket.path };
|
|
193
|
+
return {
|
|
194
|
+
configured: true,
|
|
195
|
+
kind: 's3',
|
|
196
|
+
bucket: bucket.bucket,
|
|
197
|
+
...(bucket.endpoint ? { endpoint: bucket.endpoint } : {}),
|
|
198
|
+
...(bucket.region ? { region: bucket.region } : {}),
|
|
199
|
+
prefix: bucket.prefix,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** The same target replicates the same database; a different one starts over. */
|
|
204
|
+
export function bucketIdentity(bucket: Bucket): string {
|
|
205
|
+
return bucket.kind === 'directory'
|
|
206
|
+
? `directory:${bucket.path}`
|
|
207
|
+
: `s3:${bucket.endpoint ?? 'aws'}/${bucket.bucket}/${bucket.prefix}`;
|
|
208
|
+
}
|