@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/worker-activity.ts

@@ -0,0 +1,303 @@
+import { record } from './config';
+import type { FleetDefinition, MachineDefinition } from './definition';
+import { sshArgs } from './ssh';
+import type { Check, FleetNode, Runner } from './types';
+
+/** How long a worker has been up and what its agents are doing, as T3
+ * records it: the number of agent turns under way and when the last one was
+ * requested or ended. Never the content of a session.
+ */
+export interface WorkerActivity {
+  uptimeSeconds: number | null;
+  runningSessions: number | null;
+  lastActivityAt: string | null;
+}
+
+/** One worker in fleet-status: active or parked, what it does, and what it
+ * costs while it runs (only what fleet.json declares).
+ */
+export interface WorkerUsage extends Partial<WorkerActivity> {
+  id: string;
+  state: 'active' | 'parked';
+  parkedAt?: string;
+  reachable?: boolean;
+  /** Seconds since the last agent activity, or since boot without any. */
+  idleSeconds?: number;
+  idle?: boolean;
+  hourlyRate?: number;
+  currency?: string;
+  billingNote?: string;
+}
+
+/** The installed copy fleet-sync keeps outside every revision
+ * (ansible/ubuntu/tasks/recovery.yml).
+ */
+export const ACTIVITY_PROBE =
+  'cut -d" " -f1 /proc/uptime; python3 -I -B /usr/local/lib/personal-fleet/t3-idle.py --state "${T3CODE_HOME:-$HOME/.t3}/userdata/statev2.sqlite" --activity';
+
+/** Reads a worker's uptime and agent activity over SSH; null when the
+ * worker did not answer.
+ */
+export async function workerActivity(
+  node: FleetNode,
+  run: Runner
+): Promise<WorkerActivity | null> {
+  const result = await run([...sshArgs(node), ACTIVITY_PROBE], {
+    timeoutMs: 20_000,
+  });
+  if (!result.ok && !result.stdout) return null;
+  return parseActivity(result.stdout);
+}
+
+/** The JSON line of t3-idle.py --activity, or nothing known. */
+function activityLine(line: string | undefined): Record<string, unknown> {
+  let answer: unknown = null;
+  try {
+    answer = JSON.parse(line ?? '');
+  } catch {
+    answer = null;
+  }
+  return record(answer) && answer['version'] === 1 && !answer['unknown']
+    ? answer
+    : {};
+}
+
+/** What the probe printed: the uptime, then the one JSON line of
+ * t3-idle.py --activity. Anything unreadable is unknown, never zero.
+ */
+export function parseActivity(stdout: string): WorkerActivity {
+  const [uptime, line] = stdout.split('\n');
+  const seconds = Number(uptime);
+  const known = activityLine(line);
+  const running = known['running'];
+  const last = known['lastActivity'];
+  return {
+    uptimeSeconds:
+      uptime && Number.isFinite(seconds) && seconds >= 0
+        ? Math.floor(seconds)
+        : null,
+    runningSessions:
+      Number.isInteger(running) && Number(running) >= 0
+        ? (running as number)
+        : null,
+    lastActivityAt:
+      typeof last === 'string' && !Number.isNaN(Date.parse(last))
+        ? new Date(last).toISOString()
+        : null,
+  };
+}
+
+/** `2d 3h`, `5h 12m`, `40m`: a duration for a person. */
+export function duration(seconds: number): string {
+  const minutes = Math.floor(seconds / 60);
+  const days = Math.floor(minutes / 1440);
+  const hours = Math.floor((minutes % 1440) / 60);
+  if (days) return `${days}d ${hours}h`;
+  if (hours) return `${hours}h ${minutes % 60}m`;
+  return `${minutes}m`;
+}
+
+/** Seconds since the last agent activity, or since boot when there was
+ * none; null when it cannot be told.
+ */
+function idleFor(activity: WorkerActivity, now: Date): number | null {
+  if (activity.lastActivityAt)
+    return Math.max(
+      0,
+      Math.floor((now.getTime() - Date.parse(activity.lastActivityAt)) / 1000)
+    );
+  return activity.uptimeSeconds;
+}
+
+/** `worker:idle`: a warning once a worker has run no agent turn for longer
+ * than fleet.json's parking.idleAfterHours. A worker with a turn under way
+ * is never idle, and one whose T3 cannot be read is not judged.
+ */
+export function idleCheck(
+  id: string,
+  activity: WorkerActivity,
+  idleAfterHours: number | undefined,
+  now: Date
+): { check: Check; idleSeconds?: number; idle: boolean } {
+  if (activity.runningSessions === null)
+    return {
+      idle: false,
+      check: {
+        id: 'worker:idle',
+        status: 'skip',
+        summary: `${id}'s agent activity could not be read from its T3.`,
+      },
+    };
+  if (activity.runningSessions > 0)
+    return {
+      idle: false,
+      idleSeconds: 0,
+      check: {
+        id: 'worker:idle',
+        status: 'pass',
+        summary: `${id} runs ${activity.runningSessions} agent turn(s).`,
+      },
+    };
+  const seconds = idleFor(activity, now);
+  const limit = idleAfterHours === undefined ? null : idleAfterHours * 3600;
+  const idle = seconds !== null && limit !== null && seconds > limit;
+  return {
+    idle,
+    ...(seconds === null ? {} : { idleSeconds: seconds }),
+    check: idle
+      ? {
+          id: 'worker:idle',
+          status: 'warn',
+          summary: `${id} has run no agent turn for ${duration(seconds!)}, longer than the declared ${idleAfterHours}h.`,
+          remedy: {
+            action: `Park it while nobody needs it: devfleet fleet-node-park --id ${id}. devfleet fleet-node-resume --id ${id} brings it back.`,
+          },
+        }
+      : {
+          id: 'worker:idle',
+          status: 'pass',
+          summary:
+            seconds === null
+              ? `${id} runs no agent turn.`
+              : `${id} runs no agent turn; the last agent activity was ${duration(seconds)} ago.`,
+        },
+  };
+}
+
+function billing(worker: MachineDefinition | undefined) {
+  return worker?.billing
+    ? {
+        hourlyRate: worker.billing.hourlyRate,
+        ...(worker.billing.currency
+          ? { currency: worker.billing.currency }
+          : {}),
+        ...(worker.billing.note ? { billingNote: worker.billing.note } : {}),
+      }
+    : {};
+}
+
+/** Measures every active enrolled worker and lists the parked ones: the
+ * `workers` of fleet-status, and the checks it adds to each worker's report.
+ * Parked workers are not contacted.
+ */
+export async function fleetUsage(
+  definition: FleetDefinition,
+  {
+    nodes,
+    parked,
+    run,
+    now,
+  }: {
+    nodes: FleetNode[];
+    parked: Record<string, string>;
+    run: Runner;
+    now: Date;
+  }
+): Promise<{ workers: WorkerUsage[]; checks: Map<string, Check> }> {
+  const checks = new Map<string, Check>();
+  const workers = await Promise.all(
+    definition.workers.map(async (worker): Promise<WorkerUsage> => {
+      if (parked[worker.id])
+        return {
+          id: worker.id,
+          state: 'parked',
+          parkedAt: parked[worker.id]!,
+          ...billing(worker),
+        };
+      const node = nodes.find((entry) => entry.id === worker.id);
+      const activity = node ? await workerActivity(node, run) : null;
+      if (!activity)
+        return {
+          id: worker.id,
+          state: 'active',
+          reachable: false,
+          ...billing(worker),
+        };
+      const judged = idleCheck(
+        worker.id,
+        activity,
+        definition.parking?.idleAfterHours,
+        now
+      );
+      checks.set(worker.id, judged.check);
+      return {
+        id: worker.id,
+        state: 'active',
+        reachable: true,
+        ...activity,
+        ...(judged.idleSeconds === undefined
+          ? {}
+          : { idleSeconds: judged.idleSeconds }),
+        idle: judged.idle,
+        ...billing(worker),
+      };
+    })
+  );
+  return { workers, checks };
+}
+
+/** Workers running, workers idle, and what the idle ones cost per day, by
+ * the rates fleet.json declares.
+ */
+export function usageSummary(workers: WorkerUsage[]) {
+  const running = workers.filter(
+    (worker) => worker.state === 'active' && worker.reachable
+  );
+  const idle = running.filter((worker) => worker.idle);
+  const currencies = [
+    ...new Set(idle.flatMap((worker) => worker.currency ?? [])),
+  ];
+  return {
+    running: running.length,
+    idle: idle.length,
+    parked: workers.filter((worker) => worker.state === 'parked').length,
+    idleCostPerDay:
+      Math.round(
+        idle.reduce((sum, worker) => sum + (worker.hourlyRate ?? 0) * 24, 0) *
+          100
+      ) / 100,
+    ...(currencies.length === 1 ? { currency: currencies[0] } : {}),
+  };
+}
+
+/** The lines fleet-status prints about each worker and the fleet. */
+export function usageText(workers: WorkerUsage[], now: Date): string {
+  const summary = usageSummary(workers);
+  const lines = workers.map((worker) => {
+    if (worker.state === 'parked')
+      return `  ${worker.id}: parked for ${duration(Math.max(0, (now.getTime() - Date.parse(worker.parkedAt!)) / 1000))}`;
+    if (!worker.reachable) return `  ${worker.id}: not reached`;
+    const parts = [
+      worker.uptimeSeconds == null
+        ? 'uptime unknown'
+        : `up ${duration(worker.uptimeSeconds)}`,
+      worker.runningSessions == null
+        ? 'agent turns unknown'
+        : `${worker.runningSessions} agent turn(s) running`,
+      worker.lastActivityAt
+        ? `last agent activity ${worker.lastActivityAt}`
+        : 'no agent activity recorded',
+    ];
+    if (worker.idle) parts.push('idle');
+    if (worker.hourlyRate !== undefined)
+      parts.push(
+        `${worker.hourlyRate}${worker.currency ? ` ${worker.currency}` : ''}/h`
+      );
+    return `  ${worker.id}: ${parts.join(', ')}`;
+  });
+  return [
+    'Workers:',
+    ...lines,
+    `${summary.running} running, ${summary.idle} idle, ${summary.parked} parked; idle workers cost ${summary.idleCostPerDay}${summary.currency ? ` ${summary.currency}` : ''} per day as declared.`,
+  ].join('\n');
+}
+
+/** A report's text, followed by the worker lines when there are any. */
+export function withUsage(
+  text: string,
+  usage: { workers: WorkerUsage[] } | null
+): string {
+  return usage?.workers.length
+    ? `${text}\n${usageText(usage.workers, new Date())}`
+    : text;
+}

package/src/worker-capacity.ts

@@ -2,6 +2,8 @@ import { join } from 'node:path';
 import { clockCheck } from './clock';
 import { healthChecks, parseProbe } from './machine-health';
 import type { Check, Runner } from './types';
+import policy from '../ansible/ubuntu/policy.json';
+import { supportedSessions } from './worker-sizing';
 
 export async function workerCapacity(
   root: string,
@@ -35,6 +37,9 @@ export async function workerCapacity(
       summary: check.detail,
       ...(check.fix ? { remedy: { action: check.fix } } : {}),
     })),
+    ...(sample.memTotalMb === undefined
+      ? []
+      : [sessionsCheck(sample.memTotalMb, sample.agentSessions)]),
     clockCheck(
       'worker:clock',
       sample,
@@ -44,3 +49,29 @@ export async function workerCapacity(
     ),
   ];
 }
+
+const SESSIONS_EXHAUSTED = 'AGENT_SESSION_CAPACITY_EXHAUSTED';
+
+/** How many agent sessions this worker's memory policy supports, and how
+ * many run. The size of a session is an estimate (`sizing.agentSessionMiB`),
+ * so more sessions than that only warns.
+ */
+export function sessionsCheck(memoryMiB: number, running?: number): Check {
+  const supported = supportedSessions(memoryMiB);
+  const each = policy.sizing.agentSessionMiB;
+  const now = running === undefined ? 'an unknown number' : String(running);
+  const over = running !== undefined && running > supported;
+  return {
+    id: 'worker:agent-sessions',
+    status: over ? 'warn' : 'pass',
+    summary: `${over ? `${SESSIONS_EXHAUSTED}: ` : ''}${now} agent sessions running; the memory policy supports ${supported} at once on ${memoryMiB} MiB (an estimated ${each} MiB each).`,
+    ...(over
+      ? {
+          remedy: {
+            action:
+              'More agent sessions run than the memory policy supports: each is still capped, so the largest may be stopped under pressure. Finish some, or move work to a worker with more memory.',
+          },
+        }
+      : {}),
+  };
+}

package/src/worker-credentials.ts

@@ -0,0 +1,201 @@
+import { fleet } from './fleet-config';
+import { resolveVaultConfig, type VaultConfig } from './vault-config';
+import type { Check, Runner } from './types';
+
+/** Each worker's own Vault token is kept in a document of its own, beside
+ * the fleet's shared one: `<project>/workers/<id>`, field `token`, with the
+ * token's `accessor` so the manager can revoke it without holding it. A
+ * policy that grants one worker its document grants it nothing of another's.
+ */
+export interface WorkerTokenLocation {
+  mount: string;
+  path: string;
+  field: 'token';
+}
+
+export function workerTokenLocation(
+  node: string,
+  vault: VaultConfig = resolveVaultConfig()
+): WorkerTokenLocation {
+  return {
+    mount: vault.mount,
+    path: `${vault.project}/workers/${node}`,
+    field: 'token',
+  };
+}
+
+/** The Vault policy each worker token is issued with. */
+export const workerPolicyName = (node: string): string =>
+  `fleet-worker-${node}`;
+
+/** The least a worker needs: the fleet's shared document (the proxy key and
+ * what the providers read) and its own token document. Nothing grants a
+ * worker the documents of the others, so one worker's token cannot read
+ * another's. Token renewal and lookup come from Vault's default policy.
+ */
+export function workerVaultPolicy(
+  node: string,
+  vault: VaultConfig = resolveVaultConfig(),
+  shared: string = sharedDocument()
+): string {
+  const own = workerTokenLocation(node, vault);
+  return [
+    `# dev-fleet: the policy of the worker ${node}. Written by dev-fleet; do not edit.`,
+    `path "${vault.mount}/data/${shared}" {`,
+    '  capabilities = ["read"]',
+    '}',
+    `path "${own.mount}/data/${own.path}" {`,
+    '  capabilities = ["read"]',
+    '}',
+    '',
+  ].join('\n');
+}
+
+function sharedDocument(): string {
+  const secrets = fleet.secrets;
+  return secrets.provider === 'vault'
+    ? secrets.locations['proxy-client-key'].path
+    : '';
+}
+
+const TIMEOUT = { timeoutMs: 20_000 };
+
+/** Issues a worker its own periodic, orphan token under its own policy and
+ * keeps it in its own document. The token reaches `vault kv put` on standard
+ * input, never as an argument. Run on the manager, logged in to Vault.
+ */
+export async function issueWorkerToken(
+  node: string,
+  run: Runner,
+  vault: VaultConfig = resolveVaultConfig()
+): Promise<Check> {
+  const address = `-address=${vault.addr}`;
+  const policy = await run(
+    ['vault', 'policy', 'write', address, workerPolicyName(node), '-'],
+    { ...TIMEOUT, input: workerVaultPolicy(node, vault) }
+  );
+  if (!policy.ok) return credentialFailure('VAULT_WORKER_POLICY_FAILED');
+  const created = await run(
+    [
+      'vault',
+      'token',
+      'create',
+      address,
+      `-policy=${workerPolicyName(node)}`,
+      '-orphan',
+      '-period=768h',
+      `-display-name=${workerPolicyName(node)}`,
+      '-format=json',
+    ],
+    TIMEOUT
+  );
+  const issued = created.ok ? parseIssued(created.stdout) : null;
+  if (!issued) return credentialFailure('VAULT_WORKER_TOKEN_NOT_ISSUED');
+  const location = workerTokenLocation(node, vault);
+  const stored = await run(
+    [
+      'vault',
+      'kv',
+      'put',
+      address,
+      `-mount=${location.mount}`,
+      location.path,
+      'token=-',
+      `accessor=${issued.accessor}`,
+    ],
+    { ...TIMEOUT, input: issued.token }
+  );
+  if (!stored.ok) return credentialFailure('VAULT_WORKER_TOKEN_NOT_STORED');
+  return {
+    id: 'auth:worker-token',
+    status: 'pass',
+    summary: `${node} has a token of its own, readable by nothing but itself and the manager.`,
+  };
+}
+
+function parseIssued(
+  stdout: string
+): { token: string; accessor: string } | null {
+  try {
+    const auth = (JSON.parse(stdout) as { auth?: Record<string, unknown> })
+      .auth;
+    const token = auth?.['client_token'];
+    const accessor = auth?.['accessor'];
+    return typeof token === 'string' &&
+      typeof accessor === 'string' &&
+      /^[\x21-\x7e]{16,4096}$/.test(token) &&
+      /^[A-Za-z0-9._-]{8,256}$/.test(accessor)
+      ? { token, accessor }
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Revokes a worker's token by its accessor (the manager never holds the
+ * token to do so), then removes its document and its policy. A worker that
+ * has none already is revoked: running it twice changes nothing.
+ */
+export async function revokeWorkerToken(
+  node: string,
+  run: Runner,
+  vault: VaultConfig = resolveVaultConfig()
+): Promise<Check> {
+  const address = `-address=${vault.addr}`;
+  const location = workerTokenLocation(node, vault);
+  const read = await run(
+    [
+      'vault',
+      'kv',
+      'get',
+      address,
+      `-mount=${location.mount}`,
+      '-field=accessor',
+      location.path,
+    ],
+    TIMEOUT
+  );
+  const accessor = read.ok ? read.stdout.trim() : '';
+  if (accessor) {
+    if (!/^[A-Za-z0-9._-]{8,256}$/.test(accessor))
+      return credentialFailure('VAULT_WORKER_TOKEN_NOT_REVOKED');
+    const revoked = await run(
+      ['vault', 'token', 'revoke', address, '-accessor', accessor],
+      TIMEOUT
+    );
+    if (!revoked.ok) return credentialFailure('VAULT_WORKER_TOKEN_NOT_REVOKED');
+  }
+  await run(
+    [
+      'vault',
+      'kv',
+      'metadata',
+      'delete',
+      address,
+      `-mount=${location.mount}`,
+      location.path,
+    ],
+    TIMEOUT
+  );
+  await run(
+    ['vault', 'policy', 'delete', address, workerPolicyName(node)],
+    TIMEOUT
+  );
+  return {
+    id: 'auth:worker-token',
+    status: 'pass',
+    summary: `${node}'s token is revoked and its document and policy removed.`,
+  };
+}
+
+function credentialFailure(summary: string): Check {
+  return {
+    id: 'auth:worker-token',
+    status: 'fail',
+    summary,
+    remedy: {
+      action:
+        'Log the manager in to Vault with a token that may write policies, issue tokens and write the fleet mount, then run the command again.',
+    },
+  };
+}