@crvouga/dev-fleet 0.1.0 → 0.3.0

package/src/state/store.ts

@@ -0,0 +1,169 @@
+import { Database } from 'bun:sqlite';
+import { randomUUID } from 'node:crypto';
+import { chmodSync, existsSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { FleetUsageError } from '../errors';
+
+/** Everything the manager must not lose, as one JSON text each:
+ * `definition` is the fleet configuration (fleet.json), `roster` the enrolled
+ * connection to each worker, `enrollment` the approved and applied revision.
+ */
+export const DOCUMENTS = ['definition', 'roster', 'enrollment'] as const;
+export type DocumentName = (typeof DOCUMENTS)[number];
+
+/** One version of the whole database. Every change replaces it, so two copies
+ * hold the same state exactly when their heads are equal.
+ */
+export interface Head {
+  id: string;
+  sequence: number;
+  writtenAt: string;
+}
+
+export interface DocumentInfo {
+  name: DocumentName;
+  bytes: number;
+  updatedAt: string;
+}
+
+const MAX_DOCUMENT_BYTES = 1024 * 1024;
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS documents (
+  name TEXT PRIMARY KEY,
+  body TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+) STRICT;
+CREATE TABLE IF NOT EXISTS head (
+  one INTEGER PRIMARY KEY CHECK (one = 1),
+  id TEXT NOT NULL,
+  sequence INTEGER NOT NULL,
+  written_at TEXT NOT NULL
+) STRICT;
+`;
+
+function unreadable(): FleetUsageError {
+  return new FleetUsageError(
+    'The fleet state database cannot be read. It is a local copy: run devfleet fleet-state --restore to replace it from the bucket.'
+  );
+}
+
+/** Reads never create the database: a machine without one has no state.
+ * The connection is still opened for writing, because a WAL database that
+ * was just downloaded has no shared-memory file yet and a read-only
+ * connection may not create one.
+ */
+function reading<T>(file: string, read: (db: Database) => T): T | null {
+  if (!existsSync(file)) return null;
+  let db: Database | undefined;
+  try {
+    db = new Database(file, { readwrite: true, create: false });
+    return read(db);
+  } catch {
+    throw unreadable();
+  } finally {
+    db?.close();
+  }
+}
+
+/** Litestream replicates the write-ahead log, so the database stays in WAL
+ * mode; a second devfleet process waits for the first instead of failing.
+ */
+function writing<T>(file: string, write: (db: Database) => T): T {
+  mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
+  let db: Database | undefined;
+  try {
+    db = new Database(file, { create: true, readwrite: true });
+    chmodSync(file, 0o600);
+    db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;');
+    db.exec(SCHEMA);
+    return db.transaction(write).immediate(db);
+  } catch (error) {
+    if (error instanceof FleetUsageError) throw error;
+    throw new FleetUsageError(
+      'The fleet state database cannot be written. Check the state directory permissions and free disk space.'
+    );
+  } finally {
+    db?.close();
+  }
+}
+
+function tables(db: Database): boolean {
+  return (
+    (
+      db
+        .query(
+          "SELECT count(*) AS found FROM sqlite_master WHERE type = 'table' AND name IN ('documents', 'head')"
+        )
+        .get() as { found: number }
+    ).found === 2
+  );
+}
+
+export function readDocument(file: string, name: DocumentName): string | null {
+  return reading(file, (db) => {
+    if (!tables(db)) return null;
+    const row = db
+      .query('SELECT body FROM documents WHERE name = ?1')
+      .get(name) as { body: string } | null;
+    return row?.body ?? null;
+  });
+}
+
+export function listDocuments(file: string): DocumentInfo[] {
+  return (
+    reading(file, (db) => {
+      if (!tables(db)) return [];
+      const rows = db
+        .query(
+          'SELECT name, length(CAST(body AS BLOB)) AS bytes, updated_at AS updatedAt FROM documents ORDER BY name'
+        )
+        .all() as DocumentInfo[];
+      return rows.filter((row) => DOCUMENTS.includes(row.name));
+    }) ?? []
+  );
+}
+
+export function readHead(file: string): Head | null {
+  return reading(file, (db) => {
+    if (!tables(db)) return null;
+    return db
+      .query('SELECT id, sequence, written_at AS writtenAt FROM head')
+      .get() as Head | null;
+  });
+}
+
+/** Stores one document and advances the head. Writing the text already
+ * stored changes nothing, so repeating a command never looks like new state.
+ */
+export function writeDocument(
+  file: string,
+  name: DocumentName,
+  body: string
+): Head {
+  if (Buffer.byteLength(body) > MAX_DOCUMENT_BYTES)
+    throw new FleetUsageError(
+      'A fleet state document must be smaller than 1 MiB.'
+    );
+  return writing(file, (db) => {
+    const current = db
+      .query('SELECT body FROM documents WHERE name = ?1')
+      .get(name) as { body: string } | null;
+    const head = db
+      .query('SELECT id, sequence, written_at AS writtenAt FROM head')
+      .get() as Head | null;
+    if (head && current?.body === body) return head;
+    const next: Head = {
+      id: randomUUID(),
+      sequence: (head?.sequence ?? 0) + 1,
+      writtenAt: new Date().toISOString(),
+    };
+    db.query(
+      'INSERT INTO documents (name, body, updated_at) VALUES (?1, ?2, ?3) ON CONFLICT (name) DO UPDATE SET body = ?2, updated_at = ?3'
+    ).run(name, body, next.writtenAt);
+    db.query(
+      'INSERT INTO head (one, id, sequence, written_at) VALUES (1, ?1, ?2, ?3) ON CONFLICT (one) DO UPDATE SET id = ?1, sequence = ?2, written_at = ?3'
+    ).run(next.id, next.sequence, next.writtenAt);
+    return next;
+  });
+}

package/src/state/sync.ts

@@ -0,0 +1,352 @@
+import { randomUUID } from 'node:crypto';
+import {
+  chmodSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { basename, dirname, join } from 'node:path';
+import { record } from '../config';
+import { FleetUsageError } from '../errors';
+import { FleetLockError } from '../machine/lock-error';
+import { withMachineLock } from '../machine/state';
+import { runCommand } from '../runner';
+import type { Runner } from '../types';
+import { bucketIdentity, type Bucket } from './bucket';
+import { connection } from './connection';
+import { replicate, restore } from './litestream';
+import { databasePath, replicatedPath, stateRoot } from './paths';
+import { readHead, type Head } from './store';
+
+export interface StateContext {
+  root: string;
+  bucket: Bucket | null;
+  run: Runner;
+}
+
+export function stateContext(
+  run: Runner = runCommand,
+  env = process.env
+): StateContext {
+  return {
+    root: stateRoot(env),
+    bucket: connection(env).bucket,
+    run,
+  };
+}
+
+/** How this machine's copy compares with the bucket.
+ * - `local-only`: no bucket is configured; this copy is the only one.
+ * - `unreachable`: the bucket did not answer, or did not take an upload.
+ * - `empty`: neither side holds any state.
+ * - `unpublished`: the bucket holds nothing yet; this copy can seed it.
+ * - `in-sync`: both hold the same version.
+ * - `local-ahead`: this copy has changes the bucket has not received.
+ * - `bucket-ahead`: another manager changed the bucket; this copy is stale.
+ * - `diverged`: both changed. The bucket is the source of truth.
+ */
+export type Standing =
+  | 'local-only'
+  | 'unreachable'
+  | 'empty'
+  | 'unpublished'
+  | 'in-sync'
+  | 'local-ahead'
+  | 'bucket-ahead'
+  | 'diverged';
+
+interface Replicated {
+  bucket: string;
+  head: string;
+}
+
+/** The bucket's head as of the last time this copy matched it. A copy whose
+ * head still equals it has no changes of its own and may be replaced.
+ */
+function readReplicated(root: string, bucket: Bucket): Replicated | null {
+  try {
+    const value: unknown = JSON.parse(
+      readFileSync(replicatedPath(root), 'utf8')
+    );
+    return record(value) &&
+      value['bucket'] === bucketIdentity(bucket) &&
+      typeof value['head'] === 'string'
+      ? { bucket: value['bucket'], head: value['head'] }
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+function saveReplicated(root: string, bucket: Bucket, head: Head): void {
+  const path = replicatedPath(root);
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  writeFileSync(
+    temporary,
+    JSON.stringify({
+      version: 1,
+      bucket: bucketIdentity(bucket),
+      head: head.id,
+    }) + '\n',
+    { mode: 0o600 }
+  );
+  renameSync(temporary, path);
+}
+
+/** Litestream's own record of what it has uploaded, next to the database. */
+const bookkeeping = (database: string): string =>
+  join(dirname(database), `.${basename(database)}-litestream`);
+
+function discard(database: string): void {
+  for (const path of [database, `${database}-wal`, `${database}-shm`])
+    rmSync(path, { force: true });
+  rmSync(bookkeeping(database), { recursive: true, force: true });
+}
+
+const INCOMING = /^incoming-[0-9a-f-]{36}\.db/;
+
+type Remote =
+  | { status: 'unreachable' }
+  | { status: 'absent' }
+  | { status: 'present'; file: string; head: Head };
+
+/** Downloads the bucket's current version beside the local one. */
+async function fetchRemote(
+  context: StateContext & { bucket: Bucket },
+  install = true
+): Promise<Remote> {
+  const database = databasePath(context.root);
+  const file = join(context.root, `incoming-${randomUUID()}.db`);
+  const outcome = await restore(
+    { database, bucket: context.bucket, run: context.run, install },
+    file
+  );
+  if (outcome === 'absent') return { status: 'absent' };
+  try {
+    const head = outcome === 'restored' ? readHead(file) : null;
+    if (head) return { status: 'present', file, head };
+    discard(file);
+    // A replica that holds no version yet is the same as no replica.
+    return { status: outcome === 'restored' ? 'absent' : 'unreachable' };
+  } catch {
+    // A download that cannot be read says nothing about the bucket.
+    discard(file);
+    return { status: 'unreachable' };
+  }
+}
+
+function compare(
+  local: Head | null,
+  remote: Head,
+  replicated: Replicated | null
+): Standing {
+  if (!local) return 'bucket-ahead';
+  if (remote.id === local.id) return 'in-sync';
+  if (replicated?.head === local.id) return 'bucket-ahead';
+  if (replicated?.head === remote.id) return 'local-ahead';
+  return 'diverged';
+}
+
+interface Observation {
+  standing: Standing;
+  local: Head | null;
+  remote: Remote | null;
+}
+
+async function observe(
+  context: StateContext,
+  install = true
+): Promise<Observation> {
+  const local = readHead(databasePath(context.root));
+  const { bucket } = context;
+  if (!bucket) return { standing: 'local-only', local, remote: null };
+  const remote = await fetchRemote({ ...context, bucket }, install);
+  if (remote.status === 'unreachable')
+    return { standing: 'unreachable', local, remote };
+  if (remote.status === 'absent')
+    return { standing: local ? 'unpublished' : 'empty', local, remote };
+  const replicated = readReplicated(context.root, bucket);
+  return { standing: compare(local, remote.head, replicated), local, remote };
+}
+
+/** One devfleet command at a time compares or changes the state. Downloads a
+ * killed command left behind are removed while nothing else can be using one.
+ */
+async function locked<T>(
+  root: string,
+  operation: () => Promise<T>
+): Promise<T> {
+  try {
+    return await withMachineLock(root, async () => {
+      for (const name of readdirSync(root))
+        if (INCOMING.test(name)) rmSync(join(root, name), { force: true });
+      return operation();
+    });
+  } catch (error) {
+    if (error instanceof FleetLockError)
+      throw new FleetUsageError(
+        'Another devfleet command is using the fleet state. Wait for it to finish, then retry.'
+      );
+    throw error;
+  }
+}
+
+/** Replaces a stale copy with the bucket's version and records a copy that
+ * already matches. `force` replaces a copy that has changes of its own too.
+ */
+function adopt(
+  context: StateContext,
+  observation: Observation,
+  force: boolean
+): Standing {
+  const { remote, local, standing } = observation;
+  if (!context.bucket || remote?.status !== 'present') return standing;
+  if (standing === 'in-sync' && local)
+    saveReplicated(context.root, context.bucket, local);
+  if (standing === 'in-sync' || (standing !== 'bucket-ahead' && !force))
+    return standing;
+  const database = databasePath(context.root);
+  discard(database);
+  renameSync(remote.file, database);
+  chmodSync(database, 0o600);
+  saveReplicated(context.root, context.bucket, remote.head);
+  return 'in-sync';
+}
+
+async function observed(
+  context: StateContext,
+  force: boolean
+): Promise<Standing> {
+  const observation = await observe(context);
+  try {
+    return adopt(context, observation, force);
+  } finally {
+    if (observation.remote?.status === 'present')
+      discard(observation.remote.file);
+  }
+}
+
+/** Uploads this copy, then downloads it again: the bucket holds this version
+ * only once a restore from it returns this version.
+ */
+async function upload(
+  context: StateContext & { bucket: Bucket },
+  seed: boolean
+): Promise<Standing> {
+  const database = databasePath(context.root);
+  const head = readHead(database);
+  if (!head) return 'empty';
+  // A bucket with no history starts from a full copy, not from where the
+  // uploads to another bucket stopped.
+  if (seed) rmSync(bookkeeping(database), { recursive: true, force: true });
+  const uploaded = await replicate({
+    database,
+    bucket: context.bucket,
+    run: context.run,
+  });
+  const remote: Remote = uploaded
+    ? await fetchRemote(context)
+    : { status: 'unreachable' };
+  if (remote.status === 'present') discard(remote.file);
+  if (remote.status !== 'present' || remote.head.id !== head.id)
+    return 'unreachable';
+  saveReplicated(context.root, context.bucket, head);
+  return 'in-sync';
+}
+
+/** After a change made on top of `before`: uploads it when there is a bucket
+ * to upload to and this copy differs from what the bucket last matched.
+ */
+function settle(context: StateContext, before: Standing): Promise<Standing> {
+  const { bucket } = context;
+  if (!bucket || before === 'unreachable') return Promise.resolve(before);
+  const head = readHead(databasePath(context.root));
+  if (
+    before === 'in-sync' &&
+    readReplicated(context.root, bucket)?.head === head?.id
+  )
+    return Promise.resolve('in-sync');
+  return upload(
+    { ...context, bucket },
+    before === 'empty' || before === 'unpublished'
+  );
+}
+
+/** Read-only: compares this copy with the bucket and changes neither. */
+export function standing(context: StateContext): Promise<Standing> {
+  return locked(context.root, async () => {
+    const observation = await observe(context, false);
+    if (observation.remote?.status === 'present')
+      discard(observation.remote.file);
+    return observation.standing;
+  });
+}
+
+export interface Refreshed {
+  standing: Standing;
+  /** This copy was stale and is now the bucket's version. */
+  replaced: boolean;
+}
+
+/** Brings this copy up to the bucket before it is read or changed. Never
+ * uploads, and never replaces a copy that has changes of its own.
+ */
+export function refresh(context: StateContext): Promise<Refreshed> {
+  return locked(context.root, async () => {
+    const database = databasePath(context.root);
+    const before = readHead(database)?.id;
+    const standing = await observed(context, false);
+    return { standing, replaced: readHead(database)?.id !== before };
+  });
+}
+
+/** True when this copy has changes the bucket is not known to hold. Reads
+ * local files only.
+ */
+export function pending(context: StateContext): boolean {
+  const head = readHead(databasePath(context.root));
+  return Boolean(
+    context.bucket &&
+    head &&
+    readReplicated(context.root, context.bucket)?.head !== head.id
+  );
+}
+
+/** Replaces this copy with the bucket's version, whatever this copy holds. */
+export function pull(context: StateContext): Promise<Standing> {
+  return locked(context.root, () => observed(context, true));
+}
+
+/** Uploads this copy's changes, only when the bucket has not moved since
+ * this copy last matched it. A copy with no changes of its own is brought up
+ * to the bucket instead; a diverged one is left alone and reported.
+ */
+export function push(context: StateContext): Promise<Standing> {
+  return locked(context.root, async () => {
+    const before = await observed(context, false);
+    return ['unpublished', 'local-ahead'].includes(before)
+      ? settle(context, before)
+      : before;
+  });
+}
+
+export const DIVERGED_REMEDY =
+  'This machine and the bucket both changed the fleet state. The bucket is the source of truth: keep a copy of anything you still need (devfleet fleet-state --export), then run devfleet fleet-state --restore --yes.';
+
+/** Makes one change on top of the bucket's current version and uploads it.
+ * A bucket that cannot be reached does not stop the change: this copy stays
+ * ahead and the next change, or `fleet-state --push`, uploads it.
+ */
+export function mutateState<T>(
+  context: StateContext,
+  change: (before: Standing) => T | Promise<T>
+): Promise<{ value: T; standing: Standing }> {
+  return locked(context.root, async () => {
+    const before = await observed(context, false);
+    if (before === 'diverged') throw new FleetUsageError(DIVERGED_REMEDY);
+    const value = await change(before);
+    return { value, standing: await settle(context, before) };
+  });
+}

package/src/t3/accounts.ts

@@ -46,6 +46,12 @@ const TOOLS: Record<
   },
 };
 
+/** The T3 instance of one account, and the T3 driver that runs its tool. */
+export const accountInstanceId = (account: Account): string =>
+  `personal_${slug(account.prefix)}`.slice(0, 64);
+export const accountDriver = (tool: string): string | undefined =>
+  TOOLS[tool]?.driver;
+
 /** One `<Provider> <account email>` instance per signed-in account, identical
  * on every machine because every machine reads the same proxy.
  */
@@ -61,7 +67,7 @@ export function accountInstances(
       ...variables,
       account: slug(account.email),
     };
-    instances[`personal_${slug(account.prefix)}`.slice(0, 64)] = {
+    instances[accountInstanceId(account)] = {
       driver: tool.driver,
       displayName: `${tool.label} ${account.email}`,
       enabled: true,

package/src/t3/models.ts

@@ -0,0 +1,133 @@
+import { join } from 'node:path';
+import type { Check } from '../types';
+import { accountDriver, accountInstanceId, type Roster } from './accounts';
+import type { Account } from '../agents/accounts';
+import { object, read } from './files';
+
+/** The models T3 offers for one instance: the catalogue T3 keeps for its
+ * driver (userdata/model-manifest.json) and any custom models the instance
+ * adds. Null when T3 has no list for the driver: it asks the tool itself.
+ */
+function offered(
+  manifest: Record<string, unknown>,
+  instance: unknown,
+  driver: string
+): string[] | null {
+  const catalogue = object(
+    JSON.stringify(
+      object(JSON.stringify(manifest['manifest'] ?? {}))['currentModels'] ?? {}
+    )
+  )[driver];
+  const config =
+    instance && typeof instance === 'object'
+      ? (instance as { config?: { customModels?: unknown } }).config
+      : undefined;
+  const custom = Array.isArray(config?.customModels)
+    ? config.customModels.flatMap((model: unknown) =>
+        typeof model === 'string'
+          ? [model]
+          : model &&
+              typeof model === 'object' &&
+              typeof (model as { slug?: unknown }).slug === 'string'
+            ? [(model as { slug: string }).slug]
+            : []
+      )
+    : [];
+  if (!Array.isArray(catalogue) && !custom.length) return null;
+  return [
+    ...new Set([
+      ...(Array.isArray(catalogue)
+        ? catalogue.filter(
+            (model): model is string => typeof model === 'string'
+          )
+        : []),
+      ...custom,
+    ]),
+  ].sort();
+}
+
+/** `t3:models`: for every account instance the fleet manages, T3 offers only
+ * models the proxy serves for that account, and every required model is both
+ * served and offered. A report names tools and models, never an account.
+ */
+export async function modelsCheck(options: {
+  t3Home: string;
+  roster: Roster;
+  required?: Record<string, string[]>;
+}): Promise<Check> {
+  const remedy = (action: string) => ({ remedy: { action } });
+  let accounts;
+  let settings;
+  let manifest;
+  try {
+    accounts = await options.roster();
+    settings = object(
+      await read(join(options.t3Home, 'userdata/settings.json'))
+    );
+    manifest = object(
+      await read(join(options.t3Home, 'userdata/model-manifest.json'))
+    );
+  } catch {
+    return {
+      id: 't3:models',
+      status: 'warn',
+      summary: 'T3_MODELS_UNVERIFIED',
+      ...remedy(
+        'The proxy’s model list or T3’s settings could not be read on this machine. Check that the CLI proxy answers and T3 has started once, then run fleet-sync.'
+      ),
+    };
+  }
+  const problems = mismatches(
+    accounts,
+    object(JSON.stringify(settings['providerInstances'] ?? {})),
+    manifest,
+    options.required ?? {}
+  );
+  if (problems.size)
+    return {
+      id: 't3:models',
+      status: 'warn',
+      summary: 'T3_MODELS_MISMATCH',
+      ...remedy(
+        `${[...problems].sort().join('; ')}. Serve the model on the CLI proxy, or remove it from t3.requiredModels in fleet.json; a model the proxy no longer serves fails when it is chosen in T3.`
+      ),
+    };
+  return {
+    id: 't3:models',
+    status: 'pass',
+    summary:
+      'Every model T3 offers for a managed account is one the proxy serves for it, and every required model is offered.',
+  };
+}
+
+/** What is wrong with the models of each managed account instance. */
+function mismatches(
+  accounts: Account[],
+  instances: Record<string, unknown>,
+  manifest: Record<string, unknown>,
+  required: Record<string, string[]>
+): Set<string> {
+  const problems = new Set<string>();
+  for (const account of accounts) {
+    const driver = accountDriver(account.tool);
+    const id = accountInstanceId(account);
+    if (!driver || !Object.hasOwn(instances, id)) continue;
+    const offers = offered(manifest, instances[id], driver);
+    for (const model of offers ?? [])
+      if (!account.models.includes(model))
+        problems.add(
+          `${account.tool}: T3 offers ${model}, which the proxy does not serve`
+        );
+    for (const model of required[account.tool] ?? []) {
+      if (!account.models.includes(model))
+        problems.add(
+          `${account.tool}: the proxy does not serve the required ${model}`
+        );
+      else if (offers && !offers.includes(model))
+        problems.add(
+          `${account.tool}: T3 does not offer the required ${model}`
+        );
+    }
+  }
+  return problems;
+}

package/src/t3/pairing.ts

@@ -0,0 +1,31 @@
+import { parseConfig } from '../config';
+import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
+import { mutateState, stateContext, type StateContext } from '../state/sync';
+
+/** Records the T3 environment a worker runs in the roster, the first time a
+ * verified link from the manager reaches it. From then on the link check
+ * compares against this identity, so a worker whose T3 data was replaced is
+ * told apart from one that is only offline. An identity already recorded is
+ * never rewritten.
+ */
+export async function rememberEnvironment(
+  path: string,
+  node: string,
+  environmentId: string,
+  context: () => StateContext = stateContext
+): Promise<boolean> {
+  const save = async (): Promise<boolean> => {
+    const text = await readRosterText(path);
+    if (text === null) return false;
+    const roster = parseConfig(JSON.parse(text) as unknown);
+    const entry = roster.nodes.find((candidate) => candidate.id === node);
+    if (!entry || entry.t3EnvironmentId) return false;
+    entry.t3EnvironmentId = environmentId;
+    // Parsing again refuses an identity that is not a T3 environment ID.
+    const next = parseConfig(roster);
+    await writeRosterText(path, `${JSON.stringify(next, null, 2)}\n`);
+    return true;
+  };
+  if (path !== STATE_ROSTER) return save();
+  return (await mutateState(context(), save)).value;
+}