@crvouga/dev-fleet 0.1.0 → 0.2.0

package/src/state/paths.ts

@@ -0,0 +1,40 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+/** The one directory that holds the manager's fleet state. Everything in it is
+ * a local copy: the storage bucket is the source of truth, so the directory
+ * can be deleted and restored with `fleet-state --restore`. `DEVFLEET_STATE`
+ * names another directory (a test fixture, a second fleet).
+ */
+export function stateRoot(env = process.env, home = homedir()): string {
+  return env['DEVFLEET_STATE'] || join(home, '.local/share/dev-fleet/state');
+}
+
+/** The fleet state database Litestream replicates. */
+export const databasePath = (root: string): string => join(root, 'fleet.db');
+
+/** Records the bucket's head this copy last matched (see `standing`). */
+export const replicatedPath = (root: string): string =>
+  join(root, 'replicated.json');
+
+/** The definition as a file, for the subprocesses that take a path. Rewritten
+ * from the database; never edited.
+ */
+export const definitionPath = (root: string): string =>
+  join(root, 'definition.json');
+
+/** The bucket this manager replicates to, and its credentials. The only local
+ * file that is not a copy of something else. `DEVFLEET_BUCKET` names another.
+ */
+export function bucketPath(env = process.env, home = homedir()): string {
+  return env['DEVFLEET_BUCKET'] || join(home, '.config/dev-fleet/bucket.json');
+}
+
+/** The home directory whose earlier files (kept before the fleet state
+ * existed) may still be read, or null. A state directory named with
+ * `DEVFLEET_STATE` is another fleet's: this machine's earlier files are not
+ * part of it.
+ */
+export function legacyHome(env = process.env, home = homedir()): string | null {
+  return env['DEVFLEET_STATE'] ? null : home;
+}

package/src/state/roster.ts

@@ -0,0 +1,62 @@
+import { randomUUID } from 'node:crypto';
+import { lstat, mkdir, open, readFile, rename, unlink } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname } from 'node:path';
+import { FleetUsageError } from '../errors';
+import { adoptEarlier, readLegacy } from './legacy';
+import { databasePath, stateRoot } from './paths';
+import { readDocument, writeDocument } from './store';
+
+/** The roster kept in the fleet state: what `--config` means when no file is
+ * named. A named file (`--config PATH`, `.fleet.local.json`) stays a file.
+ */
+export const STATE_ROSTER = 'state:roster';
+
+/** The roster's text, or null when there is none. */
+export async function readRosterText(
+  path: string,
+  env = process.env,
+  home = homedir()
+): Promise<string | null> {
+  if (path === STATE_ROSTER)
+    return (
+      readDocument(databasePath(stateRoot(env, home)), 'roster') ??
+      readLegacy('roster', env, home)
+    );
+  try {
+    const info = await lstat(path);
+    if (!info.isFile() || info.size > 1024 * 1024)
+      throw new FleetUsageError(
+        'Roster must be a regular file smaller than 1 MiB; symlinks are not supported.'
+      );
+    return await readFile(path, 'utf8');
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+export async function writeRosterText(
+  path: string,
+  text: string,
+  env = process.env,
+  home = homedir()
+): Promise<void> {
+  if (path === STATE_ROSTER) {
+    adoptEarlier(env, home);
+    writeDocument(databasePath(stateRoot(env, home)), 'roster', text);
+    return;
+  }
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  const handle = await open(temporary, 'wx', 0o600);
+  try {
+    await handle.writeFile(text);
+    await handle.sync();
+    await handle.close();
+    await rename(temporary, path);
+  } finally {
+    await handle.close();
+    await unlink(temporary).catch(() => undefined);
+  }
+}

package/src/state/store.ts

@@ -0,0 +1,169 @@
+import { Database } from 'bun:sqlite';
+import { randomUUID } from 'node:crypto';
+import { chmodSync, existsSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { FleetUsageError } from '../errors';
+
+/** Everything the manager must not lose, as one JSON text each:
+ * `definition` is the fleet configuration (fleet.json), `roster` the enrolled
+ * connection to each worker, `enrollment` the approved and applied revision.
+ */
+export const DOCUMENTS = ['definition', 'roster', 'enrollment'] as const;
+export type DocumentName = (typeof DOCUMENTS)[number];
+
+/** One version of the whole database. Every change replaces it, so two copies
+ * hold the same state exactly when their heads are equal.
+ */
+export interface Head {
+  id: string;
+  sequence: number;
+  writtenAt: string;
+}
+
+export interface DocumentInfo {
+  name: DocumentName;
+  bytes: number;
+  updatedAt: string;
+}
+
+const MAX_DOCUMENT_BYTES = 1024 * 1024;
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS documents (
+  name TEXT PRIMARY KEY,
+  body TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+) STRICT;
+CREATE TABLE IF NOT EXISTS head (
+  one INTEGER PRIMARY KEY CHECK (one = 1),
+  id TEXT NOT NULL,
+  sequence INTEGER NOT NULL,
+  written_at TEXT NOT NULL
+) STRICT;
+`;
+
+function unreadable(): FleetUsageError {
+  return new FleetUsageError(
+    'The fleet state database cannot be read. It is a local copy: run devfleet fleet-state --restore to replace it from the bucket.'
+  );
+}
+
+/** Reads never create the database: a machine without one has no state.
+ * The connection is still opened for writing, because a WAL database that
+ * was just downloaded has no shared-memory file yet and a read-only
+ * connection may not create one.
+ */
+function reading<T>(file: string, read: (db: Database) => T): T | null {
+  if (!existsSync(file)) return null;
+  let db: Database | undefined;
+  try {
+    db = new Database(file, { readwrite: true, create: false });
+    return read(db);
+  } catch {
+    throw unreadable();
+  } finally {
+    db?.close();
+  }
+}
+
+/** Litestream replicates the write-ahead log, so the database stays in WAL
+ * mode; a second devfleet process waits for the first instead of failing.
+ */
+function writing<T>(file: string, write: (db: Database) => T): T {
+  mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
+  let db: Database | undefined;
+  try {
+    db = new Database(file, { create: true, readwrite: true });
+    chmodSync(file, 0o600);
+    db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;');
+    db.exec(SCHEMA);
+    return db.transaction(write).immediate(db);
+  } catch (error) {
+    if (error instanceof FleetUsageError) throw error;
+    throw new FleetUsageError(
+      'The fleet state database cannot be written. Check the state directory permissions and free disk space.'
+    );
+  } finally {
+    db?.close();
+  }
+}
+
+function tables(db: Database): boolean {
+  return (
+    (
+      db
+        .query(
+          "SELECT count(*) AS found FROM sqlite_master WHERE type = 'table' AND name IN ('documents', 'head')"
+        )
+        .get() as { found: number }
+    ).found === 2
+  );
+}
+
+export function readDocument(file: string, name: DocumentName): string | null {
+  return reading(file, (db) => {
+    if (!tables(db)) return null;
+    const row = db
+      .query('SELECT body FROM documents WHERE name = ?1')
+      .get(name) as { body: string } | null;
+    return row?.body ?? null;
+  });
+}
+
+export function listDocuments(file: string): DocumentInfo[] {
+  return (
+    reading(file, (db) => {
+      if (!tables(db)) return [];
+      const rows = db
+        .query(
+          'SELECT name, length(CAST(body AS BLOB)) AS bytes, updated_at AS updatedAt FROM documents ORDER BY name'
+        )
+        .all() as DocumentInfo[];
+      return rows.filter((row) => DOCUMENTS.includes(row.name));
+    }) ?? []
+  );
+}
+
+export function readHead(file: string): Head | null {
+  return reading(file, (db) => {
+    if (!tables(db)) return null;
+    return db
+      .query('SELECT id, sequence, written_at AS writtenAt FROM head')
+      .get() as Head | null;
+  });
+}
+
+/** Stores one document and advances the head. Writing the text already
+ * stored changes nothing, so repeating a command never looks like new state.
+ */
+export function writeDocument(
+  file: string,
+  name: DocumentName,
+  body: string
+): Head {
+  if (Buffer.byteLength(body) > MAX_DOCUMENT_BYTES)
+    throw new FleetUsageError(
+      'A fleet state document must be smaller than 1 MiB.'
+    );
+  return writing(file, (db) => {
+    const current = db
+      .query('SELECT body FROM documents WHERE name = ?1')
+      .get(name) as { body: string } | null;
+    const head = db
+      .query('SELECT id, sequence, written_at AS writtenAt FROM head')
+      .get() as Head | null;
+    if (head && current?.body === body) return head;
+    const next: Head = {
+      id: randomUUID(),
+      sequence: (head?.sequence ?? 0) + 1,
+      writtenAt: new Date().toISOString(),
+    };
+    db.query(
+      'INSERT INTO documents (name, body, updated_at) VALUES (?1, ?2, ?3) ON CONFLICT (name) DO UPDATE SET body = ?2, updated_at = ?3'
+    ).run(name, body, next.writtenAt);
+    db.query(
+      'INSERT INTO head (one, id, sequence, written_at) VALUES (1, ?1, ?2, ?3) ON CONFLICT (one) DO UPDATE SET id = ?1, sequence = ?2, written_at = ?3'
+    ).run(next.id, next.sequence, next.writtenAt);
+    return next;
+  });
+}

package/src/state/sync.ts

@@ -0,0 +1,352 @@
+import { randomUUID } from 'node:crypto';
+import {
+  chmodSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { basename, dirname, join } from 'node:path';
+import { record } from '../config';
+import { FleetUsageError } from '../errors';
+import { FleetLockError } from '../machine/lock-error';
+import { withMachineLock } from '../machine/state';
+import { runCommand } from '../runner';
+import type { Runner } from '../types';
+import { bucketIdentity, type Bucket } from './bucket';
+import { connection } from './connection';
+import { replicate, restore } from './litestream';
+import { databasePath, replicatedPath, stateRoot } from './paths';
+import { readHead, type Head } from './store';
+
+export interface StateContext {
+  root: string;
+  bucket: Bucket | null;
+  run: Runner;
+}
+
+export function stateContext(
+  run: Runner = runCommand,
+  env = process.env
+): StateContext {
+  return {
+    root: stateRoot(env),
+    bucket: connection(env).bucket,
+    run,
+  };
+}
+
+/** How this machine's copy compares with the bucket.
+ * - `local-only`: no bucket is configured; this copy is the only one.
+ * - `unreachable`: the bucket did not answer, or did not take an upload.
+ * - `empty`: neither side holds any state.
+ * - `unpublished`: the bucket holds nothing yet; this copy can seed it.
+ * - `in-sync`: both hold the same version.
+ * - `local-ahead`: this copy has changes the bucket has not received.
+ * - `bucket-ahead`: another manager changed the bucket; this copy is stale.
+ * - `diverged`: both changed. The bucket is the source of truth.
+ */
+export type Standing =
+  | 'local-only'
+  | 'unreachable'
+  | 'empty'
+  | 'unpublished'
+  | 'in-sync'
+  | 'local-ahead'
+  | 'bucket-ahead'
+  | 'diverged';
+
+interface Replicated {
+  bucket: string;
+  head: string;
+}
+
+/** The bucket's head as of the last time this copy matched it. A copy whose
+ * head still equals it has no changes of its own and may be replaced.
+ */
+function readReplicated(root: string, bucket: Bucket): Replicated | null {
+  try {
+    const value: unknown = JSON.parse(
+      readFileSync(replicatedPath(root), 'utf8')
+    );
+    return record(value) &&
+      value['bucket'] === bucketIdentity(bucket) &&
+      typeof value['head'] === 'string'
+      ? { bucket: value['bucket'], head: value['head'] }
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+function saveReplicated(root: string, bucket: Bucket, head: Head): void {
+  const path = replicatedPath(root);
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  writeFileSync(
+    temporary,
+    JSON.stringify({
+      version: 1,
+      bucket: bucketIdentity(bucket),
+      head: head.id,
+    }) + '\n',
+    { mode: 0o600 }
+  );
+  renameSync(temporary, path);
+}
+
+/** Litestream's own record of what it has uploaded, next to the database. */
+const bookkeeping = (database: string): string =>
+  join(dirname(database), `.${basename(database)}-litestream`);
+
+function discard(database: string): void {
+  for (const path of [database, `${database}-wal`, `${database}-shm`])
+    rmSync(path, { force: true });
+  rmSync(bookkeeping(database), { recursive: true, force: true });
+}
+
+const INCOMING = /^incoming-[0-9a-f-]{36}\.db/;
+
+type Remote =
+  | { status: 'unreachable' }
+  | { status: 'absent' }
+  | { status: 'present'; file: string; head: Head };
+
+/** Downloads the bucket's current version beside the local one. */
+async function fetchRemote(
+  context: StateContext & { bucket: Bucket },
+  install = true
+): Promise<Remote> {
+  const database = databasePath(context.root);
+  const file = join(context.root, `incoming-${randomUUID()}.db`);
+  const outcome = await restore(
+    { database, bucket: context.bucket, run: context.run, install },
+    file
+  );
+  if (outcome === 'absent') return { status: 'absent' };
+  try {
+    const head = outcome === 'restored' ? readHead(file) : null;
+    if (head) return { status: 'present', file, head };
+    discard(file);
+    // A replica that holds no version yet is the same as no replica.
+    return { status: outcome === 'restored' ? 'absent' : 'unreachable' };
+  } catch {
+    // A download that cannot be read says nothing about the bucket.
+    discard(file);
+    return { status: 'unreachable' };
+  }
+}
+
+function compare(
+  local: Head | null,
+  remote: Head,
+  replicated: Replicated | null
+): Standing {
+  if (!local) return 'bucket-ahead';
+  if (remote.id === local.id) return 'in-sync';
+  if (replicated?.head === local.id) return 'bucket-ahead';
+  if (replicated?.head === remote.id) return 'local-ahead';
+  return 'diverged';
+}
+
+interface Observation {
+  standing: Standing;
+  local: Head | null;
+  remote: Remote | null;
+}
+
+async function observe(
+  context: StateContext,
+  install = true
+): Promise<Observation> {
+  const local = readHead(databasePath(context.root));
+  const { bucket } = context;
+  if (!bucket) return { standing: 'local-only', local, remote: null };
+  const remote = await fetchRemote({ ...context, bucket }, install);
+  if (remote.status === 'unreachable')
+    return { standing: 'unreachable', local, remote };
+  if (remote.status === 'absent')
+    return { standing: local ? 'unpublished' : 'empty', local, remote };
+  const replicated = readReplicated(context.root, bucket);
+  return { standing: compare(local, remote.head, replicated), local, remote };
+}
+
+/** One devfleet command at a time compares or changes the state. Downloads a
+ * killed command left behind are removed while nothing else can be using one.
+ */
+async function locked<T>(
+  root: string,
+  operation: () => Promise<T>
+): Promise<T> {
+  try {
+    return await withMachineLock(root, async () => {
+      for (const name of readdirSync(root))
+        if (INCOMING.test(name)) rmSync(join(root, name), { force: true });
+      return operation();
+    });
+  } catch (error) {
+    if (error instanceof FleetLockError)
+      throw new FleetUsageError(
+        'Another devfleet command is using the fleet state. Wait for it to finish, then retry.'
+      );
+    throw error;
+  }
+}
+
+/** Replaces a stale copy with the bucket's version and records a copy that
+ * already matches. `force` replaces a copy that has changes of its own too.
+ */
+function adopt(
+  context: StateContext,
+  observation: Observation,
+  force: boolean
+): Standing {
+  const { remote, local, standing } = observation;
+  if (!context.bucket || remote?.status !== 'present') return standing;
+  if (standing === 'in-sync' && local)
+    saveReplicated(context.root, context.bucket, local);
+  if (standing === 'in-sync' || (standing !== 'bucket-ahead' && !force))
+    return standing;
+  const database = databasePath(context.root);
+  discard(database);
+  renameSync(remote.file, database);
+  chmodSync(database, 0o600);
+  saveReplicated(context.root, context.bucket, remote.head);
+  return 'in-sync';
+}
+
+async function observed(
+  context: StateContext,
+  force: boolean
+): Promise<Standing> {
+  const observation = await observe(context);
+  try {
+    return adopt(context, observation, force);
+  } finally {
+    if (observation.remote?.status === 'present')
+      discard(observation.remote.file);
+  }
+}
+
+/** Uploads this copy, then downloads it again: the bucket holds this version
+ * only once a restore from it returns this version.
+ */
+async function upload(
+  context: StateContext & { bucket: Bucket },
+  seed: boolean
+): Promise<Standing> {
+  const database = databasePath(context.root);
+  const head = readHead(database);
+  if (!head) return 'empty';
+  // A bucket with no history starts from a full copy, not from where the
+  // uploads to another bucket stopped.
+  if (seed) rmSync(bookkeeping(database), { recursive: true, force: true });
+  const uploaded = await replicate({
+    database,
+    bucket: context.bucket,
+    run: context.run,
+  });
+  const remote: Remote = uploaded
+    ? await fetchRemote(context)
+    : { status: 'unreachable' };
+  if (remote.status === 'present') discard(remote.file);
+  if (remote.status !== 'present' || remote.head.id !== head.id)
+    return 'unreachable';
+  saveReplicated(context.root, context.bucket, head);
+  return 'in-sync';
+}
+
+/** After a change made on top of `before`: uploads it when there is a bucket
+ * to upload to and this copy differs from what the bucket last matched.
+ */
+function settle(context: StateContext, before: Standing): Promise<Standing> {
+  const { bucket } = context;
+  if (!bucket || before === 'unreachable') return Promise.resolve(before);
+  const head = readHead(databasePath(context.root));
+  if (
+    before === 'in-sync' &&
+    readReplicated(context.root, bucket)?.head === head?.id
+  )
+    return Promise.resolve('in-sync');
+  return upload(
+    { ...context, bucket },
+    before === 'empty' || before === 'unpublished'
+  );
+}
+
+/** Read-only: compares this copy with the bucket and changes neither. */
+export function standing(context: StateContext): Promise<Standing> {
+  return locked(context.root, async () => {
+    const observation = await observe(context, false);
+    if (observation.remote?.status === 'present')
+      discard(observation.remote.file);
+    return observation.standing;
+  });
+}
+
+export interface Refreshed {
+  standing: Standing;
+  /** This copy was stale and is now the bucket's version. */
+  replaced: boolean;
+}
+
+/** Brings this copy up to the bucket before it is read or changed. Never
+ * uploads, and never replaces a copy that has changes of its own.
+ */
+export function refresh(context: StateContext): Promise<Refreshed> {
+  return locked(context.root, async () => {
+    const database = databasePath(context.root);
+    const before = readHead(database)?.id;
+    const standing = await observed(context, false);
+    return { standing, replaced: readHead(database)?.id !== before };
+  });
+}
+
+/** True when this copy has changes the bucket is not known to hold. Reads
+ * local files only.
+ */
+export function pending(context: StateContext): boolean {
+  const head = readHead(databasePath(context.root));
+  return Boolean(
+    context.bucket &&
+    head &&
+    readReplicated(context.root, context.bucket)?.head !== head.id
+  );
+}
+
+/** Replaces this copy with the bucket's version, whatever this copy holds. */
+export function pull(context: StateContext): Promise<Standing> {
+  return locked(context.root, () => observed(context, true));
+}
+
+/** Uploads this copy's changes, only when the bucket has not moved since
+ * this copy last matched it. A copy with no changes of its own is brought up
+ * to the bucket instead; a diverged one is left alone and reported.
+ */
+export function push(context: StateContext): Promise<Standing> {
+  return locked(context.root, async () => {
+    const before = await observed(context, false);
+    return ['unpublished', 'local-ahead'].includes(before)
+      ? settle(context, before)
+      : before;
+  });
+}
+
+export const DIVERGED_REMEDY =
+  'This machine and the bucket both changed the fleet state. The bucket is the source of truth: keep a copy of anything you still need (devfleet fleet-state --export), then run devfleet fleet-state --restore --yes.';
+
+/** Makes one change on top of the bucket's current version and uploads it.
+ * A bucket that cannot be reached does not stop the change: this copy stays
+ * ahead and the next change, or `fleet-state --push`, uploads it.
+ */
+export function mutateState<T>(
+  context: StateContext,
+  change: (before: Standing) => T | Promise<T>
+): Promise<{ value: T; standing: Standing }> {
+  return locked(context.root, async () => {
+    const before = await observed(context, false);
+    if (before === 'diverged') throw new FleetUsageError(DIVERGED_REMEDY);
+    const value = await change(before);
+    return { value, standing: await settle(context, before) };
+  });
+}