@crvouga/dev-fleet 0.1.0 → 0.3.0

package/src/state/check.ts

@@ -0,0 +1,74 @@
+import type { Check, Status } from '../types';
+import { DIVERGED_REMEDY, type Standing } from './sync';
+
+const INIT =
+  'Name a bucket under "state" in fleet.json and pass its credentials in DEVFLEET_BUCKET_ACCESS_KEY_ID and DEVFLEET_BUCKET_SECRET_ACCESS_KEY (see docs/state.md); without a fleet.json, devfleet fleet-state --init --bucket NAME --endpoint https://HOST. Until then a lost manager loses the roster, the approved revision and the configuration last applied.';
+
+const FINDINGS: Record<
+  Standing,
+  { status: Status; summary: string; action?: string }
+> = {
+  'in-sync': {
+    status: 'pass',
+    summary: 'The bucket holds the fleet state this machine works from.',
+  },
+  'local-only': {
+    status: 'warn',
+    summary:
+      'No bucket is configured: this machine holds the only copy of the fleet state.',
+    action: INIT,
+  },
+  empty: {
+    status: 'warn',
+    summary: 'Neither this machine nor the bucket holds any fleet state yet.',
+    action:
+      'Run devfleet fleet-sync from the project that holds your fleet.json: it records the configuration, the roster and the approved revision. devfleet fleet-state --example prints a fleet.json to start from.',
+  },
+  unpublished: {
+    status: 'warn',
+    summary:
+      'The bucket holds no fleet state yet; this machine has the only copy.',
+    action: 'Upload it: devfleet fleet-state --push.',
+  },
+  'local-ahead': {
+    status: 'warn',
+    summary: 'This machine has fleet state the bucket has not received.',
+    action: 'Upload it: devfleet fleet-state --push.',
+  },
+  'bucket-ahead': {
+    status: 'warn',
+    summary:
+      'The bucket holds a newer fleet state than this machine; another manager changed it.',
+    action:
+      'The next fleet-sync takes it automatically; devfleet fleet-state --restore takes it now.',
+  },
+  unreachable: {
+    status: 'fail',
+    summary:
+      'The bucket did not answer, so the fleet state could not be compared or uploaded.',
+    action:
+      'Check the network, the bucket named under "state" in fleet.json (or saved in bucket.json) and its credentials: DEVFLEET_BUCKET_ACCESS_KEY_ID and DEVFLEET_BUCKET_SECRET_ACCESS_KEY for this command, or the ones devfleet fleet-state --init saved on this machine. Then run devfleet fleet-state --push, which also installs the pinned Litestream when it is missing (that needs mise, which the first fleet-sync on a machine installs). Details are withheld to protect credentials.',
+  },
+  diverged: {
+    status: 'fail',
+    summary: 'This machine and the bucket hold different fleet states.',
+    action: DIVERGED_REMEDY,
+  },
+};
+
+/** Whether the bucket holds this machine's fleet state. `advisory` reports a
+ * problem as a warning: for a run that succeeded in everything else.
+ */
+export function replicaCheck(
+  standing: Standing,
+  options: { advisory?: boolean } = {}
+): Check {
+  const finding = FINDINGS[standing];
+  return {
+    id: 'state:replica',
+    status:
+      options.advisory && finding.status === 'fail' ? 'warn' : finding.status,
+    summary: finding.summary,
+    ...(finding.action ? { remedy: { action: finding.action } } : {}),
+  };
+}

package/src/state/connection.ts

@@ -0,0 +1,92 @@
+import { homedir } from 'node:os';
+import { FleetUsageError } from '../errors';
+import { declaredState } from '../fleet-config';
+import {
+  ACCESS_KEY_ID_ENV,
+  SECRET_ACCESS_KEY_ENV,
+  bucketIdentity,
+  parseLocation,
+  readBucket,
+  withCredentials,
+  type Bucket,
+} from './bucket';
+import { bucketPath } from './paths';
+
+/** How a devfleet command reaches the bucket, and where each part came from.
+ * Never a credential itself.
+ */
+export interface Connection {
+  bucket: Bucket | null;
+  /** `configuration`: the `state` block of the fleet configuration.
+   * `saved`: this machine's bucket.json.
+   */
+  location: 'configuration' | 'saved' | null;
+  /** `environment`: the two `DEVFLEET_BUCKET_*` variables. `saved`: this
+   * machine's bucket.json. `ambient`: none given, so the standard AWS
+   * environment or profile.
+   */
+  credentials: 'environment' | 'saved' | 'ambient' | null;
+}
+
+/** The credentials passed to this command, or null. */
+export function environmentCredentials(
+  env: NodeJS.ProcessEnv
+): { accessKeyId: string; secretAccessKey: string } | null {
+  const accessKeyId = env[ACCESS_KEY_ID_ENV];
+  const secretAccessKey = env[SECRET_ACCESS_KEY_ENV];
+  if (!accessKeyId && !secretAccessKey) return null;
+  if (!accessKeyId || !secretAccessKey)
+    throw new FleetUsageError(
+      `Set both ${ACCESS_KEY_ID_ENV} and ${SECRET_ACCESS_KEY_ENV}, or neither. Credentials are never taken from arguments.`
+    );
+  return { accessKeyId, secretAccessKey };
+}
+
+/** The configuration says where the bucket is; credentials are passed to the
+ * command. A machine that runs unattended (the schedule, a hand-off from a
+ * worker) gets no environment, so `fleet-state --init` may save both there:
+ * a saved location is used when the configuration names none, and saved
+ * credentials only for the bucket they were saved for.
+ */
+export function resolveConnection(
+  declared: Bucket | null,
+  saved: Bucket | null,
+  env: NodeJS.ProcessEnv
+): Connection {
+  const target = declared ?? saved;
+  if (!target) return { bucket: null, location: null, credentials: null };
+  const location = declared ? 'configuration' : 'saved';
+  if (target.kind === 'directory')
+    return { bucket: target, location, credentials: null };
+  const given = environmentCredentials(env);
+  if (given)
+    return {
+      bucket: withCredentials(target, given),
+      location,
+      credentials: 'environment',
+    };
+  if (
+    saved?.kind === 's3' &&
+    saved.accessKeyId &&
+    bucketIdentity(saved) === bucketIdentity(target)
+  )
+    return {
+      bucket: withCredentials(target, saved),
+      location,
+      credentials: 'saved',
+    };
+  return {
+    bucket: withCredentials(target, {}),
+    location,
+    credentials: 'ambient',
+  };
+}
+
+export function connection(env = process.env, home = homedir()): Connection {
+  const declared = declaredState(env, home);
+  return resolveConnection(
+    declared ? parseLocation(declared) : null,
+    readBucket(bucketPath(env, home)),
+    env
+  );
+}

package/src/state/enrollment.ts

@@ -0,0 +1,45 @@
+import { existsSync } from 'node:fs';
+import { FleetUsageError } from '../errors';
+import {
+  parseEnrollment,
+  readEnrollment,
+  type Enrollment,
+} from '../machine/state';
+import { adoptEarlier } from './legacy';
+import { databasePath, stateRoot } from './paths';
+import { readDocument, writeDocument } from './store';
+
+/** The manager's approved and applied revision, from the fleet state. A
+ * machine that has no state database yet reads the file that held it before
+ * (`enrollment.json` in `earlier`); its first approval moves it.
+ */
+export async function loadEnrollment(
+  earlier: string | null,
+  env = process.env
+): Promise<Enrollment | null> {
+  const database = databasePath(stateRoot(env));
+  if (earlier && !existsSync(database)) return readEnrollment(earlier);
+  const stored = readDocument(database, 'enrollment');
+  if (stored === null) return null;
+  let value: unknown;
+  try {
+    value = JSON.parse(stored);
+  } catch {
+    value = null;
+  }
+  const enrollment = parseEnrollment(value);
+  if (!enrollment)
+    throw new FleetUsageError(
+      'Invalid enrollment in the fleet state. Restore it from the bucket with devfleet fleet-state --restore; do not enroll a replacement using another machine’s identity.'
+    );
+  return enrollment;
+}
+
+export function storeEnrollment(state: Enrollment, env = process.env): void {
+  adoptEarlier(env);
+  writeDocument(
+    databasePath(stateRoot(env)),
+    'enrollment',
+    JSON.stringify(state) + '\n'
+  );
+}

package/src/state/legacy.ts

@@ -0,0 +1,89 @@
+import { existsSync, lstatSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { parseConfig } from '../config';
+import { parseDefinition } from '../definition';
+import { parseEnrollment } from '../machine/state';
+import { databasePath, definitionPath, legacyHome, stateRoot } from './paths';
+import { DOCUMENTS, writeDocument, type DocumentName } from './store';
+
+/** Where each document was kept before the fleet state existed. A machine
+ * that has no state database yet still reads them; its first change moves
+ * all of them into the state, and nothing reads or writes them afterwards.
+ */
+const LEGACY: Record<DocumentName, string> = {
+  // The copy fleet-sync installs; it stays the launchers' copy afterwards.
+  definition: '.config/dev-fleet/fleet.json',
+  roster: '.config/personal/fleet.json',
+  enrollment: '.local/share/personal/fleet/manager/enrollment.json',
+};
+
+export function legacyPath(
+  name: DocumentName,
+  env = process.env,
+  home = homedir()
+): string | null {
+  const earlier = legacyHome(env, home);
+  return earlier ? join(earlier, LEGACY[name]) : null;
+}
+
+/** The earlier file's text, or null: when there is no such file, and once
+ * this machine has a state database, which then holds everything.
+ */
+export function readLegacy(
+  name: DocumentName,
+  env = process.env,
+  home = homedir()
+): string | null {
+  const path = legacyPath(name, env, home);
+  if (!path || existsSync(databasePath(stateRoot(env, home)))) return null;
+  try {
+    const info = lstatSync(path);
+    if (!info.isFile() || info.size > 1024 * 1024) return null;
+    return readFileSync(path, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+/** Whether a document's text is what its readers accept. */
+const VALID: Record<DocumentName, (value: unknown) => boolean> = {
+  definition: (value) => Boolean(parseDefinition(value)),
+  roster: (value) => Boolean(parseConfig(value)),
+  enrollment: (value) => parseEnrollment(value) !== null,
+};
+
+function valid(name: DocumentName, text: string): boolean {
+  try {
+    return VALID[name](JSON.parse(text));
+  } catch {
+    return false;
+  }
+}
+
+/** Run before the first change on a machine. A state database that does not
+ * exist yet starts with what the machine kept in files, so creating it never
+ * hides a document the machine still depends on. Returns what was moved.
+ */
+export function adoptEarlier(
+  env = process.env,
+  home = homedir()
+): DocumentName[] {
+  const root = stateRoot(env, home);
+  if (existsSync(databasePath(root))) return [];
+  const named = env['DEVFLEET_CONFIG'];
+  const earlier = DOCUMENTS.map((name) => ({
+    name,
+    text: readLegacy(name, env, home),
+  }));
+  const adopted: DocumentName[] = [];
+  for (const { name, text } of earlier) {
+    // A configuration kept in a file of your own stays there.
+    if (name === 'definition' && named && named !== definitionPath(root))
+      continue;
+    if (text === null || !valid(name, text)) continue;
+    writeDocument(databasePath(root), name, text);
+    adopted.push(name);
+  }
+  return adopted;
+}

package/src/state/litestream.ts

@@ -0,0 +1,143 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { basename, dirname, join } from 'node:path';
+import { stringify } from 'yaml';
+import { pinnedPath } from '../agents/toolchain';
+import { PACKAGE_ROOT } from '../package-info';
+import type { Runner } from '../types';
+import type { Bucket } from './bucket';
+
+/** Where this code runs from: its mise.toml pins Litestream. */
+const SOURCE = PACKAGE_ROOT;
+
+export interface Replication {
+  database: string;
+  bucket: Bucket;
+  run: Runner;
+  /** False for a read-only command: it does not install the binary. */
+  install?: boolean;
+}
+
+const located = new WeakMap<Runner, Promise<string | null>>();
+
+/** The pinned Litestream binary. It is a lazy mise pin, installed the first
+ * time the manager needs it; workers never do.
+ */
+function binary(run: Runner, install: boolean): Promise<string | null> {
+  // An installed package is a directory nobody ran `mise trust` in. Its pins
+  // are trusted for these commands only; nothing is recorded on the machine.
+  const options = {
+    cwd: SOURCE,
+    env: { PATH: pinnedPath(), MISE_TRUSTED_CONFIG_PATHS: SOURCE },
+  };
+  const locate = async (): Promise<string | null> => {
+    const which = await run(['mise', 'which', 'litestream'], options);
+    const path = which.stdout.trim();
+    return which.ok && path.startsWith('/') ? path : null;
+  };
+  const known = located.get(run);
+  if (known) return known;
+  const found = locate().then(async (path) => {
+    if (path || !install) return path;
+    await run(['mise', 'install', 'litestream'], {
+      ...options,
+      timeoutMs: 180_000,
+    });
+    return locate();
+  });
+  // A missing binary is looked for again: a later command may install it.
+  void found.then((path) => {
+    if (!path) located.delete(run);
+  });
+  located.set(run, found);
+  return found;
+}
+
+function replica(database: string, bucket: Bucket): Record<string, unknown> {
+  if (bucket.kind === 'directory')
+    return { type: 'file', path: join(bucket.path, basename(database)) };
+  return {
+    type: 's3',
+    bucket: bucket.bucket,
+    path: `${bucket.prefix}/${basename(database)}`,
+    ...(bucket.region ? { region: bucket.region } : {}),
+    // Every S3-compatible service answers path-style requests.
+    ...(bucket.endpoint
+      ? { endpoint: bucket.endpoint, 'force-path-style': true }
+      : {}),
+  };
+}
+
+/** The Litestream configuration for one database. It never holds a
+ * credential: those reach Litestream through its environment.
+ */
+export function litestreamConfig(database: string, bucket: Bucket): string {
+  return stringify({
+    logging: { level: 'error' },
+    // Each change uploads a full snapshot (the database is a few kilobytes),
+    // so a restore reads one object however long the history is.
+    snapshot: { interval: '24h', retention: '720h' },
+    dbs: [{ path: database, replica: replica(database, bucket) }],
+  });
+}
+
+export function litestreamEnvironment(bucket: Bucket): Record<string, string> {
+  return bucket.kind === 's3' && bucket.accessKeyId && bucket.secretAccessKey
+    ? {
+        // The standard names: they replace any keys in the operator's shell,
+        // which Litestream would otherwise prefer.
+        AWS_ACCESS_KEY_ID: bucket.accessKeyId,
+        AWS_SECRET_ACCESS_KEY: bucket.secretAccessKey,
+        // These keys are not a session's: a token left in the shell must
+        // not be sent with them.
+        AWS_SESSION_TOKEN: '',
+      }
+    : {};
+}
+
+/** Litestream is run once per change rather than as a daemon: a devfleet
+ * command that changed the state has not finished until the bucket has it,
+ * and nothing has to be kept running on the manager.
+ */
+async function invoke(
+  { database, bucket, run, install }: Replication,
+  command: string,
+  args: string[]
+): Promise<boolean> {
+  const litestream = await binary(run, install !== false);
+  if (!litestream) return false;
+  const config = join(dirname(database), 'litestream.yml');
+  mkdirSync(dirname(database), { recursive: true, mode: 0o700 });
+  writeFileSync(config, litestreamConfig(database, bucket), { mode: 0o600 });
+  const result = await run(
+    [litestream, command, '-no-expand-env', '-config', config, ...args],
+    // A bucket that does not answer must not hold a fleet-sync up for long.
+    { timeoutMs: 60_000, env: litestreamEnvironment(bucket) }
+  );
+  return result.ok;
+}
+
+/** Uploads everything the bucket does not have yet. */
+export async function replicate(target: Replication): Promise<boolean> {
+  return invoke(target, 'replicate', [
+    '-once',
+    '-force-snapshot',
+    '-enforce-retention',
+  ]);
+}
+
+/** Downloads the bucket's latest version to `output`, which must not exist.
+ * `absent` means the bucket is reachable and holds no state yet.
+ */
+export async function restore(
+  target: Replication,
+  output: string
+): Promise<'restored' | 'absent' | 'failed'> {
+  const ran = await invoke(target, 'restore', [
+    '-if-replica-exists',
+    '-o',
+    output,
+    target.database,
+  ]);
+  if (!ran) return 'failed';
+  return existsSync(output) ? 'restored' : 'absent';
+}

package/src/state/operator.ts

@@ -0,0 +1,20 @@
+/** Reads the `definition` document from the fleet state under `root`. */
+type DefinitionReader = (root: string) => string | null;
+
+let reader: DefinitionReader | null = null;
+
+/** Called by the devfleet CLI. Only it reads the fleet's state directly, so
+ * a change to the configuration reaches the manager's launchers and
+ * scheduled jobs the way it reaches every worker: as the copy fleet-sync
+ * installs. The reader is handed in rather than imported because launchers
+ * and the worker probe load the configuration under Node, which cannot open
+ * the state database. Null undoes it (tests).
+ */
+export function actAsOperator(read: DefinitionReader | null): void {
+  reader = read;
+}
+
+export const isOperator = (): boolean => reader !== null;
+
+export const stateDefinition = (root: string): string | null =>
+  reader?.(root) ?? null;

package/src/state/paths.ts

@@ -0,0 +1,40 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+/** The one directory that holds the manager's fleet state. Everything in it is
+ * a local copy: the storage bucket is the source of truth, so the directory
+ * can be deleted and restored with `fleet-state --restore`. `DEVFLEET_STATE`
+ * names another directory (a test fixture, a second fleet).
+ */
+export function stateRoot(env = process.env, home = homedir()): string {
+  return env['DEVFLEET_STATE'] || join(home, '.local/share/dev-fleet/state');
+}
+
+/** The fleet state database Litestream replicates. */
+export const databasePath = (root: string): string => join(root, 'fleet.db');
+
+/** Records the bucket's head this copy last matched (see `standing`). */
+export const replicatedPath = (root: string): string =>
+  join(root, 'replicated.json');
+
+/** The definition as a file, for the subprocesses that take a path. Rewritten
+ * from the database; never edited.
+ */
+export const definitionPath = (root: string): string =>
+  join(root, 'definition.json');
+
+/** The bucket this manager replicates to, and its credentials. The only local
+ * file that is not a copy of something else. `DEVFLEET_BUCKET` names another.
+ */
+export function bucketPath(env = process.env, home = homedir()): string {
+  return env['DEVFLEET_BUCKET'] || join(home, '.config/dev-fleet/bucket.json');
+}
+
+/** The home directory whose earlier files (kept before the fleet state
+ * existed) may still be read, or null. A state directory named with
+ * `DEVFLEET_STATE` is another fleet's: this machine's earlier files are not
+ * part of it.
+ */
+export function legacyHome(env = process.env, home = homedir()): string | null {
+  return env['DEVFLEET_STATE'] ? null : home;
+}

package/src/state/roster.ts

@@ -0,0 +1,62 @@
+import { randomUUID } from 'node:crypto';
+import { lstat, mkdir, open, readFile, rename, unlink } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname } from 'node:path';
+import { FleetUsageError } from '../errors';
+import { adoptEarlier, readLegacy } from './legacy';
+import { databasePath, stateRoot } from './paths';
+import { readDocument, writeDocument } from './store';
+
+/** The roster kept in the fleet state: what `--config` means when no file is
+ * named. A named file (`--config PATH`, `.fleet.local.json`) stays a file.
+ */
+export const STATE_ROSTER = 'state:roster';
+
+/** The roster's text, or null when there is none. */
+export async function readRosterText(
+  path: string,
+  env = process.env,
+  home = homedir()
+): Promise<string | null> {
+  if (path === STATE_ROSTER)
+    return (
+      readDocument(databasePath(stateRoot(env, home)), 'roster') ??
+      readLegacy('roster', env, home)
+    );
+  try {
+    const info = await lstat(path);
+    if (!info.isFile() || info.size > 1024 * 1024)
+      throw new FleetUsageError(
+        'Roster must be a regular file smaller than 1 MiB; symlinks are not supported.'
+      );
+    return await readFile(path, 'utf8');
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+export async function writeRosterText(
+  path: string,
+  text: string,
+  env = process.env,
+  home = homedir()
+): Promise<void> {
+  if (path === STATE_ROSTER) {
+    adoptEarlier(env, home);
+    writeDocument(databasePath(stateRoot(env, home)), 'roster', text);
+    return;
+  }
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  const handle = await open(temporary, 'wx', 0o600);
+  try {
+    await handle.writeFile(text);
+    await handle.sync();
+    await handle.close();
+    await rename(temporary, path);
+  } finally {
+    await handle.close();
+    await unlink(temporary).catch(() => undefined);
+  }
+}