@crvouga/dev-fleet 0.6.0 → 0.8.0

package/src/state/store.ts

@@ -1,7 +1,14 @@
 import { Database } from 'bun:sqlite';
 import { randomUUID } from 'node:crypto';
-import { chmodSync, existsSync, mkdirSync } from 'node:fs';
-import { dirname } from 'node:path';
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+} from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { FleetUsageError } from '../errors';
 
 /** Everything the manager must not lose, as one JSON text each:
@@ -35,19 +42,64 @@ export interface DocumentInfo {
 
 const MAX_DOCUMENT_BYTES = 1024 * 1024;
 
-const SCHEMA = `
-CREATE TABLE IF NOT EXISTS documents (
-  name TEXT PRIMARY KEY,
-  body TEXT NOT NULL,
-  updated_at TEXT NOT NULL
-) STRICT;
-CREATE TABLE IF NOT EXISTS head (
-  one INTEGER PRIMARY KEY CHECK (one = 1),
-  id TEXT NOT NULL,
-  sequence INTEGER NOT NULL,
-  written_at TEXT NOT NULL
-) STRICT;
-`;
+export interface Migration {
+  version: number;
+  sql: string;
+}
+
+const MIGRATIONS_DIRECTORY = fileURLToPath(
+  new URL('./migrations/', import.meta.url)
+);
+
+/** The schema as numbered files (`0001-documents.sql`, …), applied in order.
+ * A database records the last one it received in `PRAGMA user_version`; one
+ * written before the schema had a version holds exactly the first.
+ */
+export function migrations(directory = MIGRATIONS_DIRECTORY): Migration[] {
+  return readdirSync(directory)
+    .filter((name) => /^\d{4}-[a-z0-9-]+\.sql$/.test(name))
+    .sort()
+    .map((name, index) => {
+      const version = Number(name.slice(0, 4));
+      if (version !== index + 1)
+        throw new Error(`State migrations must be numbered from 1: ${name}`);
+      return { version, sql: readFileSync(join(directory, name), 'utf8') };
+    });
+}
+
+const MIGRATIONS = migrations();
+
+/** The schema version this dev-fleet writes and the newest it can read. */
+export const SCHEMA_VERSION = MIGRATIONS.length;
+
+/** A fleet state written by a newer dev-fleet: this one must neither read nor
+ * change it, or a rollback (or an old second manager) would damage it.
+ */
+export class StateNewerError extends FleetUsageError {}
+
+function schemaVersion(db: Database): number {
+  return (db.query('PRAGMA user_version').get() as { user_version: number })
+    .user_version;
+}
+
+/** Refuses a database a newer dev-fleet wrote; returns its version. */
+function supported(db: Database): number {
+  const version = schemaVersion(db);
+  if (version > SCHEMA_VERSION)
+    throw new StateNewerError(
+      `FLEET_STATE_NEWER_THAN_CLI: the fleet state has schema version ${version}, and this dev-fleet reads version ${SCHEMA_VERSION} at most. Nothing was read or changed. Install the dev-fleet version the fleet runs (npx devfleet --version on the manager that wrote it) and run the command again.`
+    );
+  return version;
+}
+
+/** Brings a database forward to `SCHEMA_VERSION`, one migration at a time. */
+function migrate(db: Database): void {
+  const from = supported(db);
+  for (const migration of MIGRATIONS.filter((entry) => entry.version > from)) {
+    db.exec(migration.sql);
+    db.exec(`PRAGMA user_version = ${migration.version}`);
+  }
+}
 
 function unreadable(): FleetUsageError {
   return new FleetUsageError(
@@ -65,8 +117,10 @@ function reading<T>(file: string, read: (db: Database) => T): T | null {
   let db: Database | undefined;
   try {
     db = new Database(file, { readwrite: true, create: false });
+    supported(db);
     return read(db);
-  } catch {
+  } catch (error) {
+    if (error instanceof FleetUsageError) throw error;
     throw unreadable();
   } finally {
     db?.close();
@@ -82,9 +136,16 @@ function writing<T>(file: string, write: (db: Database) => T): T {
   try {
     db = new Database(file, { create: true, readwrite: true });
     chmodSync(file, 0o600);
+    // Checked before anything is written, the journal mode included.
+    supported(db);
     db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;');
-    db.exec(SCHEMA);
-    return db.transaction(write).immediate(db);
+    const opened = db;
+    return opened
+      .transaction(() => {
+        migrate(opened);
+        return write(opened);
+      })
+      .immediate();
   } catch (error) {
     if (error instanceof FleetUsageError) throw error;
     throw new FleetUsageError(
@@ -140,6 +201,21 @@ export function readHead(file: string): Head | null {
   });
 }
 
+function nextHead(db: Database): Head {
+  const head = db
+    .query('SELECT id, sequence, written_at AS writtenAt FROM head')
+    .get() as Head | null;
+  const next: Head = {
+    id: randomUUID(),
+    sequence: (head?.sequence ?? 0) + 1,
+    writtenAt: new Date().toISOString(),
+  };
+  db.query(
+    'INSERT INTO head (one, id, sequence, written_at) VALUES (1, ?1, ?2, ?3) ON CONFLICT (one) DO UPDATE SET id = ?1, sequence = ?2, written_at = ?3'
+  ).run(next.id, next.sequence, next.writtenAt);
+  return next;
+}
+
 /** Stores one document and advances the head. Writing the text already
  * stored changes nothing, so repeating a command never looks like new state.
  */
@@ -160,17 +236,40 @@ export function writeDocument(
       .query('SELECT id, sequence, written_at AS writtenAt FROM head')
       .get() as Head | null;
     if (head && current?.body === body) return head;
-    const next: Head = {
-      id: randomUUID(),
-      sequence: (head?.sequence ?? 0) + 1,
-      writtenAt: new Date().toISOString(),
-    };
+    const next = nextHead(db);
     db.query(
       'INSERT INTO documents (name, body, updated_at) VALUES (?1, ?2, ?3) ON CONFLICT (name) DO UPDATE SET body = ?2, updated_at = ?3'
     ).run(name, body, next.writtenAt);
-    db.query(
-      'INSERT INTO head (one, id, sequence, written_at) VALUES (1, ?1, ?2, ?3) ON CONFLICT (one) DO UPDATE SET id = ?1, sequence = ?2, written_at = ?3'
-    ).run(next.id, next.sequence, next.writtenAt);
     return next;
   });
 }
+
+/** Which manager may change the fleet state, and until when. */
+export interface Lease {
+  holder: string;
+  expiresAt: string;
+}
+
+export function readLease(file: string): Lease | null {
+  return reading(file, (db) => {
+    const present = db
+      .query(
+        "SELECT count(*) AS found FROM sqlite_master WHERE type = 'table' AND name = 'lease'"
+      )
+      .get() as { found: number };
+    if (!present.found) return null;
+    return db
+      .query('SELECT holder, expires_at AS expiresAt FROM lease')
+      .get() as Lease | null;
+  });
+}
+
+/** Records the lease and advances the head, so that the bucket carries it. */
+export function writeLease(file: string, lease: Lease): Head {
+  return writing(file, (db) => {
+    db.query(
+      'INSERT INTO lease (one, holder, expires_at) VALUES (1, ?1, ?2) ON CONFLICT (one) DO UPDATE SET holder = ?1, expires_at = ?2'
+    ).run(lease.holder, lease.expiresAt);
+    return nextHead(db);
+  });
+}

package/src/state/sync.ts

@@ -18,12 +18,29 @@ import { bucketIdentity, type Bucket } from './bucket';
 import { connection } from './connection';
 import { replicate, restore } from './litestream';
 import { databasePath, replicatedPath, stateRoot } from './paths';
-import { readHead, type Head } from './store';
+import { managerHolder } from './holder';
+import {
+  StateNewerError,
+  readHead,
+  readLease,
+  writeLease,
+  type Head,
+  type Lease,
+} from './store';
+
+export type { Lease } from './store';
 
 export interface StateContext {
   root: string;
   bucket: Bucket | null;
   run: Runner;
+  /** This machine, as the manager lease names it. Without one the lease is
+   * neither checked nor taken.
+   */
+  holder?: string | null;
+  /** Take the lease from another manager before it expired. */
+  takeOver?: boolean;
+  now?: () => Date;
 }
 
 export function stateContext(
@@ -34,12 +51,14 @@ export function stateContext(
     root: stateRoot(env),
     bucket: connection(env).bucket,
     run,
+    holder: managerHolder(env),
   };
 }
 
 /** How this machine's copy compares with the bucket.
  * - `local-only`: no bucket is configured; this copy is the only one.
  * - `unreachable`: the bucket did not answer, or did not take an upload.
+ * - `corrupt`: the bucket answered with a version that cannot be read.
  * - `empty`: neither side holds any state.
  * - `unpublished`: the bucket holds nothing yet; this copy can seed it.
  * - `in-sync`: both hold the same version.
@@ -50,6 +69,7 @@ export function stateContext(
 export type Standing =
   | 'local-only'
   | 'unreachable'
+  | 'corrupt'
   | 'empty'
   | 'unpublished'
   | 'in-sync'
@@ -109,19 +129,24 @@ const INCOMING = /^incoming-[0-9a-f-]{36}\.db/;
 
 type Remote =
   | { status: 'unreachable' }
+  | { status: 'corrupt' }
   | { status: 'absent' }
   | { status: 'present'; file: string; head: Head };
 
-/** Downloads the bucket's current version beside the local one. */
+/** Downloads the bucket's current version beside the local one, or the
+ * version it held at `timestamp`.
+ */
 async function fetchRemote(
   context: StateContext & { bucket: Bucket },
-  install = true
+  install = true,
+  timestamp?: string
 ): Promise<Remote> {
   const database = databasePath(context.root);
   const file = join(context.root, `incoming-${randomUUID()}.db`);
   const outcome = await restore(
     { database, bucket: context.bucket, run: context.run, install },
-    file
+    file,
+    timestamp
   );
   if (outcome === 'absent') return { status: 'absent' };
   try {
@@ -130,10 +155,12 @@ async function fetchRemote(
     discard(file);
     // A replica that holds no version yet is the same as no replica.
     return { status: outcome === 'restored' ? 'absent' : 'unreachable' };
-  } catch {
-    // A download that cannot be read says nothing about the bucket.
+  } catch (error) {
     discard(file);
-    return { status: 'unreachable' };
+    // A newer dev-fleet wrote it: refused, never taken for damage.
+    if (error instanceof StateNewerError) throw error;
+    // The bucket answered, and what it holds cannot be read.
+    return { status: 'corrupt' };
   }
 }
 
@@ -163,8 +190,8 @@ async function observe(
   const { bucket } = context;
   if (!bucket) return { standing: 'local-only', local, remote: null };
   const remote = await fetchRemote({ ...context, bucket }, install);
-  if (remote.status === 'unreachable')
-    return { standing: 'unreachable', local, remote };
+  if (remote.status === 'unreachable' || remote.status === 'corrupt')
+    return { standing: remote.status, local, remote };
   if (remote.status === 'absent')
     return { standing: local ? 'unpublished' : 'empty', local, remote };
   const replicated = readReplicated(context.root, bucket);
@@ -261,7 +288,9 @@ async function upload(
  */
 function settle(context: StateContext, before: Standing): Promise<Standing> {
   const { bucket } = context;
-  if (!bucket || before === 'unreachable') return Promise.resolve(before);
+  // A damaged replica keeps its history for a point-in-time restore.
+  if (!bucket || before === 'unreachable' || before === 'corrupt')
+    return Promise.resolve(before);
   const head = readHead(databasePath(context.root));
   if (
     before === 'in-sync' &&
@@ -319,6 +348,40 @@ export function pull(context: StateContext): Promise<Standing> {
   return locked(context.root, () => observed(context, true));
 }
 
+/** Replaces this copy with the version the bucket held at `timestamp`, then
+ * uploads it as the bucket's current version: the way back from a replica
+ * whose latest version is damaged. `unreachable` when the bucket holds no
+ * readable version at that time.
+ */
+export function pullAt(
+  context: StateContext,
+  timestamp: string
+): Promise<Standing> {
+  if (Number.isNaN(Date.parse(timestamp)))
+    throw new FleetUsageError(
+      '--timestamp needs a time such as 2026-11-01T00:00:00Z.'
+    );
+  const { bucket } = context;
+  if (!bucket)
+    throw new FleetUsageError(
+      'No bucket is configured, so there is no earlier version to restore.'
+    );
+  return locked(context.root, async () => {
+    const remote = await fetchRemote(
+      { ...context, bucket },
+      true,
+      new Date(timestamp).toISOString()
+    );
+    if (remote.status !== 'present')
+      return remote.status === 'absent' ? 'empty' : remote.status;
+    const database = databasePath(context.root);
+    discard(database);
+    renameSync(remote.file, database);
+    chmodSync(database, 0o600);
+    return upload({ ...context, bucket }, true);
+  });
+}
+
 /** Uploads this copy's changes, only when the bucket has not moved since
  * this copy last matched it. A copy with no changes of its own is brought up
  * to the bucket instead; a diverged one is left alone and reported.
@@ -332,6 +395,62 @@ export function push(context: StateContext): Promise<Standing> {
   });
 }
 
+/** How long a manager holds the lease after its last change. A machine that
+ * wakes up later than this may take it without being asked.
+ */
+export const LEASE_DAYS = 7;
+const LEASE_MS = LEASE_DAYS * 24 * 60 * 60 * 1000;
+
+/** Another manager holds the lease: this one changes nothing. */
+export class LeaseHeldError extends FleetUsageError {}
+
+/** The lease another manager holds over this copy, when it is not expired
+ * and this machine was not told to take it over. Reads local files only: call
+ * it on the bucket's version, after a refresh.
+ */
+export function leaseConflict(
+  context: StateContext,
+  now: Date = (context.now ?? (() => new Date()))()
+): Lease | null {
+  const lease = readLease(databasePath(context.root));
+  if (!context.holder || !lease || lease.holder === context.holder) return null;
+  if (Date.parse(lease.expiresAt) <= now.getTime() || context.takeOver)
+    return null;
+  return lease;
+}
+
+export function leaseHeldMessage(lease: Lease): string {
+  return `FLEET_MANAGER_LEASE_HELD: another manager (${lease.holder}) holds the fleet state until ${lease.expiresAt}. Nothing was changed. Run devfleet from that manager; to make this machine the manager instead, repeat with --take-over once the other one is gone (see docs/manager-recovery.md).`;
+}
+
+/** Refuses when another manager holds the lease. Call it before the first
+ * write of a session.
+ */
+export function refuseHeldLease(context: StateContext): void {
+  const held = leaseConflict(context);
+  if (held) throw new LeaseHeldError(leaseHeldMessage(held));
+}
+
+/** Takes or renews the lease for this machine: when it is not this machine's
+ * yet, or half spent. Call it at the end of a session that may write.
+ */
+export function claimLease(
+  context: StateContext,
+  now: Date = (context.now ?? (() => new Date()))()
+): void {
+  refuseHeldLease(context);
+  if (!context.holder) return;
+  const lease = readLease(databasePath(context.root));
+  const fresh =
+    lease?.holder === context.holder &&
+    Date.parse(lease.expiresAt) - now.getTime() > LEASE_MS / 2;
+  if (fresh) return;
+  writeLease(databasePath(context.root), {
+    holder: context.holder,
+    expiresAt: new Date(now.getTime() + LEASE_MS).toISOString(),
+  });
+}
+
 export const DIVERGED_REMEDY =
   'This machine and the bucket both changed the fleet state. The bucket is the source of truth: keep a copy of anything you still need (devfleet fleet-state --export), then run devfleet fleet-state --restore --yes.';
 
@@ -346,7 +465,9 @@ export function mutateState<T>(
   return locked(context.root, async () => {
     const before = await observed(context, false);
     if (before === 'diverged') throw new FleetUsageError(DIVERGED_REMEDY);
+    refuseHeldLease(context);
     const value = await change(before);
+    claimLease(context);
     return { value, standing: await settle(context, before) };
   });
 }

package/src/worker-credentials.ts

@@ -0,0 +1,201 @@
+import { fleet } from './fleet-config';
+import { resolveVaultConfig, type VaultConfig } from './vault-config';
+import type { Check, Runner } from './types';
+
+/** Each worker's own Vault token is kept in a document of its own, beside
+ * the fleet's shared one: `<project>/workers/<id>`, field `token`, with the
+ * token's `accessor` so the manager can revoke it without holding it. A
+ * policy that grants one worker its document grants it nothing of another's.
+ */
+export interface WorkerTokenLocation {
+  mount: string;
+  path: string;
+  field: 'token';
+}
+
+export function workerTokenLocation(
+  node: string,
+  vault: VaultConfig = resolveVaultConfig()
+): WorkerTokenLocation {
+  return {
+    mount: vault.mount,
+    path: `${vault.project}/workers/${node}`,
+    field: 'token',
+  };
+}
+
+/** The Vault policy each worker token is issued with. */
+export const workerPolicyName = (node: string): string =>
+  `fleet-worker-${node}`;
+
+/** The least a worker needs: the fleet's shared document (the proxy key and
+ * what the providers read) and its own token document. Nothing grants a
+ * worker the documents of the others, so one worker's token cannot read
+ * another's. Token renewal and lookup come from Vault's default policy.
+ */
+export function workerVaultPolicy(
+  node: string,
+  vault: VaultConfig = resolveVaultConfig(),
+  shared: string = sharedDocument()
+): string {
+  const own = workerTokenLocation(node, vault);
+  return [
+    `# dev-fleet: the policy of the worker ${node}. Written by dev-fleet; do not edit.`,
+    `path "${vault.mount}/data/${shared}" {`,
+    '  capabilities = ["read"]',
+    '}',
+    `path "${own.mount}/data/${own.path}" {`,
+    '  capabilities = ["read"]',
+    '}',
+    '',
+  ].join('\n');
+}
+
+function sharedDocument(): string {
+  const secrets = fleet.secrets;
+  return secrets.provider === 'vault'
+    ? secrets.locations['proxy-client-key'].path
+    : '';
+}
+
+const TIMEOUT = { timeoutMs: 20_000 };
+
+/** Issues a worker its own periodic, orphan token under its own policy and
+ * keeps it in its own document. The token reaches `vault kv put` on standard
+ * input, never as an argument. Run on the manager, logged in to Vault.
+ */
+export async function issueWorkerToken(
+  node: string,
+  run: Runner,
+  vault: VaultConfig = resolveVaultConfig()
+): Promise<Check> {
+  const address = `-address=${vault.addr}`;
+  const policy = await run(
+    ['vault', 'policy', 'write', address, workerPolicyName(node), '-'],
+    { ...TIMEOUT, input: workerVaultPolicy(node, vault) }
+  );
+  if (!policy.ok) return credentialFailure('VAULT_WORKER_POLICY_FAILED');
+  const created = await run(
+    [
+      'vault',
+      'token',
+      'create',
+      address,
+      `-policy=${workerPolicyName(node)}`,
+      '-orphan',
+      '-period=768h',
+      `-display-name=${workerPolicyName(node)}`,
+      '-format=json',
+    ],
+    TIMEOUT
+  );
+  const issued = created.ok ? parseIssued(created.stdout) : null;
+  if (!issued) return credentialFailure('VAULT_WORKER_TOKEN_NOT_ISSUED');
+  const location = workerTokenLocation(node, vault);
+  const stored = await run(
+    [
+      'vault',
+      'kv',
+      'put',
+      address,
+      `-mount=${location.mount}`,
+      location.path,
+      'token=-',
+      `accessor=${issued.accessor}`,
+    ],
+    { ...TIMEOUT, input: issued.token }
+  );
+  if (!stored.ok) return credentialFailure('VAULT_WORKER_TOKEN_NOT_STORED');
+  return {
+    id: 'auth:worker-token',
+    status: 'pass',
+    summary: `${node} has a token of its own, readable by nothing but itself and the manager.`,
+  };
+}
+
+function parseIssued(
+  stdout: string
+): { token: string; accessor: string } | null {
+  try {
+    const auth = (JSON.parse(stdout) as { auth?: Record<string, unknown> })
+      .auth;
+    const token = auth?.['client_token'];
+    const accessor = auth?.['accessor'];
+    return typeof token === 'string' &&
+      typeof accessor === 'string' &&
+      /^[\x21-\x7e]{16,4096}$/.test(token) &&
+      /^[A-Za-z0-9._-]{8,256}$/.test(accessor)
+      ? { token, accessor }
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Revokes a worker's token by its accessor (the manager never holds the
+ * token to do so), then removes its document and its policy. A worker that
+ * has none already is revoked: running it twice changes nothing.
+ */
+export async function revokeWorkerToken(
+  node: string,
+  run: Runner,
+  vault: VaultConfig = resolveVaultConfig()
+): Promise<Check> {
+  const address = `-address=${vault.addr}`;
+  const location = workerTokenLocation(node, vault);
+  const read = await run(
+    [
+      'vault',
+      'kv',
+      'get',
+      address,
+      `-mount=${location.mount}`,
+      '-field=accessor',
+      location.path,
+    ],
+    TIMEOUT
+  );
+  const accessor = read.ok ? read.stdout.trim() : '';
+  if (accessor) {
+    if (!/^[A-Za-z0-9._-]{8,256}$/.test(accessor))
+      return credentialFailure('VAULT_WORKER_TOKEN_NOT_REVOKED');
+    const revoked = await run(
+      ['vault', 'token', 'revoke', address, '-accessor', accessor],
+      TIMEOUT
+    );
+    if (!revoked.ok) return credentialFailure('VAULT_WORKER_TOKEN_NOT_REVOKED');
+  }
+  await run(
+    [
+      'vault',
+      'kv',
+      'metadata',
+      'delete',
+      address,
+      `-mount=${location.mount}`,
+      location.path,
+    ],
+    TIMEOUT
+  );
+  await run(
+    ['vault', 'policy', 'delete', address, workerPolicyName(node)],
+    TIMEOUT
+  );
+  return {
+    id: 'auth:worker-token',
+    status: 'pass',
+    summary: `${node}'s token is revoked and its document and policy removed.`,
+  };
+}
+
+function credentialFailure(summary: string): Check {
+  return {
+    id: 'auth:worker-token',
+    status: 'fail',
+    summary,
+    remedy: {
+      action:
+        'Log the manager in to Vault with a token that may write policies, issue tokens and write the fleet mount, then run the command again.',
+    },
+  };
+}