@crvouga/dev-fleet 0.5.0 → 0.7.0

package/src/state/store.ts

@@ -1,7 +1,14 @@
 import { Database } from 'bun:sqlite';
 import { randomUUID } from 'node:crypto';
-import { chmodSync, existsSync, mkdirSync } from 'node:fs';
-import { dirname } from 'node:path';
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+} from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { FleetUsageError } from '../errors';
 
 /** Everything the manager must not lose, as one JSON text each:
@@ -35,19 +42,64 @@ export interface DocumentInfo {
 
 const MAX_DOCUMENT_BYTES = 1024 * 1024;
 
-const SCHEMA = `
-CREATE TABLE IF NOT EXISTS documents (
-  name TEXT PRIMARY KEY,
-  body TEXT NOT NULL,
-  updated_at TEXT NOT NULL
-) STRICT;
-CREATE TABLE IF NOT EXISTS head (
-  one INTEGER PRIMARY KEY CHECK (one = 1),
-  id TEXT NOT NULL,
-  sequence INTEGER NOT NULL,
-  written_at TEXT NOT NULL
-) STRICT;
-`;
+export interface Migration {
+  version: number;
+  sql: string;
+}
+
+const MIGRATIONS_DIRECTORY = fileURLToPath(
+  new URL('./migrations/', import.meta.url)
+);
+
+/** The schema as numbered files (`0001-documents.sql`, …), applied in order.
+ * A database records the last one it received in `PRAGMA user_version`; one
+ * written before the schema had a version holds exactly the first.
+ */
+export function migrations(directory = MIGRATIONS_DIRECTORY): Migration[] {
+  return readdirSync(directory)
+    .filter((name) => /^\d{4}-[a-z0-9-]+\.sql$/.test(name))
+    .sort()
+    .map((name, index) => {
+      const version = Number(name.slice(0, 4));
+      if (version !== index + 1)
+        throw new Error(`State migrations must be numbered from 1: ${name}`);
+      return { version, sql: readFileSync(join(directory, name), 'utf8') };
+    });
+}
+
+const MIGRATIONS = migrations();
+
+/** The schema version this dev-fleet writes and the newest it can read. */
+export const SCHEMA_VERSION = MIGRATIONS.length;
+
+/** A fleet state written by a newer dev-fleet: this one must neither read nor
+ * change it, or a rollback (or an old second manager) would damage it.
+ */
+export class StateNewerError extends FleetUsageError {}
+
+function schemaVersion(db: Database): number {
+  return (db.query('PRAGMA user_version').get() as { user_version: number })
+    .user_version;
+}
+
+/** Refuses a database a newer dev-fleet wrote; returns its version. */
+function supported(db: Database): number {
+  const version = schemaVersion(db);
+  if (version > SCHEMA_VERSION)
+    throw new StateNewerError(
+      `FLEET_STATE_NEWER_THAN_CLI: the fleet state has schema version ${version}, and this dev-fleet reads version ${SCHEMA_VERSION} at most. Nothing was read or changed. Install the dev-fleet version the fleet runs (npx devfleet --version on the manager that wrote it) and run the command again.`
+    );
+  return version;
+}
+
+/** Brings a database forward to `SCHEMA_VERSION`, one migration at a time. */
+function migrate(db: Database): void {
+  const from = supported(db);
+  for (const migration of MIGRATIONS.filter((entry) => entry.version > from)) {
+    db.exec(migration.sql);
+    db.exec(`PRAGMA user_version = ${migration.version}`);
+  }
+}
 
 function unreadable(): FleetUsageError {
   return new FleetUsageError(
@@ -65,8 +117,10 @@ function reading<T>(file: string, read: (db: Database) => T): T | null {
   let db: Database | undefined;
   try {
     db = new Database(file, { readwrite: true, create: false });
+    supported(db);
     return read(db);
-  } catch {
+  } catch (error) {
+    if (error instanceof FleetUsageError) throw error;
     throw unreadable();
   } finally {
     db?.close();
@@ -82,9 +136,16 @@ function writing<T>(file: string, write: (db: Database) => T): T {
   try {
     db = new Database(file, { create: true, readwrite: true });
     chmodSync(file, 0o600);
+    // Checked before anything is written, the journal mode included.
+    supported(db);
     db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;');
-    db.exec(SCHEMA);
-    return db.transaction(write).immediate(db);
+    const opened = db;
+    return opened
+      .transaction(() => {
+        migrate(opened);
+        return write(opened);
+      })
+      .immediate();
   } catch (error) {
     if (error instanceof FleetUsageError) throw error;
     throw new FleetUsageError(
@@ -140,6 +201,21 @@ export function readHead(file: string): Head | null {
   });
 }
 
+function nextHead(db: Database): Head {
+  const head = db
+    .query('SELECT id, sequence, written_at AS writtenAt FROM head')
+    .get() as Head | null;
+  const next: Head = {
+    id: randomUUID(),
+    sequence: (head?.sequence ?? 0) + 1,
+    writtenAt: new Date().toISOString(),
+  };
+  db.query(
+    'INSERT INTO head (one, id, sequence, written_at) VALUES (1, ?1, ?2, ?3) ON CONFLICT (one) DO UPDATE SET id = ?1, sequence = ?2, written_at = ?3'
+  ).run(next.id, next.sequence, next.writtenAt);
+  return next;
+}
+
 /** Stores one document and advances the head. Writing the text already
  * stored changes nothing, so repeating a command never looks like new state.
  */
@@ -160,17 +236,40 @@ export function writeDocument(
       .query('SELECT id, sequence, written_at AS writtenAt FROM head')
       .get() as Head | null;
     if (head && current?.body === body) return head;
-    const next: Head = {
-      id: randomUUID(),
-      sequence: (head?.sequence ?? 0) + 1,
-      writtenAt: new Date().toISOString(),
-    };
+    const next = nextHead(db);
     db.query(
       'INSERT INTO documents (name, body, updated_at) VALUES (?1, ?2, ?3) ON CONFLICT (name) DO UPDATE SET body = ?2, updated_at = ?3'
     ).run(name, body, next.writtenAt);
-    db.query(
-      'INSERT INTO head (one, id, sequence, written_at) VALUES (1, ?1, ?2, ?3) ON CONFLICT (one) DO UPDATE SET id = ?1, sequence = ?2, written_at = ?3'
-    ).run(next.id, next.sequence, next.writtenAt);
     return next;
   });
 }
+
+/** Which manager may change the fleet state, and until when. */
+export interface Lease {
+  holder: string;
+  expiresAt: string;
+}
+
+export function readLease(file: string): Lease | null {
+  return reading(file, (db) => {
+    const present = db
+      .query(
+        "SELECT count(*) AS found FROM sqlite_master WHERE type = 'table' AND name = 'lease'"
+      )
+      .get() as { found: number };
+    if (!present.found) return null;
+    return db
+      .query('SELECT holder, expires_at AS expiresAt FROM lease')
+      .get() as Lease | null;
+  });
+}
+
+/** Records the lease and advances the head, so that the bucket carries it. */
+export function writeLease(file: string, lease: Lease): Head {
+  return writing(file, (db) => {
+    db.query(
+      'INSERT INTO lease (one, holder, expires_at) VALUES (1, ?1, ?2) ON CONFLICT (one) DO UPDATE SET holder = ?1, expires_at = ?2'
+    ).run(lease.holder, lease.expiresAt);
+    return nextHead(db);
+  });
+}

package/src/state/sync.ts

@@ -18,12 +18,29 @@ import { bucketIdentity, type Bucket } from './bucket';
 import { connection } from './connection';
 import { replicate, restore } from './litestream';
 import { databasePath, replicatedPath, stateRoot } from './paths';
-import { readHead, type Head } from './store';
+import { managerHolder } from './holder';
+import {
+  StateNewerError,
+  readHead,
+  readLease,
+  writeLease,
+  type Head,
+  type Lease,
+} from './store';
+
+export type { Lease } from './store';
 
 export interface StateContext {
   root: string;
   bucket: Bucket | null;
   run: Runner;
+  /** This machine, as the manager lease names it. Without one the lease is
+   * neither checked nor taken.
+   */
+  holder?: string | null;
+  /** Take the lease from another manager before it expired. */
+  takeOver?: boolean;
+  now?: () => Date;
 }
 
 export function stateContext(
@@ -34,12 +51,14 @@ export function stateContext(
     root: stateRoot(env),
     bucket: connection(env).bucket,
     run,
+    holder: managerHolder(env),
   };
 }
 
 /** How this machine's copy compares with the bucket.
  * - `local-only`: no bucket is configured; this copy is the only one.
  * - `unreachable`: the bucket did not answer, or did not take an upload.
+ * - `corrupt`: the bucket answered with a version that cannot be read.
  * - `empty`: neither side holds any state.
  * - `unpublished`: the bucket holds nothing yet; this copy can seed it.
  * - `in-sync`: both hold the same version.
@@ -50,6 +69,7 @@ export function stateContext(
 export type Standing =
   | 'local-only'
   | 'unreachable'
+  | 'corrupt'
   | 'empty'
   | 'unpublished'
   | 'in-sync'
@@ -109,19 +129,24 @@ const INCOMING = /^incoming-[0-9a-f-]{36}\.db/;
 
 type Remote =
   | { status: 'unreachable' }
+  | { status: 'corrupt' }
   | { status: 'absent' }
   | { status: 'present'; file: string; head: Head };
 
-/** Downloads the bucket's current version beside the local one. */
+/** Downloads the bucket's current version beside the local one, or the
+ * version it held at `timestamp`.
+ */
 async function fetchRemote(
   context: StateContext & { bucket: Bucket },
-  install = true
+  install = true,
+  timestamp?: string
 ): Promise<Remote> {
   const database = databasePath(context.root);
   const file = join(context.root, `incoming-${randomUUID()}.db`);
   const outcome = await restore(
     { database, bucket: context.bucket, run: context.run, install },
-    file
+    file,
+    timestamp
   );
   if (outcome === 'absent') return { status: 'absent' };
   try {
@@ -130,10 +155,12 @@ async function fetchRemote(
     discard(file);
     // A replica that holds no version yet is the same as no replica.
     return { status: outcome === 'restored' ? 'absent' : 'unreachable' };
-  } catch {
-    // A download that cannot be read says nothing about the bucket.
+  } catch (error) {
     discard(file);
-    return { status: 'unreachable' };
+    // A newer dev-fleet wrote it: refused, never taken for damage.
+    if (error instanceof StateNewerError) throw error;
+    // The bucket answered, and what it holds cannot be read.
+    return { status: 'corrupt' };
   }
 }
 
@@ -163,8 +190,8 @@ async function observe(
   const { bucket } = context;
   if (!bucket) return { standing: 'local-only', local, remote: null };
   const remote = await fetchRemote({ ...context, bucket }, install);
-  if (remote.status === 'unreachable')
-    return { standing: 'unreachable', local, remote };
+  if (remote.status === 'unreachable' || remote.status === 'corrupt')
+    return { standing: remote.status, local, remote };
   if (remote.status === 'absent')
     return { standing: local ? 'unpublished' : 'empty', local, remote };
   const replicated = readReplicated(context.root, bucket);
@@ -261,7 +288,9 @@ async function upload(
  */
 function settle(context: StateContext, before: Standing): Promise<Standing> {
   const { bucket } = context;
-  if (!bucket || before === 'unreachable') return Promise.resolve(before);
+  // A damaged replica keeps its history for a point-in-time restore.
+  if (!bucket || before === 'unreachable' || before === 'corrupt')
+    return Promise.resolve(before);
   const head = readHead(databasePath(context.root));
   if (
     before === 'in-sync' &&
@@ -319,6 +348,40 @@ export function pull(context: StateContext): Promise<Standing> {
   return locked(context.root, () => observed(context, true));
 }
 
+/** Replaces this copy with the version the bucket held at `timestamp`, then
+ * uploads it as the bucket's current version: the way back from a replica
+ * whose latest version is damaged. `unreachable` when the bucket holds no
+ * readable version at that time.
+ */
+export function pullAt(
+  context: StateContext,
+  timestamp: string
+): Promise<Standing> {
+  if (Number.isNaN(Date.parse(timestamp)))
+    throw new FleetUsageError(
+      '--timestamp needs a time such as 2026-11-01T00:00:00Z.'
+    );
+  const { bucket } = context;
+  if (!bucket)
+    throw new FleetUsageError(
+      'No bucket is configured, so there is no earlier version to restore.'
+    );
+  return locked(context.root, async () => {
+    const remote = await fetchRemote(
+      { ...context, bucket },
+      true,
+      new Date(timestamp).toISOString()
+    );
+    if (remote.status !== 'present')
+      return remote.status === 'absent' ? 'empty' : remote.status;
+    const database = databasePath(context.root);
+    discard(database);
+    renameSync(remote.file, database);
+    chmodSync(database, 0o600);
+    return upload({ ...context, bucket }, true);
+  });
+}
+
 /** Uploads this copy's changes, only when the bucket has not moved since
  * this copy last matched it. A copy with no changes of its own is brought up
  * to the bucket instead; a diverged one is left alone and reported.
@@ -332,6 +395,62 @@ export function push(context: StateContext): Promise<Standing> {
   });
 }
 
+/** How long a manager holds the lease after its last change. A machine that
+ * wakes up later than this may take it without being asked.
+ */
+export const LEASE_DAYS = 7;
+const LEASE_MS = LEASE_DAYS * 24 * 60 * 60 * 1000;
+
+/** Another manager holds the lease: this one changes nothing. */
+export class LeaseHeldError extends FleetUsageError {}
+
+/** The lease another manager holds over this copy, when it is not expired
+ * and this machine was not told to take it over. Reads local files only: call
+ * it on the bucket's version, after a refresh.
+ */
+export function leaseConflict(
+  context: StateContext,
+  now: Date = (context.now ?? (() => new Date()))()
+): Lease | null {
+  const lease = readLease(databasePath(context.root));
+  if (!context.holder || !lease || lease.holder === context.holder) return null;
+  if (Date.parse(lease.expiresAt) <= now.getTime() || context.takeOver)
+    return null;
+  return lease;
+}
+
+export function leaseHeldMessage(lease: Lease): string {
+  return `FLEET_MANAGER_LEASE_HELD: another manager (${lease.holder}) holds the fleet state until ${lease.expiresAt}. Nothing was changed. Run devfleet from that manager; to make this machine the manager instead, repeat with --take-over once the other one is gone (see docs/manager-recovery.md).`;
+}
+
+/** Refuses when another manager holds the lease. Call it before the first
+ * write of a session.
+ */
+export function refuseHeldLease(context: StateContext): void {
+  const held = leaseConflict(context);
+  if (held) throw new LeaseHeldError(leaseHeldMessage(held));
+}
+
+/** Takes or renews the lease for this machine: when it is not this machine's
+ * yet, or half spent. Call it at the end of a session that may write.
+ */
+export function claimLease(
+  context: StateContext,
+  now: Date = (context.now ?? (() => new Date()))()
+): void {
+  refuseHeldLease(context);
+  if (!context.holder) return;
+  const lease = readLease(databasePath(context.root));
+  const fresh =
+    lease?.holder === context.holder &&
+    Date.parse(lease.expiresAt) - now.getTime() > LEASE_MS / 2;
+  if (fresh) return;
+  writeLease(databasePath(context.root), {
+    holder: context.holder,
+    expiresAt: new Date(now.getTime() + LEASE_MS).toISOString(),
+  });
+}
+
 export const DIVERGED_REMEDY =
   'This machine and the bucket both changed the fleet state. The bucket is the source of truth: keep a copy of anything you still need (devfleet fleet-state --export), then run devfleet fleet-state --restore --yes.';
 
@@ -346,7 +465,9 @@ export function mutateState<T>(
   return locked(context.root, async () => {
     const before = await observed(context, false);
     if (before === 'diverged') throw new FleetUsageError(DIVERGED_REMEDY);
+    refuseHeldLease(context);
     const value = await change(before);
+    claimLease(context);
     return { value, standing: await settle(context, before) };
   });
 }

package/src/worker-access-checks.ts

@@ -4,29 +4,79 @@ import { secretsProvider } from './secrets';
 import { registry } from './agents/registry';
 import { resolveVaultConfig, usesVault } from './vault-config';
 import { record } from './config';
+import { githubCredential } from './github/credential';
+import { hostId } from './topology';
+import { hostname } from 'node:os';
+import { githubSignInState } from './github/sync';
 
 /** Inject every subprocess; never put credentials in argv, reports, or logs. */
-export async function workerAccessChecks(run: Runner): Promise<Check[]> {
-  const policy = fleet.worker!;
-  const github = await run(['gh', 'api', 'user', '--jq', '.login']);
+export async function workerAccessChecks(
+  run: Runner,
+  nodeId?: string
+): Promise<Check[]> {
   return [
-    {
-      id: 'worker:github-principal',
-      status:
-        github.ok && github.stdout.trim() === policy.githubLogin
-          ? 'pass'
-          : 'fail',
-      summary:
-        'Worker GitHub authentication must match the declared common account.',
-      remedy: {
-        action:
-          'Authenticate this worker with the declared GitHub account. Credentials stay on their original machines.',
-      },
-    },
+    await githubSync(run, nodeId),
     await proxyAuthentication(run),
     ...(usesVault() ? [await vaultRenewalCheck(run)] : []),
   ];
 }
+
+const GITHUB_REMEDIES: Record<string, string> = {
+  GITHUB_TOKEN_MISSING:
+    'The secrets provider holds no GitHub token for this worker. Store it under github-token (github-token-<node> with github.credential "per-node"); the next sign-in job or fleet-sync signs the worker in.',
+  GITHUB_TOKEN_REJECTED:
+    'GitHub refuses the stored token: it was revoked or has expired. Store a new one under github-token; nothing about the secret store itself needs fixing.',
+  GITHUB_LOGIN_MISMATCH:
+    'The stored token belongs to another account than worker.githubLogin. Store the fleet login’s token, or correct worker.githubLogin in the fleet configuration.',
+  GITHUB_UNREACHABLE:
+    'GitHub did not answer from this worker. Check its network and DNS; the worker keeps the sign-in it has.',
+  GITHUB_SIGN_IN_FAILED:
+    'The stored token is valid but this worker has not adopted it. Inspect the job with: journalctl --user -u personal-worker-github.service, or run fleet-sync.',
+};
+
+/** GitHub sign-in is measured on its own, so a GitHub fault is never reported
+ * as a fault of the secret store's renewal, and the code names the system to
+ * fix.
+ */
+async function githubSync(run: Runner, nodeId?: string): Promise<Check> {
+  const login = fleet.worker?.githubLogin;
+  if (!login)
+    return {
+      id: 'worker:github-sync',
+      status: 'fail',
+      summary: 'GITHUB_LOGIN_UNDECLARED',
+      remedy: {
+        action: 'Declare the fleet GitHub login as worker.githubLogin.',
+      },
+    };
+  const code = await githubSignInState({
+    run,
+    login,
+    token: githubCredential(fleet, nodeId ?? hostId(hostname()), secrets(run)),
+  });
+  return code === 'GITHUB_OK'
+    ? {
+        id: 'worker:github-sync',
+        status: 'pass',
+        summary:
+          'Worker GitHub authentication matches the declared common account.',
+      }
+    : {
+        id: 'worker:github-sync',
+        status: 'fail',
+        summary: code,
+        remedy: { action: GITHUB_REMEDIES[code]! },
+      };
+}
+
+const secrets = (run: Runner) =>
+  secretsProvider(fleet.secrets, {
+    run: async (argv) => {
+      const result = await run(argv, { timeoutMs: 20_000 });
+      return result.ok ? result.stdout : null;
+    },
+  });
+
 /** Vault refuses a read the token's policy does not allow with a 403. */
 const PERMISSION_DENIED = /permission denied|Code: 403/i;
 

package/src/worker-readiness.ts

@@ -16,7 +16,7 @@ export async function workerReadiness(
   root = process.cwd()
 ): Promise<Check[]> {
   const host = await workerHostChecks(userInfo().username, run);
-  const access = await workerAccessChecks(run);
+  const access = await workerAccessChecks(run, nodeId);
   const revision = await run(['git', 'rev-parse', 'HEAD']);
   const singleton = await run(
     [