@crvouga/dev-fleet 0.2.0 → 0.4.0

package/src/state/history.ts

@@ -0,0 +1,172 @@
+import { record } from '../config';
+import { databasePath, stateRoot } from './paths';
+import { readDocument, writeDocument } from './store';
+
+/** Who started a fleet-sync run: a person (or an agent) at the manager, or
+ * the manager's own schedule.
+ */
+export type Trigger = 'operator' | 'schedule';
+
+/** One fleet-sync run, as the history keeps it: codes and node names only,
+ * never a remedy, a path or anything a subprocess printed.
+ */
+export interface SyncRun {
+  startedAt: string;
+  finishedAt: string;
+  durationMs: number;
+  trigger: Trigger;
+  revision: string | null;
+  ok: boolean;
+  exitCode: number;
+  /** The IDs of the checks that failed, once each. */
+  failing: string[];
+  /** The machines that were not ready when the run ended. */
+  notReady: string[];
+}
+
+/** The schedule the manager was last told to keep. */
+export interface ScheduleRecord {
+  enabled: boolean;
+  intervalSeconds: number;
+  changedAt: string;
+}
+
+export interface History {
+  version: 1;
+  schedule?: ScheduleRecord;
+  runs: SyncRun[];
+}
+
+/** The history is capped twice: by age and by count, so the document stays
+ * far below the fleet state's limit on one document.
+ */
+export const HISTORY_DAYS = 90;
+export const HISTORY_RUNS = 500;
+
+const DAY_MS = 86_400_000;
+
+function strings(value: unknown): value is string[] {
+  return Array.isArray(value) && value.every((v) => typeof v === 'string');
+}
+
+function syncRun(value: unknown): value is SyncRun {
+  return (
+    record(value) &&
+    typeof value['startedAt'] === 'string' &&
+    typeof value['finishedAt'] === 'string' &&
+    typeof value['durationMs'] === 'number' &&
+    (value['trigger'] === 'operator' || value['trigger'] === 'schedule') &&
+    (value['revision'] === null || typeof value['revision'] === 'string') &&
+    typeof value['ok'] === 'boolean' &&
+    typeof value['exitCode'] === 'number' &&
+    strings(value['failing']) &&
+    strings(value['notReady'])
+  );
+}
+
+function scheduleRecord(value: unknown): value is ScheduleRecord {
+  return (
+    record(value) &&
+    typeof value['enabled'] === 'boolean' &&
+    typeof value['intervalSeconds'] === 'number' &&
+    typeof value['changedAt'] === 'string'
+  );
+}
+
+/** A document that cannot be read is an empty history: it only ever
+ * describes the past, and the next run starts it again.
+ */
+export function parseHistory(text: string | null): History {
+  let value: unknown = null;
+  try {
+    value = text === null ? null : JSON.parse(text);
+  } catch {
+    value = null;
+  }
+  if (!record(value) || value['version'] !== 1) return { version: 1, runs: [] };
+  return {
+    version: 1,
+    ...(scheduleRecord(value['schedule'])
+      ? { schedule: value['schedule'] }
+      : {}),
+    runs: Array.isArray(value['runs']) ? value['runs'].filter(syncRun) : [],
+  };
+}
+
+export function readHistory(env = process.env): History {
+  return parseHistory(readDocument(databasePath(stateRoot(env)), 'history'));
+}
+
+function writeHistory(history: History, env: NodeJS.ProcessEnv): void {
+  writeDocument(
+    databasePath(stateRoot(env)),
+    'history',
+    JSON.stringify(history) + '\n'
+  );
+}
+
+/** Keeps the runs of the last HISTORY_DAYS days, and at most HISTORY_RUNS. */
+export function capped(runs: SyncRun[], now: Date): SyncRun[] {
+  const oldest = now.getTime() - HISTORY_DAYS * DAY_MS;
+  return runs
+    .filter((run) => Date.parse(run.finishedAt) >= oldest)
+    .slice(-HISTORY_RUNS);
+}
+
+export function appendRun(
+  run: SyncRun,
+  env = process.env,
+  now = new Date()
+): History {
+  const history = readHistory(env);
+  const next = { ...history, runs: capped([...history.runs, run], now) };
+  writeHistory(next, env);
+  return next;
+}
+
+export function recordSchedule(
+  schedule: ScheduleRecord,
+  env = process.env
+): void {
+  writeHistory({ ...readHistory(env), schedule }, env);
+}
+
+/** What a fleet-sync report says about one run. */
+export function summarizeRun(
+  data: unknown,
+  timing: { startedAt: Date; finishedAt: Date },
+  trigger: Trigger,
+  exitCode: number
+): SyncRun {
+  const report = record(data) ? data : {};
+  const checks = (value: unknown): Record<string, unknown>[] =>
+    Array.isArray(value) ? value.filter(record) : [];
+  const nodes = checks(report['nodes']);
+  const failing = new Set<string>();
+  for (const check of [
+    ...checks(report['checks']),
+    ...nodes.flatMap((node) => checks(node['checks'])),
+  ])
+    if (check['status'] === 'fail' && typeof check['id'] === 'string')
+      failing.add(check['id']);
+  const notReady = nodes
+    .filter((node) =>
+      checks(node['checks']).some((check) => check['status'] === 'fail')
+    )
+    .map((node) => node['node'])
+    .filter((node): node is string => typeof node === 'string');
+  return {
+    startedAt: timing.startedAt.toISOString(),
+    finishedAt: timing.finishedAt.toISOString(),
+    durationMs: timing.finishedAt.getTime() - timing.startedAt.getTime(),
+    trigger,
+    revision:
+      typeof report['desiredRevision'] === 'string'
+        ? report['desiredRevision']
+        : null,
+    ok: exitCode === 0 && report['ok'] !== false,
+    exitCode,
+    failing: [...failing].sort(),
+    notReady: [...new Set(notReady)].sort(),
+  };
+}

package/src/state/store.ts

@@ -9,7 +9,14 @@ import { FleetUsageError } from '../errors';
  * connection to each worker, `enrollment` the approved and applied revision.
  */
 export const DOCUMENTS = ['definition', 'roster', 'enrollment'] as const;
+/** Kept when there is something to keep; a fleet without them is complete:
+ * `history` is the record of recent fleet-sync runs and of the schedule.
+ */
+export const OPTIONAL_DOCUMENTS = ['history'] as const;
 export type DocumentName = (typeof DOCUMENTS)[number];
+/** Every document the fleet state may hold. */
+export type StoredName = DocumentName | (typeof OPTIONAL_DOCUMENTS)[number];
+const KNOWN: readonly string[] = [...DOCUMENTS, ...OPTIONAL_DOCUMENTS];
 
 /** One version of the whole database. Every change replaces it, so two copies
  * hold the same state exactly when their heads are equal.
@@ -21,7 +28,7 @@ export interface Head {
 }
 
 export interface DocumentInfo {
-  name: DocumentName;
+  name: StoredName;
   bytes: number;
   updatedAt: string;
 }
@@ -100,7 +107,7 @@ function tables(db: Database): boolean {
   );
 }
 
-export function readDocument(file: string, name: DocumentName): string | null {
+export function readDocument(file: string, name: StoredName): string | null {
   return reading(file, (db) => {
     if (!tables(db)) return null;
     const row = db
@@ -119,7 +126,7 @@ export function listDocuments(file: string): DocumentInfo[] {
           'SELECT name, length(CAST(body AS BLOB)) AS bytes, updated_at AS updatedAt FROM documents ORDER BY name'
         )
         .all() as DocumentInfo[];
-      return rows.filter((row) => DOCUMENTS.includes(row.name));
+      return rows.filter((row) => KNOWN.includes(row.name));
     }) ?? []
   );
 }
@@ -138,7 +145,7 @@ export function readHead(file: string): Head | null {
  */
 export function writeDocument(
   file: string,
-  name: DocumentName,
+  name: StoredName,
   body: string
 ): Head {
   if (Buffer.byteLength(body) > MAX_DOCUMENT_BYTES)

package/src/t3/accounts.ts

@@ -46,6 +46,12 @@ const TOOLS: Record<
   },
 };
 
+/** The T3 instance of one account, and the T3 driver that runs its tool. */
+export const accountInstanceId = (account: Account): string =>
+  `personal_${slug(account.prefix)}`.slice(0, 64);
+export const accountDriver = (tool: string): string | undefined =>
+  TOOLS[tool]?.driver;
+
 /** One `<Provider> <account email>` instance per signed-in account, identical
  * on every machine because every machine reads the same proxy.
  */
@@ -61,7 +67,7 @@ export function accountInstances(
       ...variables,
       account: slug(account.email),
     };
-    instances[`personal_${slug(account.prefix)}`.slice(0, 64)] = {
+    instances[accountInstanceId(account)] = {
       driver: tool.driver,
       displayName: `${tool.label} ${account.email}`,
       enabled: true,

package/src/t3/models.ts

@@ -0,0 +1,133 @@
+import { join } from 'node:path';
+import type { Check } from '../types';
+import { accountDriver, accountInstanceId, type Roster } from './accounts';
+import type { Account } from '../agents/accounts';
+import { object, read } from './files';
+
+/** The models T3 offers for one instance: the catalogue T3 keeps for its
+ * driver (userdata/model-manifest.json) and any custom models the instance
+ * adds. Null when T3 has no list for the driver: it asks the tool itself.
+ */
+function offered(
+  manifest: Record<string, unknown>,
+  instance: unknown,
+  driver: string
+): string[] | null {
+  const catalogue = object(
+    JSON.stringify(
+      object(JSON.stringify(manifest['manifest'] ?? {}))['currentModels'] ?? {}
+    )
+  )[driver];
+  const config =
+    instance && typeof instance === 'object'
+      ? (instance as { config?: { customModels?: unknown } }).config
+      : undefined;
+  const custom = Array.isArray(config?.customModels)
+    ? config.customModels.flatMap((model: unknown) =>
+        typeof model === 'string'
+          ? [model]
+          : model &&
+              typeof model === 'object' &&
+              typeof (model as { slug?: unknown }).slug === 'string'
+            ? [(model as { slug: string }).slug]
+            : []
+      )
+    : [];
+  if (!Array.isArray(catalogue) && !custom.length) return null;
+  return [
+    ...new Set([
+      ...(Array.isArray(catalogue)
+        ? catalogue.filter(
+            (model): model is string => typeof model === 'string'
+          )
+        : []),
+      ...custom,
+    ]),
+  ].sort();
+}
+
+/** `t3:models`: for every account instance the fleet manages, T3 offers only
+ * models the proxy serves for that account, and every required model is both
+ * served and offered. A report names tools and models, never an account.
+ */
+export async function modelsCheck(options: {
+  t3Home: string;
+  roster: Roster;
+  required?: Record<string, string[]>;
+}): Promise<Check> {
+  const remedy = (action: string) => ({ remedy: { action } });
+  let accounts;
+  let settings;
+  let manifest;
+  try {
+    accounts = await options.roster();
+    settings = object(
+      await read(join(options.t3Home, 'userdata/settings.json'))
+    );
+    manifest = object(
+      await read(join(options.t3Home, 'userdata/model-manifest.json'))
+    );
+  } catch {
+    return {
+      id: 't3:models',
+      status: 'warn',
+      summary: 'T3_MODELS_UNVERIFIED',
+      ...remedy(
+        'The proxy’s model list or T3’s settings could not be read on this machine. Check that the CLI proxy answers and T3 has started once, then run fleet-sync.'
+      ),
+    };
+  }
+  const problems = mismatches(
+    accounts,
+    object(JSON.stringify(settings['providerInstances'] ?? {})),
+    manifest,
+    options.required ?? {}
+  );
+  if (problems.size)
+    return {
+      id: 't3:models',
+      status: 'warn',
+      summary: 'T3_MODELS_MISMATCH',
+      ...remedy(
+        `${[...problems].sort().join('; ')}. Serve the model on the CLI proxy, or remove it from t3.requiredModels in fleet.json; a model the proxy no longer serves fails when it is chosen in T3.`
+      ),
+    };
+  return {
+    id: 't3:models',
+    status: 'pass',
+    summary:
+      'Every model T3 offers for a managed account is one the proxy serves for it, and every required model is offered.',
+  };
+}
+
+/** What is wrong with the models of each managed account instance. */
+function mismatches(
+  accounts: Account[],
+  instances: Record<string, unknown>,
+  manifest: Record<string, unknown>,
+  required: Record<string, string[]>
+): Set<string> {
+  const problems = new Set<string>();
+  for (const account of accounts) {
+    const driver = accountDriver(account.tool);
+    const id = accountInstanceId(account);
+    if (!driver || !Object.hasOwn(instances, id)) continue;
+    const offers = offered(manifest, instances[id], driver);
+    for (const model of offers ?? [])
+      if (!account.models.includes(model))
+        problems.add(
+          `${account.tool}: T3 offers ${model}, which the proxy does not serve`
+        );
+    for (const model of required[account.tool] ?? []) {
+      if (!account.models.includes(model))
+        problems.add(
+          `${account.tool}: the proxy does not serve the required ${model}`
+        );
+      else if (offers && !offers.includes(model))
+        problems.add(
+          `${account.tool}: T3 does not offer the required ${model}`
+        );
+    }
+  }
+  return problems;
+}

package/src/t3/pairing.ts

@@ -0,0 +1,31 @@
+import { parseConfig } from '../config';
+import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
+import { mutateState, stateContext, type StateContext } from '../state/sync';
+
+/** Records the T3 environment a worker runs in the roster, the first time a
+ * verified link from the manager reaches it. From then on the link check
+ * compares against this identity, so a worker whose T3 data was replaced is
+ * told apart from one that is only offline. An identity already recorded is
+ * never rewritten.
+ */
+export async function rememberEnvironment(
+  path: string,
+  node: string,
+  environmentId: string,
+  context: () => StateContext = stateContext
+): Promise<boolean> {
+  const save = async (): Promise<boolean> => {
+    const text = await readRosterText(path);
+    if (text === null) return false;
+    const roster = parseConfig(JSON.parse(text) as unknown);
+    const entry = roster.nodes.find((candidate) => candidate.id === node);
+    if (!entry || entry.t3EnvironmentId) return false;
+    entry.t3EnvironmentId = environmentId;
+    // Parsing again refuses an identity that is not a T3 environment ID.
+    const next = parseConfig(roster);
+    await writeRosterText(path, `${JSON.stringify(next, null, 2)}\n`);
+    return true;
+  };
+  if (path !== STATE_ROSTER) return save();
+  return (await mutateState(context(), save)).value;
+}

package/src/t3/profile.ts

@@ -0,0 +1,203 @@
+import { dirname, join } from 'node:path';
+import type { Check } from '../types';
+import { canonical, object, read, write } from './files';
+
+/** What happens to one declared setting. `release` stops managing a key the
+ * fleet no longer declares and leaves its value to the machine.
+ */
+export type ProfileAction =
+  'create' | 'update' | 'delete' | 'release' | 'unchanged' | 'conflict';
+export type ProfileChange = { key: string; action: ProfileAction };
+
+/** The values the fleet last wrote, kept beside T3's settings: the base of
+ * the three-way merge between what is declared, what T3 holds and what was
+ * applied before.
+ */
+export const profileRecord = (settings: string): string =>
+  join(dirname(settings), 'personal-profile.json');
+
+const same = (a: unknown, b: unknown): boolean => canonical(a) === canonical(b);
+
+/** Managed keys converge, keys nobody declared are left alone, and a key
+ * changed on the machine since the fleet last wrote it (or that existed
+ * before the fleet declared it) is a conflict, never overwritten.
+ */
+export function planProfile(
+  settings: Record<string, unknown>,
+  applied: Record<string, unknown>,
+  declared: Record<string, unknown>
+) {
+  const next = { ...settings };
+  const record = { ...applied };
+  const changes: ProfileChange[] = [];
+  const keys = [
+    ...new Set([...Object.keys(declared), ...Object.keys(applied)]),
+  ].sort();
+  for (const key of keys) {
+    const present = Object.hasOwn(settings, key);
+    const managed = Object.hasOwn(applied, key);
+    const action = actionFor(key, declared, {
+      present,
+      managed,
+      current: settings[key],
+      base: applied[key],
+    });
+    changes.push({ key, action });
+    if (action === 'create' || action === 'update') {
+      next[key] = declared[key];
+      record[key] = declared[key];
+    } else if (action === 'unchanged') record[key] = declared[key];
+    else if (action === 'delete') {
+      delete next[key];
+      delete record[key];
+    } else if (action === 'release') delete record[key];
+  }
+  return {
+    settings: next,
+    record,
+    changes,
+    settingsChanged: !same(next, settings),
+    recordChanged: !same(record, applied),
+  };
+}
+
+function actionFor(
+  key: string,
+  declared: Record<string, unknown>,
+  {
+    present,
+    managed,
+    current,
+    base,
+  }: { present: boolean; managed: boolean; current: unknown; base: unknown }
+): ProfileAction {
+  if (!Object.hasOwn(declared, key)) {
+    if (present && same(current, base)) return 'delete';
+    return 'release';
+  }
+  const wanted = declared[key];
+  if (present && same(current, wanted)) return 'unchanged';
+  if (!present) return managed ? 'conflict' : 'create';
+  return managed && same(current, base) ? 'update' : 'conflict';
+}
+
+export interface ProfileOptions {
+  apply: boolean;
+  /** Tries before a settings file T3 keeps rewriting is reported busy. */
+  attempts?: number;
+  pause?: () => Promise<void>;
+  /** Runs between the read and the write: where T3 may write too. */
+  beforeWrite?: () => Promise<void>;
+}
+
+/** Plans, and with `apply` writes, the declared profile. T3 writes the same
+ * file and offers no compare-and-swap, so a write happens only when the file
+ * is still what was read; otherwise the merge is planned again, a bounded
+ * number of times.
+ */
+export async function reconcileProfile(
+  settingsPath: string,
+  declared: Record<string, unknown>,
+  options: ProfileOptions
+) {
+  const recordPath = profileRecord(settingsPath);
+  const attempts = options.attempts ?? 3;
+  const pause =
+    options.pause ?? (() => new Promise((done) => setTimeout(done, 500)));
+  for (let attempt = 1; ; attempt++) {
+    const raw = await read(settingsPath);
+    const recordRaw = await read(recordPath);
+    const plan = planProfile(
+      object(raw),
+      readRecord(recordRaw, settingsPath),
+      declared
+    );
+    if (!options.apply || (!plan.settingsChanged && !plan.recordChanged))
+      return { ...plan, applied: false };
+    await options.beforeWrite?.();
+    if (!plan.settingsChanged || (await read(settingsPath)) === raw) {
+      if (plan.settingsChanged)
+        await write(
+          settingsPath,
+          JSON.stringify(plan.settings, null, 2) + '\n'
+        );
+      await write(
+        recordPath,
+        canonical({ version: 1, target: settingsPath, settings: plan.record }) +
+          '\n'
+      );
+      return { ...plan, applied: true };
+    }
+    if (attempt >= attempts) throw new Error('T3_SETTINGS_BUSY');
+    await pause();
+  }
+}
+
+function readRecord(
+  raw: string | null,
+  target: string
+): Record<string, unknown> {
+  if (raw === null) return {};
+  const record = object(raw);
+  if (record['version'] !== 1 || record['target'] !== target)
+    throw new Error('T3_PROFILE_RECORD_INVALID');
+  return object(JSON.stringify(record['settings'] ?? {}));
+}
+
+/** `t3:profile`: the declared settings on this machine. A report names the
+ * keys and what happens to them, never a value: a tool server's settings may
+ * carry a credential.
+ */
+export async function profileCheck(
+  settingsPath: string,
+  declared: Record<string, unknown>,
+  options: ProfileOptions
+): Promise<Check> {
+  const remedy = (action: string) => ({ remedy: { action } });
+  try {
+    const result = await reconcileProfile(settingsPath, declared, options);
+    const conflicts = result.changes.filter(
+      (change) => change.action === 'conflict'
+    );
+    if (conflicts.length)
+      return {
+        id: 't3:profile',
+        status: 'warn',
+        summary: 'T3_PROFILE_CONFLICT',
+        ...remedy(
+          `These T3 settings were changed on this machine since dev-fleet last wrote them, or existed before they were declared, and were left as they are: ${conflicts.map((change) => change.key).join(', ')}. Put the value every machine should hold under t3.profile in fleet.json, or remove the setting on this machine, then run fleet-sync.`
+        ),
+      };
+    const pending = result.changes.filter(
+      (change) => !['unchanged', 'conflict'].includes(change.action)
+    );
+    if (!options.apply && pending.length)
+      return {
+        id: 't3:profile',
+        status: 'warn',
+        summary: 'T3_PROFILE_PENDING',
+        ...remedy(
+          `fleet-sync would change these T3 settings: ${pending.map((change) => `${change.key} (${change.action})`).join(', ')}.`
+        ),
+      };
+    return {
+      id: 't3:profile',
+      status: 'pass',
+      summary: Object.keys(declared).length
+        ? 'T3 on this machine holds every setting of the declared profile; settings nobody declared are its own.'
+        : 'No T3 profile is declared; every setting stays local.',
+    };
+  } catch (error) {
+    const busy = error instanceof Error && error.message === 'T3_SETTINGS_BUSY';
+    return {
+      id: 't3:profile',
+      status: 'fail',
+      summary: busy ? 'T3_SETTINGS_BUSY' : 'T3_PROFILE_UNVERIFIED',
+      ...remedy(
+        busy
+          ? 'T3 kept rewriting its settings while dev-fleet merged the profile. Run fleet-sync again; nothing was overwritten.'
+          : 'Repair T3’s settings.json (or restore personal-profile.json from a backup) on this machine, then run fleet-sync.'
+      ),
+    };
+  }
+}