@crvouga/dev-fleet 0.3.0 → 0.4.0

package/schema/fleet.schema.json

@@ -107,6 +107,14 @@
                 "description": "The port the worker T3 server listens on, normally 3773."
               }
             }
+          },
+          "toolProfiles": {
+            "type": "array",
+            "uniqueItems": true,
+            "items": {
+              "enum": ["web", "go", "rust", "jvm"]
+            },
+            "description": "Tool profiles this worker installs on top of the fleet-wide ones: web (pnpm, Deno, Yarn), go, rust (with CMake and Ninja), jvm (Temurin). Each is a locked tool set dev-fleet ships."
           }
         }
       }
@@ -191,6 +199,49 @@
               "type": "string",
               "format": "email",
               "description": "Read-only Beszel account visitors are signed in as."
+            },
+            "backups": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["schedule", "keep", "storage"],
+              "description": "Scheduled off-machine backups of the monitoring hub. fleet-sync sets them through the hub's API, with the keys of the secrets monitoring-backup-access-key and monitoring-backup-secret-key; without this block the hub's backups are only read, and beszel:backup warns.",
+              "properties": {
+                "schedule": {
+                  "type": "string",
+                  "pattern": "^[0-9*/,-]+( [0-9*/,-]+){4}$",
+                  "description": "A cron expression in the hub's time zone, for example 0 3 * * *."
+                },
+                "keep": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 365,
+                  "description": "How many scheduled backups the hub keeps."
+                },
+                "storage": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["bucket", "endpoint", "region"],
+                  "description": "S3-compatible storage off the manager. Never its keys.",
+                  "properties": {
+                    "bucket": {
+                      "type": "string",
+                      "pattern": "^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"
+                    },
+                    "endpoint": {
+                      "type": "string",
+                      "pattern": "^https://[A-Za-z0-9.-]+(:[0-9]+)?$"
+                    },
+                    "region": {
+                      "type": "string",
+                      "pattern": "^[a-z0-9-]{1,32}$"
+                    },
+                    "forcePathStyle": {
+                      "type": "boolean",
+                      "description": "Path-style requests, for services that need them (MinIO)."
+                    }
+                  }
+                }
+              }
             }
           }
         }
@@ -263,9 +314,26 @@
         },
         "neverSleep": {
           "const": true
+        },
+        "toolProfiles": {
+          "type": "array",
+          "uniqueItems": true,
+          "items": {
+            "enum": ["web", "go", "rust", "jvm"]
+          },
+          "description": "Tool profiles every worker installs on top of the base tools: web (pnpm, Deno, Yarn), go, rust (with CMake and Ninja), jvm (Temurin). Each worker passes a tool:<name> check for each of their tools."
+        },
+        "tools": {
+          "type": "array",
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "pattern": "^[A-Za-z0-9@:/._-]{1,75}$"
+          },
+          "description": "Single tools every worker installs: names of tools dev-fleet's mise.toml pins (for example fd or ripgrep). A tool it does not pin is refused; ask for it in an issue."
         }
       },
-      "description": "Policy every worker must meet. All but the two logins and the capabilities are fixed."
+      "description": "Policy every worker must meet. All but the two logins, the capabilities and the tools are fixed."
     },
     "source": {
       "type": "object",
@@ -475,12 +543,121 @@
                       "pattern": "^[A-Za-z0-9_]+$"
                     }
                   }
+                },
+                "monitoring-backup-access-key": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["path", "field"],
+                  "properties": {
+                    "path": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9._/-]+$"
+                    },
+                    "field": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9_]+$"
+                    }
+                  },
+                  "description": "Only with services.monitoring.backups: the access key of the hub's backup storage."
+                },
+                "monitoring-backup-secret-key": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["path", "field"],
+                  "properties": {
+                    "path": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9._/-]+$"
+                    },
+                    "field": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9_]+$"
+                    }
+                  },
+                  "description": "Only with services.monitoring.backups: the secret key of the hub's backup storage."
+                },
+                "notify-webhook-url": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["path", "field"],
+                  "properties": {
+                    "path": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9._/-]+$"
+                    },
+                    "field": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9_]+$"
+                    }
+                  },
+                  "description": "Only with the webhook notify target: the URL fleet-watch POSTs JSON to."
+                },
+                "notify-ntfy-url": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["path", "field"],
+                  "properties": {
+                    "path": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9._/-]+$"
+                    },
+                    "field": {
+                      "type": "string",
+                      "pattern": "^[A-Za-z0-9_]+$"
+                    }
+                  },
+                  "description": "Only with the ntfy notify target: the ntfy topic URL fleet-watch publishes to."
                 }
               }
             }
           }
         }
       ]
+    },
+    "git": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "email"],
+      "description": "The Git identity every node, the manager included, commits with, and the GitHub credential helper it pushes with. fleet-sync writes them in ~/.config/git/config and measures them as worker:git-identity and worker:git-push-auth.",
+      "properties": {
+        "name": {
+          "type": "string",
+          "pattern": "^[^\"\\\\\\n\\r]{1,100}$",
+          "description": "user.name of every commit."
+        },
+        "email": {
+          "type": "string",
+          "format": "email",
+          "description": "user.email of every commit, for example the account's noreply address."
+        },
+        "verifyRepository": {
+          "type": "string",
+          "pattern": "^https://github\\.com/[A-Za-z0-9_.-]{1,100}/[A-Za-z0-9_.-]{1,100}$",
+          "description": "A GitHub repository the fleet's account can push to, ideally private. Each node lists it through git with its own credentials."
+        }
+      }
+    },
+    "notify": {
+      "description": "Where fleet-watch sends a notification when a machine becomes unhealthy or recovers. Each target's address is a secret: notify-webhook-url (a JSON POST) or notify-ntfy-url (an ntfy topic URL).",
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["targets"],
+      "properties": {
+        "targets": {
+          "type": "array",
+          "minItems": 1,
+          "uniqueItems": true,
+          "items": {
+            "enum": ["webhook", "ntfy"]
+          }
+        },
+        "repeatHours": {
+          "description": "Notify again about a machine that stays unhealthy this often. 0 never repeats; the default is 24.",
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 720
+        }
+      }
     }
   }
 }

package/schema/output/check.schema.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "check.schema.json",
+  "title": "One check of a dev-fleet report",
+  "description": "The id is listed in schema/check-ids.json.",
+  "type": "object",
+  "required": ["id", "status", "summary"],
+  "properties": {
+    "id": { "type": "string", "pattern": "^[a-z0-9-]+(:[A-Za-z0-9:._-]+)?$" },
+    "status": { "enum": ["pass", "warn", "fail", "skip"] },
+    "summary": { "type": "string" },
+    "required": { "type": "boolean" },
+    "phaseDurationMs": { "type": "number" },
+    "remedy": {
+      "type": "object",
+      "required": ["action"],
+      "properties": {
+        "action": { "type": "string" },
+        "command": { "type": "string" }
+      }
+    }
+  }
+}

package/schema/output/error.schema.json

@@ -0,0 +1,14 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "error.schema.json",
+  "title": "A dev-fleet error, version 1",
+  "description": "What a command prints with --json when it fails before a report: raw subprocess output and exception details are withheld.",
+  "type": "object",
+  "required": ["version", "ok", "code", "error"],
+  "properties": {
+    "version": { "const": 1 },
+    "ok": { "const": false },
+    "code": { "type": "string" },
+    "error": { "type": "string" }
+  }
+}

package/schema/output/fleet-report.schema.json

@@ -0,0 +1,79 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "fleet-report.schema.json",
+  "title": "The fleet-report output, version 1",
+  "type": "object",
+  "required": [
+    "version",
+    "command",
+    "ok",
+    "since",
+    "runs",
+    "succeeded",
+    "successRate",
+    "meanDurationMs",
+    "failingChecks",
+    "notReady",
+    "entries"
+  ],
+  "properties": {
+    "version": { "const": 1 },
+    "command": { "const": "fleet-report" },
+    "ok": { "type": "boolean" },
+    "since": { "type": "string" },
+    "runs": { "type": "integer" },
+    "succeeded": { "type": "integer" },
+    "successRate": { "type": ["number", "null"] },
+    "meanDurationMs": { "type": ["number", "null"] },
+    "failingChecks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["id", "runs"],
+        "properties": {
+          "id": { "type": "string" },
+          "runs": { "type": "integer" }
+        }
+      }
+    },
+    "notReady": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["node", "ms"],
+        "properties": {
+          "node": { "type": "string" },
+          "ms": { "type": "number" }
+        }
+      }
+    },
+    "entries": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "startedAt",
+          "finishedAt",
+          "durationMs",
+          "trigger",
+          "revision",
+          "ok",
+          "exitCode",
+          "failing",
+          "notReady"
+        ],
+        "properties": {
+          "startedAt": { "type": "string" },
+          "finishedAt": { "type": "string" },
+          "durationMs": { "type": "number" },
+          "trigger": { "enum": ["operator", "schedule"] },
+          "revision": { "type": ["string", "null"] },
+          "ok": { "type": "boolean" },
+          "exitCode": { "type": "integer" },
+          "failing": { "type": "array", "items": { "type": "string" } },
+          "notReady": { "type": "array", "items": { "type": "string" } }
+        }
+      }
+    }
+  }
+}

package/schema/output/fleet-sync.schema.json

@@ -0,0 +1,112 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "fleet-sync.schema.json",
+  "title": "The fleet-sync report, version 3",
+  "description": "What devfleet fleet-sync prints with --json. A preview (--dry-run) adds plan.",
+  "type": "object",
+  "required": [
+    "version",
+    "command",
+    "execution",
+    "ok",
+    "applied",
+    "desiredRevision",
+    "checks",
+    "nodes",
+    "candidates",
+    "dashboardUrl"
+  ],
+  "properties": {
+    "version": { "const": 3 },
+    "command": { "const": "fleet-sync" },
+    "execution": { "const": "manager-push" },
+    "ok": { "type": "boolean" },
+    "applied": { "type": "boolean" },
+    "desiredRevision": { "type": ["string", "null"] },
+    "checks": { "type": "array", "items": { "$ref": "check.schema.json" } },
+    "candidates": { "type": "integer" },
+    "dashboardUrl": { "type": ["string", "null"] },
+    "planFingerprint": { "type": "string" },
+    "nodes": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "node",
+          "checks",
+          "reachability",
+          "desiredRevision",
+          "appliedRevision",
+          "durationMs"
+        ],
+        "properties": {
+          "node": { "type": "string" },
+          "role": { "enum": ["manager", "worker"] },
+          "reachability": {
+            "enum": ["online", "offline", "missing", "conflict", "unknown"]
+          },
+          "desiredRevision": { "type": ["string", "null"] },
+          "appliedRevision": { "type": ["string", "null"] },
+          "durationMs": { "type": "number" },
+          "checks": {
+            "type": "array",
+            "items": { "$ref": "check.schema.json" }
+          }
+        }
+      }
+    },
+    "plan": {
+      "type": "object",
+      "required": ["state", "revision", "known", "tools", "restarts", "nodes"],
+      "properties": {
+        "state": { "type": "string" },
+        "known": { "type": "boolean" },
+        "revision": {
+          "type": "object",
+          "required": ["from", "to"],
+          "properties": {
+            "from": { "type": ["string", "null"] },
+            "to": { "type": "string" }
+          }
+        },
+        "tools": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["tool", "from", "to"],
+            "properties": {
+              "tool": { "type": "string" },
+              "from": { "type": ["string", "null"] },
+              "to": { "type": ["string", "null"] }
+            }
+          }
+        },
+        "restarts": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["service", "reason", "endsRunningTurns"],
+            "properties": {
+              "service": { "type": "string" },
+              "reason": { "type": "string" },
+              "endsRunningTurns": { "type": "boolean" }
+            }
+          }
+        },
+        "nodes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["node", "from", "to", "changes"],
+            "properties": {
+              "node": { "type": "string" },
+              "from": { "type": ["string", "null"] },
+              "to": { "type": "string" },
+              "changes": { "type": "boolean" }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/schema/output/report.schema.json

@@ -0,0 +1,38 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "report.schema.json",
+  "title": "A dev-fleet diagnostic report, version 1",
+  "description": "What fleet-status, fleet-state and the other diagnostic commands print with --json.",
+  "type": "object",
+  "required": ["version", "command", "ok", "generatedAt", "nodes", "summary"],
+  "properties": {
+    "version": { "const": 1 },
+    "command": { "type": "string" },
+    "ok": { "type": "boolean" },
+    "generatedAt": { "type": "string" },
+    "nodes": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["node", "checks"],
+        "properties": {
+          "node": { "type": "string" },
+          "checks": {
+            "type": "array",
+            "items": { "$ref": "check.schema.json" }
+          }
+        }
+      }
+    },
+    "summary": {
+      "type": "object",
+      "required": ["pass", "warn", "fail", "skip"],
+      "properties": {
+        "pass": { "type": "integer" },
+        "warn": { "type": "integer" },
+        "fail": { "type": "integer" },
+        "skip": { "type": "integer" }
+      }
+    }
+  }
+}

package/src/auth-checks.ts

@@ -5,6 +5,7 @@ import { record } from './config';
 import { codingAuth, serviceAuth } from './service-auth';
 import { workstationChecks } from './workstation-checks';
 import { toolchainChecks } from './toolchain-checks';
+import { workerSelection } from '../bootstrap/worker-tools';
 import type { Capability, Check, Runner, CommandResult } from './types';
 
 export async function authCheck(
@@ -24,6 +25,8 @@ export async function authCheck(
       root,
       run,
       definition.workers.some((node) => node.id === nodeId)
+        ? workerSelection(definition, nodeId)
+        : undefined
     );
   return basicAuth(capability, run, root);
 }

package/src/cli/agent-guide.ts

@@ -63,7 +63,7 @@ export function agentGuide(release: Release) {
       'devfleet --interactive (explicit opt-in; TTY required, disabled in CI)',
     groups: {
       'fleet-*':
-        'Developer computers: fleet-status, fleet-discover, fleet-sync, fleet-state, fleet-node-sync, fleet-repair, fleet-nodes, fleet-node-add',
+        'Developer computers: fleet-status, fleet-discover, fleet-sync, fleet-state, fleet-report, fleet-node-sync, fleet-repair, fleet-nodes, fleet-node-add',
       't3-*': 'Declared T3 provider instances: t3-status, t3-apply, t3-cleanup',
       cliproxy: 'The CLI proxy service on the manager; arguments are forwarded',
     },
@@ -89,9 +89,11 @@ export function agentGuide(release: Release) {
         '130': 'Cancelled',
       },
       errors:
-        '{version:1,ok:false,code,error}; raw subprocess output and exception details are withheld',
+        'schema/output/error.schema.json: {version:1,ok:false,code,error}; raw subprocess output and exception details are withheld',
       reports:
-        '{version:1,command,ok,generatedAt,nodes:[{node,checks:[{id,status,summary,remedy?}]}],summary,coverage}',
+        'Each output has a JSON schema in this package; parse by it and by its version. Diagnostic reports (fleet-status, fleet-state, fleet-repair): schema/output/report.schema.json, version 1. fleet-sync: schema/output/fleet-sync.schema.json, version 3 (a --dry-run adds plan). fleet-report: schema/output/fleet-report.schema.json, version 1.',
+      checks:
+        'schema/check-ids.json lists every check ID with what a passing check means; * stands for a name. Each failing check carries its own remedy.',
       statuses: ['pass', 'warn', 'fail', 'skip'],
       coverage:
         'Default fleet-status tests the manager and all configured nodes. --local tests only this manager; --tailnet and --workers cover Tailscale computers. A missing roster fails explicitly. T3/VS Code end-to-end checks remain manual.',

package/src/cli/fleet-report.ts

@@ -0,0 +1,126 @@
+import { FleetUsageError } from '../errors';
+import { readHistory, type SyncRun } from '../state/history';
+import type { CommandOutcome } from './contracts';
+
+export interface FleetReportOptions {
+  since?: string;
+  json?: boolean;
+}
+
+const UNITS: Record<string, number> = {
+  m: 60_000,
+  h: 3_600_000,
+  d: 86_400_000,
+};
+
+/** `30d`, `12h` or `90m`: how far back the report looks. */
+export function parseSince(value = '30d'): number {
+  const match = /^([1-9]\d{0,4})([mhd])$/.exec(value);
+  if (!match)
+    throw new FleetUsageError(
+      '--since takes a duration such as 30d, 12h or 90m.'
+    );
+  return Number(match[1]) * UNITS[match[2]!]!;
+}
+
+export interface Reliability {
+  runs: number;
+  succeeded: number;
+  successRate: number | null;
+  meanDurationMs: number | null;
+  failingChecks: { id: string; runs: number }[];
+  notReady: { node: string; ms: number }[];
+}
+
+/** How long each machine was not ready: from the end of a run that left it
+ * not ready to the end of the next run that found it ready (or until now).
+ */
+function notReadyTime(runs: SyncRun[], now: Date): Map<string, number> {
+  const since = new Map<string, number>();
+  const total = new Map<string, number>();
+  const add = (node: string, until: number) => {
+    total.set(node, (total.get(node) ?? 0) + until - since.get(node)!);
+    since.delete(node);
+  };
+  for (const run of runs) {
+    const at = Date.parse(run.finishedAt);
+    for (const node of [...since.keys()])
+      if (!run.notReady.includes(node)) add(node, at);
+    for (const node of run.notReady) if (!since.has(node)) since.set(node, at);
+  }
+  for (const node of [...since.keys()]) add(node, now.getTime());
+  return total;
+}
+
+export function reliability(runs: SyncRun[], now: Date): Reliability {
+  const succeeded = runs.filter((run) => run.ok).length;
+  const counts = new Map<string, number>();
+  for (const run of runs)
+    for (const id of run.failing) counts.set(id, (counts.get(id) ?? 0) + 1);
+  return {
+    runs: runs.length,
+    succeeded,
+    successRate: runs.length ? succeeded / runs.length : null,
+    meanDurationMs: runs.length
+      ? Math.round(
+          runs.reduce((sum, run) => sum + run.durationMs, 0) / runs.length
+        )
+      : null,
+    failingChecks: [...counts]
+      .map(([id, count]) => ({ id, runs: count }))
+      .sort((a, b) => b.runs - a.runs || a.id.localeCompare(b.id))
+      .slice(0, 10),
+    notReady: [...notReadyTime(runs, now)]
+      .map(([node, ms]) => ({ node, ms }))
+      .sort((a, b) => b.ms - a.ms || a.node.localeCompare(b.node)),
+  };
+}
+
+/** Read-only: the reliability of fleet-sync over a period, from the run
+ * history kept in the fleet state.
+ */
+export function fleetReport(
+  options: FleetReportOptions,
+  env = process.env,
+  now = new Date()
+): CommandOutcome {
+  const since = options.since ?? '30d';
+  const from = now.getTime() - parseSince(since);
+  const runs = readHistory(env).runs.filter(
+    (run) => Date.parse(run.finishedAt) >= from
+  );
+  const summary = reliability(runs, now);
+  const data = {
+    version: 1,
+    command: 'fleet-report',
+    ok: true,
+    since,
+    ...summary,
+    entries: runs,
+  };
+  return { exitCode: 0, data, text: render(since, summary) };
+}
+
+function minutes(ms: number): string {
+  return `${Math.round(ms / 60_000)} min`;
+}
+
+function render(since: string, summary: Reliability): string {
+  if (!summary.runs) return `No fleet-sync run in the last ${since}.`;
+  const lines = [
+    `fleet-sync over the last ${since}: ${summary.runs} runs, ${summary.succeeded} succeeded (${Math.round((summary.successRate ?? 0) * 100)}%), ${Math.round((summary.meanDurationMs ?? 0) / 1000)} s on average.`,
+  ];
+  if (summary.failingChecks.length)
+    lines.push(
+      'Most frequent failing checks:',
+      ...summary.failingChecks.map(
+        (check) => `  ${check.id}  ${check.runs} runs`
+      )
+    );
+  if (summary.notReady.length)
+    lines.push(
+      'Time not ready:',
+      ...summary.notReady.map((node) => `  ${node.node}  ${minutes(node.ms)}`)
+    );
+  return lines.join('\n');
+}