@yadurajfleetos/cli 0.11.0 → 0.13.0

package/dist/commands/doctor.js

@@ -3,6 +3,56 @@ import { loadProfile } from '../config.js';
 import { c, relativeTime } from '../render.js';
 import { glyph, rule, task } from '../ui.js';
 const icon = (state) => state === 'ok' ? glyph.ok : state === 'warn' ? glyph.warn : glyph.fail;
+/**
+ * Services that answer on a health path but do not declare one.
+ *
+ * `fleet init` writes this into every manifest it generates:
+ *
+ *   # No health check: container state decides whether this is up.
+ *   # Add one once you know a path that returns 2xx —
+ *
+ * That is a research task handed to the reader, about a program the node is
+ * already running. The node does the research now -- it asks the container
+ * which of a handful of paths answers -- and this is where the answer is
+ * reported, because the sweep settles seconds after a deploy returns and there
+ * is nothing useful to say while it is still running.
+ *
+ * The negative result is deliberately not a warning. A service where nothing
+ * answered is a service whose manifest is already correct, and telling somebody
+ * their correct configuration is a problem is how a health report gets ignored.
+ *
+ * Separated from the command so both outcomes can be tested without a control
+ * plane -- and the one that matters is the suggestion, which a fleet whose
+ * services all declare health checks never produces.
+ */
+export function healthPathCheck(services) {
+    const swept = services.filter((s) => s.discoveredHealth);
+    const answering = swept
+        .map((s) => ({
+        name: s.name,
+        // The first 2xx-3xx, which is the order the node tried them in: a
+        // dedicated endpoint before "/", because a check that renders the whole
+        // application every ten seconds is the worse of two working answers.
+        path: s.discoveredHealth.find((c) => c.status >= 200 && c.status < 400)?.path,
+    }))
+        .filter((s) => Boolean(s.path));
+    if (!answering.length) {
+        return {
+            state: 'ok',
+            label: 'health paths',
+            detail: swept.length
+                ? `${swept.length} service(s) without a health check answered nothing — container state is the only evidence, as declared.`
+                : 'Every service declares a health check, or none has been swept yet.',
+        };
+    }
+    const named = answering.map((s) => `${s.name} → ${s.path}`).join(', ');
+    return {
+        state: 'warn',
+        label: 'health paths',
+        detail: `${named}. These answer 2xx but declare no health check, so a deploy is confirmed on container state alone.`,
+        remedy: `Add \`health: { path: ${answering[0].path} }\` to ${answering[0].name} in fleet.yaml. Without one, a container that starts and then fails every request still counts as a successful deploy.`,
+    };
+}
 /**
  * A build failure carries the whole buildx transcript. One summary line
  * belongs in a health report; `fleet deployments` is where the rest lives.
@@ -204,6 +254,7 @@ export const doctorCommand = {
                 detail: result.github.error ?? 'Not configured; public repositories can still deploy.',
                 remedy: 'Set GITHUB_APP_ID and GITHUB_APP_PRIVATE_KEY, restart the control plane, then connect repositories in Dashboard → Settings.',
             });
+        checks.push(healthPathCheck(result.services));
         checks.push(alertCheck(result.alerts));
         checks.push({ state: 'ok', label: 'control-plane version', detail: result.health.version ?? 'version not reported' });
         if (flags.json)

package/dist/commands/index.js

@@ -5,12 +5,13 @@ import { alertsCommand } from './alerts.js';
 import { configCommand, useCommand } from './config.js';
 import { doctorCommand } from './doctor.js';
 import { upCommand } from './up.js';
+import { tuneCommand } from './tune.js';
 import { openCommand } from './open.js';
 import { downCommand } from './down.js';
 import { unpairCommand, agentCommand } from './unpair.js';
 import { secretsCommand } from './secrets.js';
 import { backupCommand, backupsCommand, restoreCommand } from './backups.js';
-import { applyCommand, deployCommand, deploymentsCommand, initCommand, importCommand, explainCommand, logsCommand, removeServiceCommand, restartCommand, rollbackCommand, rescheduleCommand, servicesCommand, validateCommand, whereCommand, } from './services.js';
+import { applyCommand, deployCommand, deploymentsCommand, initCommand, diagnoseCommand, importCommand, explainCommand, logsCommand, removeServiceCommand, restartCommand, rollbackCommand, rescheduleCommand, servicesCommand, validateCommand, whereCommand, } from './services.js';
 export const commands = {
     up: upCommand,
     open: openCommand,
@@ -20,7 +21,9 @@ export const commands = {
     config: configCommand,
     use: useCommand,
     doctor: doctorCommand,
+    tune: tuneCommand,
     init: initCommand,
+    diagnose: diagnoseCommand,
     import: importCommand,
     explain: explainCommand,
     validate: validateCommand,

package/dist/commands/services.js

@@ -580,6 +580,52 @@ async function answerQuestions(questions, flags) {
     }
     return Object.keys(answers).length ? answers : null;
 }
+/**
+ * Ask the control plane why something is wrong.
+ *
+ * Distinct from `explain`, which reads a failure log you already have. This
+ * goes and finds the evidence: the deployment history, what the node says it
+ * is running, the container's output, whether the public address answers.
+ */
+export const diagnoseCommand = {
+    async run(args, flags) {
+        const fleetId = await requireFleet(typeof flags.fleet === 'string' ? flags.fleet : undefined);
+        const question = args.join(' ').trim();
+        if (!question) {
+            throw new CliError('usage: fleet diagnose "<what is wrong>"\n' +
+                '   eg: fleet diagnose "why is backend returning 502?"', EXIT.usage);
+        }
+        const { body } = await task('looking', async () => request('POST', `/fleets/${fleetId}/diagnose`, { body: { question } }), 
+        // What it looked at, so the wait is legible rather than a spinner.
+        { done: (r) => ('calls' in r.body ? `looked at ${r.body.calls.length} thing(s)` : 'done') });
+        if (flags.json)
+            return console.log(JSON.stringify(body, null, 2));
+        if (body.status === 'disabled') {
+            return console.log(`${glyph.warn} ${c.yellow('unavailable')}  ${body.reason}`);
+        }
+        // What it looked at, always — the reader can repeat any of it by hand, and
+        // a diagnosis you cannot retrace is a diagnosis you have to take on faith.
+        for (const call of body.calls) {
+            const detail = Object.values(call.args)[0];
+            console.log(c.dim(`  · ${call.tool}${detail ? ` ${String(detail)}` : ''}`));
+        }
+        if (body.status === 'inconclusive') {
+            console.log(`\n${glyph.warn} ${c.yellow('inconclusive')}  ${body.reason}`);
+            return;
+        }
+        console.log(`\n${body.summary}\n`);
+        for (const f of body.findings) {
+            console.log(`  ${c.bold(f.claim)}`);
+            console.log(c.dim(`    ${f.evidence}`));
+        }
+        if (body.next.length) {
+            console.log(`\n  ${c.dim('next')}`);
+            for (const n of body.next)
+                console.log(`  ${glyph.info ?? '·'} ${n}`);
+        }
+        console.log(c.dim(`\n  ${body.model}`));
+    },
+};
 export const initCommand = {
     async run(args, flags) {
         const { detect, manifestTemplate } = await import('../detect.js');

package/dist/commands/tune.js

@@ -0,0 +1,61 @@
+import { request, requireFleet } from '../api.js';
+import { c } from '../render.js';
+import { glyph, rule } from '../ui.js';
+import { asQuantity, tuneRam, MIN_OBSERVATION_HOURS } from '../tune.js';
+/**
+ * Reservations, checked against what the services actually used.
+ *
+ * It proposes and never applies. Every number here is the system inferring
+ * something about a machine, and the lesson of every inference this project has
+ * shipped is that one leaving its evidence needs a person between it and the
+ * manifest — a review once invented a node from a compose service name, and
+ * once replaced a `build:` with `image: nginx:alpine` and served the welcome
+ * page over somebody's site. Both were caught by a guardrail. A person reading
+ * a diff is the cheapest guardrail there is.
+ */
+export const tuneCommand = {
+    async run(_args, flags) {
+        const fleetId = await requireFleet(typeof flags.fleet === 'string' ? flags.fleet : undefined);
+        const { body } = await request('GET', `/fleets/${fleetId}/services`);
+        const advice = body.services.map((s) => tuneRam(s));
+        if (flags.json)
+            return console.log(JSON.stringify({ fleetId, advice }, null, 2));
+        console.log(`\n${rule('tune · reservations against measured use')}`);
+        const advised = advice.filter((a) => a.verdict === 'advise');
+        const tight = advice.filter((a) => a.verdict === 'tight');
+        const waiting = advice.filter((a) => a.verdict === 'too-soon' || a.verdict === 'no-data');
+        for (const a of advice) {
+            if (a.verdict === 'advise') {
+                console.log(`${glyph.warn} ${c.bold(a.name.padEnd(18))} reserves ${asQuantity(a.from)}, peaks at ${a.peak}MB` +
+                    ` → ${c.bold(asQuantity(a.to))}`);
+            }
+            else if (a.verdict === 'tight') {
+                console.log(`${glyph.warn} ${c.bold(a.name.padEnd(18))} peaks at ${a.peak}MB of ${asQuantity(a.requestRamMb)}` +
+                    ` — close to its limit, which is also where the kernel kills it`);
+            }
+            else if (a.verdict === 'fits') {
+                console.log(`${glyph.ok} ${c.bold(a.name.padEnd(18))} peaks at ${a.peak}MB — about right`);
+            }
+            else if (a.verdict === 'too-soon') {
+                console.log(`${glyph.info} ${c.dim(a.name.padEnd(18))} ${c.dim(`watched for ${a.hours}h; needs ${MIN_OBSERVATION_HOURS}h`)}`);
+            }
+            else {
+                console.log(`${glyph.info} ${c.dim(a.name.padEnd(18))} ${c.dim('not measured yet')}`);
+            }
+        }
+        if (advised.length) {
+            console.log(`\n  ${c.dim('edit fleet.yaml, then')} fleet up`);
+            for (const a of advised) {
+                if (a.verdict !== 'advise')
+                    continue;
+                console.log(`  ${c.dim(`${a.name}:`)} resources: { ram: ${asQuantity(a.to)} }`);
+            }
+        }
+        if (!advised.length && !tight.length) {
+            console.log(`\n  ${c.dim(waiting.length === advice.length
+                ? 'Nothing has been watched long enough to advise on yet.'
+                : 'Every measured reservation is about right.')}`);
+        }
+        console.log();
+    },
+};

package/dist/index.js

@@ -27,6 +27,7 @@ const GROUPS = [
             ['apply [file]', 'Apply a fleet.yaml to the fleet'],
             ['deploy <service>', 'Build, schedule, and roll out'],
             ['explain <service>', 'Read a failed deploy and say what to do about it'],
+            ['diagnose "<question>"', 'Investigate why something is wrong, and cite what it looked at'],
         ],
     ],
     [
@@ -37,6 +38,7 @@ const GROUPS = [
             ['services', 'List services and where they are running'],
             ['nodes', 'List nodes'],
             ['where <service>', 'Explain where a service would be placed, and why'],
+            ['tune', 'Compare each reservation with the memory the service actually used'],
             ['deployments <service>', 'Deployment history'],
             ['logs <service> --follow', 'Follow the latest agent-reported container tail'],
             ['events', 'Unified event timeline'],

package/dist/tune.js

@@ -0,0 +1,61 @@
+/**
+ * What a service should reserve, from what it has been seen using.
+ *
+ * `fleet init` writes `resources: { ram: 512Mi }` because 512Mi is a round
+ * number and nothing in a repository says otherwise. Measured on the fleet this
+ * was written for: a steady 60MB and 20MB against exactly that reservation. The
+ * scheduler plans capacity around the number in the manifest for the life of
+ * the service, so the gap is invisible on one node and is the difference
+ * between a service fitting and `no_eligible_node` on a fleet where it matters.
+ *
+ * The judgement lives here rather than in the command so both outcomes can be
+ * tested without a control plane — and the outcome that matters is the refusal,
+ * which a fleet with weeks of history never produces.
+ */
+/**
+ * How long a service must have been watched before its peak means anything.
+ *
+ * A service observed for four minutes has not been observed; it has been
+ * glanced at. Nothing that runs a nightly job, or serves a morning, has shown
+ * its peak yet, and advising a reservation from that is how a tuned fleet
+ * starts OOM-killing at 3am.
+ */
+export const MIN_OBSERVATION_HOURS = 24;
+/**
+ * Headroom over the observed peak.
+ *
+ * Double, which is generous, and deliberately so. The cost of too much headroom
+ * is capacity the scheduler reserves and does not use; the cost of too little
+ * is the kernel killing a container in production. Those are not symmetric, and
+ * a tool that trims to the bone the first time it is run does not get run twice.
+ */
+const HEADROOM = 2;
+/** Reservations are read by people. 118 is a measurement; 128 is a decision. */
+function round(mb) {
+    const steps = [64, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, 3072, 4096];
+    return steps.find((s) => s >= mb) ?? Math.ceil(mb / 1024) * 1024;
+}
+export function tuneRam(svc, now = Date.now()) {
+    if (svc.observedRamPeakMb === null || !svc.observedRamSince) {
+        return { verdict: 'no-data', name: svc.name };
+    }
+    const hours = (now - new Date(svc.observedRamSince).getTime()) / 3_600_000;
+    if (hours < MIN_OBSERVATION_HOURS) {
+        return { verdict: 'too-soon', name: svc.name, hours: Math.max(0, Math.round(hours * 10) / 10) };
+    }
+    const peak = svc.observedRamPeakMb;
+    // Near its limit. Not a saving, and worth saying out loud: the reservation is
+    // also the container's hard limit, so a service peaking at four fifths of it
+    // is one traffic spike from being killed.
+    if (peak >= svc.requestRamMb * 0.8) {
+        return { verdict: 'tight', name: svc.name, peak, requestRamMb: svc.requestRamMb };
+    }
+    const want = round(peak * HEADROOM);
+    if (want >= svc.requestRamMb)
+        return { verdict: 'fits', name: svc.name, peak };
+    return { verdict: 'advise', name: svc.name, from: svc.requestRamMb, to: want, peak };
+}
+/** Megabytes as a manifest writes them. */
+export function asQuantity(mb) {
+    return mb % 1024 === 0 ? `${mb / 1024}Gi` : `${mb}Mi`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yadurajfleetos/cli",
-  "version": "0.11.0",
+  "version": "0.13.0",
   "description": "Fleet OS command-line interface for deploying and orchestrating services on user-owned hardware",
   "type": "module",
   "license": "MIT",