@celilo/cli 2.0.0 → 2.2.0

package/src/services/control-plane-health.test.ts

@@ -0,0 +1,66 @@
+/**
+ * The control plane's health check (celilo#1225).
+ *
+ * The mapping is what these pin. The failure that made this code exist was not
+ * a wrong assertion — the hook's `existsSync(db_path)` was a perfectly correct
+ * question — it was a correct question asked from inside a jail that does not
+ * bind the directory, so a present database read as absent. Nothing in a unit
+ * test can catch that shape; what a unit test CAN hold is that every finding
+ * reaches the operator with its status and its remediation intact, because the
+ * unattended path is the one nobody is watching.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { findingAsHealthCheck } from './control-plane-health';
+import type { FleetFinding } from './fleet-checks';
+
+function finding(over: Partial<FleetFinding> = {}): FleetFinding {
+  return {
+    id: 'dispatcher',
+    title: 'Event dispatcher',
+    status: 'ok',
+    summary: 'a dispatcher is live',
+    detail: [],
+    remediation: null,
+    autoFixable: false,
+    ...over,
+  };
+}
+
+describe('findingAsHealthCheck', () => {
+  test("ok becomes pass — celilo's vocabulary is not the health path's", () => {
+    expect(findingAsHealthCheck(finding({ status: 'ok' })).status).toBe('pass');
+    expect(findingAsHealthCheck(finding({ status: 'warn' })).status).toBe('warn');
+    expect(findingAsHealthCheck(finding({ status: 'fail' })).status).toBe('fail');
+  });
+
+  test('the check is named by the finding id, so an operator can grep for it', () => {
+    const item = findingAsHealthCheck(finding({ id: 'schema_drift' }));
+    expect(item.name).toBe('schema_drift');
+    expect(item.message).toBe('a dispatcher is live');
+  });
+
+  test('the remediation survives the mapping', () => {
+    // This is the line that tells an operator what to DO, and the health path
+    // is the one that runs unattended every fifteen minutes. Dropping it here
+    // would leave a failing check with no next step, quietly.
+    const item = findingAsHealthCheck(
+      finding({
+        status: 'fail',
+        detail: ['no live dispatcher — heartbeat absent'],
+        remediation: 'systemctl enable --now celilo-events.service',
+      }),
+    );
+    expect(item.details).toContain('no live dispatcher — heartbeat absent');
+    expect(item.details).toContain('Fix: systemctl enable --now celilo-events.service');
+  });
+
+  test('a healthy finding carries no empty details field', () => {
+    expect(findingAsHealthCheck(finding()).details).toBeUndefined();
+  });
+
+  test('detail lines survive without a remediation', () => {
+    const item = findingAsHealthCheck(finding({ status: 'warn', detail: ['two dispatchers'] }));
+    expect(item.details).toBe('two dispatchers');
+  });
+});

package/src/services/control-plane-health.ts

@@ -0,0 +1,67 @@
+/**
+ * The control plane's health check, run by celilo rather than by a hook.
+ *
+ * ## Why this is not a hook
+ *
+ * It was `modules/celilo-mgmt/scripts/health-check.ts` until celilo#1225, and
+ * it failed under the hook jail in the way that is worst: **it reported a
+ * healthy fleet's database missing.** The hook asked `existsSync(db_path)`,
+ * the jail does not bind celilo's data directory (deliberately — that is where
+ * `master.key` and `celilo.db` live), so the answer was `false` and the deploy
+ * failed on a file that was sitting right there. Its other two checks shelled
+ * out to the CLI, which a jailed hook cannot reach at all.
+ *
+ * Every one of those was celilo inspecting itself. Deleting `on_install` for
+ * that reason and leaving this behind would have fixed one hook and left the
+ * deploy failing on the next.
+ *
+ * ## What replaced it is stronger, not merely relocated
+ *
+ * The hook asked two questions: does a file exist, and is this hostname in
+ * `celilo machine list`. `runFleetChecks` is what celilo already runs for
+ * `celilo system doctor`, and it asks eight, including the four-part dispatcher
+ * probe whose whole point is that "a process is up" reports green on a
+ * dispatcher that is unsupervised, on stale code, and delivering nothing.
+ *
+ * So the module gets a better check by having none of its own. That is the
+ * outcome to keep in mind before anyone adds a hook back: a health_check hook
+ * here can only re-ask, worse, questions celilo can answer directly.
+ */
+
+import type { DbClient } from '../db/client';
+import { withCeliloBusAsync } from './control-plane-bootstrap';
+import { type FleetFinding, runFleetChecks } from './fleet-checks';
+import type { HealthCheckItem } from './health-runner';
+
+/** `FleetFinding`'s three statuses in `HealthCheckItem`'s vocabulary. */
+const STATUS: Record<FleetFinding['status'], HealthCheckItem['status']> = {
+  ok: 'pass',
+  warn: 'warn',
+  fail: 'fail',
+};
+
+/**
+ * One finding as a health item.
+ *
+ * `detail` and `remediation` are folded into `details` rather than dropped:
+ * the health path is what runs unattended every fifteen minutes, so the line
+ * telling an operator what to DO has to survive the mapping.
+ */
+export function findingAsHealthCheck(finding: FleetFinding): HealthCheckItem {
+  const details = [
+    ...finding.detail,
+    ...(finding.remediation ? [`Fix: ${finding.remediation}`] : []),
+  ];
+  return {
+    name: finding.id,
+    status: STATUS[finding.status],
+    message: finding.summary,
+    ...(details.length > 0 ? { details: details.join('\n') } : {}),
+  };
+}
+
+/** Run celilo's own fleet checks and report them as the control plane's health. */
+export async function controlPlaneHealthChecks(db: DbClient): Promise<HealthCheckItem[]> {
+  const findings = await withCeliloBusAsync((bus) => runFleetChecks(bus, db));
+  return findings.map(findingAsHealthCheck);
+}

package/src/services/deploy-preflight.ts

@@ -174,8 +174,14 @@ export async function runPreflight(
     for (const variable of manifest.variables.owns) {
       if (!variable.required) continue;
 
-      // Infrastructure/terraform variables are auto-derived during deploy
-      if (variable.source === 'infrastructure' || variable.source === 'terraform') {
+      // Infrastructure/terraform variables are auto-derived during deploy;
+      // hook-owned variables are discovered by the module's own hooks at
+      // runtime (hook-owned-state D2). None of the three is operator input.
+      if (
+        variable.source === 'infrastructure' ||
+        variable.source === 'terraform' ||
+        variable.source === 'hook'
+      ) {
         continue;
       }
 

package/src/services/deploy-validation.test.ts

@@ -129,6 +129,28 @@ describe('findMissingRequiredVariables (deploy path)', () => {
     expect(missing[0].value_label).toBeUndefined();
   });
 
+  // Hook-owned-state task 2.3 (design D2): a `source: hook` variable is
+  // discovered by the module's own hook at runtime. It is never operator
+  // input, so an unwritten one must not read as missing configuration — not
+  // for the interview, and not as a deploy-blocking error.
+  test('a source: hook variable is never reported missing, even required and unwritten', async () => {
+    const manifest = {
+      celilo_contract: '1.0',
+      variables: {
+        owns: [
+          {
+            name: 'public_ip',
+            type: 'string',
+            required: true,
+            source: 'hook',
+          },
+        ],
+      },
+    } as unknown as ModuleManifest;
+    const missing = await findMissingRequiredVariables('testmod', manifest, db);
+    expect(missing).toEqual([]);
+  });
+
   test('configured secrets are not reported as missing', async () => {
     db.insert(secrets)
       .values({

package/src/services/deploy-validation.ts

@@ -430,6 +430,14 @@ export async function findMissingRequiredVariables(
         continue; // These are auto-populated from Terraform outputs
       }
 
+      // Hook-owned variables are discovered by the module's own hooks at
+      // runtime (hook-owned-state D2). An unwritten one is not a missing
+      // configuration value — prompting for it or failing validation would
+      // ask the operator to guess what a hook has not learned yet.
+      if (variable.source === 'hook') {
+        continue;
+      }
+
       // Check if variable is configured
       let isConfigured = false;
 

package/src/services/deployed-systems.ts

@@ -48,6 +48,18 @@ function rowToSystem(row: typeof moduleSystems.$inferSelect): DeployedSystem {
  * Returns [] for API-only modules (e.g. namecheap) — that is a modeled state,
  * not an error.
  */
+/**
+ * The module that IS celilo's control plane.
+ *
+ * celilo knows this module by name in several places — its privileged
+ * capability allow-list, the subnet it trusts, the fleet checks that ask where
+ * the control plane runs, and the deploy step that initialises the box's own
+ * state. The id lives here because every one of those already reaches for
+ * `getModuleSystems`, and three separate private copies of the string is how
+ * they drift.
+ */
+export const CONTROL_PLANE_MODULE_ID = 'celilo-mgmt';
+
 export function getModuleSystems(moduleId: string, db: DbClient): DeployedSystem[] {
   const rows = db.select().from(moduleSystems).where(eq(moduleSystems.moduleId, moduleId)).all();
   return rows.map(rowToSystem).sort((a, b) => a.name.localeCompare(b.name));

package/src/services/dns-discovery.test.ts

@@ -0,0 +1,147 @@
+/**
+ * Moved from `modules/celilo-mgmt/scripts/discovery.test.ts` with celilo#1225.
+ *
+ * The parser cases came across unchanged. The `discoverDns` cases are new and
+ * could not have been written there: the module's version read the filesystem
+ * directly, so the file-preference rule — the part with the actual failure mode
+ * — had no seam to test through.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { type ResolverFileReader, chooseDns, discoverDns, parseNameservers } from './dns-discovery';
+
+const SYSTEMD = '/run/systemd/resolve/resolv.conf';
+const ETC = '/etc/resolv.conf';
+
+/** A host offering exactly the files given, and nothing else. */
+function host(files: Record<string, string>): ResolverFileReader {
+  return (path) => files[path] ?? null;
+}
+
+describe('parseNameservers', () => {
+  test('extracts nameserver IPs in order', () => {
+    const resolv = ['# comment', 'nameserver 9.9.9.9', 'nameserver 8.8.4.4', 'search lan'].join(
+      '\n',
+    );
+    expect(parseNameservers(resolv)).toEqual(['9.9.9.9', '8.8.4.4']);
+  });
+
+  test('returns empty for no nameservers', () => {
+    expect(parseNameservers('search lan\noptions edns0')).toEqual([]);
+  });
+});
+
+describe('chooseDns', () => {
+  test('drops loopback stubs (systemd-resolved 127.0.0.53)', () => {
+    expect(chooseDns(['127.0.0.53'])).toEqual({ primary: '1.1.1.1', fallback: '1.1.1.1' });
+  });
+
+  test('uses the first two real upstreams', () => {
+    expect(chooseDns(['9.9.9.9', '8.8.4.4', '1.0.0.1'])).toEqual({
+      primary: '9.9.9.9',
+      fallback: '8.8.4.4',
+    });
+  });
+
+  test('falls back to 1.1.1.1 for the second when only one upstream', () => {
+    expect(chooseDns(['9.9.9.9'])).toEqual({ primary: '9.9.9.9', fallback: '1.1.1.1' });
+  });
+});
+
+describe('discoverDns', () => {
+  test('prefers the systemd upstream file over the stub in /etc/resolv.conf', () => {
+    // The case the file order exists for. Reading only /etc/resolv.conf here
+    // discovers 127.0.0.53, which no other box on the network can reach.
+    const servers = discoverDns(
+      host({
+        [SYSTEMD]: 'nameserver 9.9.9.9\nnameserver 8.8.4.4\n',
+        [ETC]: 'nameserver 127.0.0.53\n',
+      }),
+    );
+    expect(servers).toEqual({ primary: '9.9.9.9', fallback: '8.8.4.4' });
+  });
+
+  test('falls through to /etc/resolv.conf when the systemd file yields nothing real', () => {
+    const servers = discoverDns(
+      host({
+        [SYSTEMD]: 'nameserver 127.0.0.53\n',
+        [ETC]: 'nameserver 192.0.2.10\nnameserver 192.0.2.11\n',
+      }),
+    );
+    expect(servers).toEqual({ primary: '192.0.2.10', fallback: '192.0.2.11' });
+  });
+
+  test('reads /etc/resolv.conf on a host with no systemd resolver file', () => {
+    expect(discoverDns(host({ [ETC]: 'nameserver 192.0.2.10\n' }))).toEqual({
+      primary: '192.0.2.10',
+      fallback: '1.1.1.1',
+    });
+  });
+
+  test('accepts a loopback-only /etc/resolv.conf as the public fallback, not as an error', () => {
+    // Last resort: whatever it gives is taken, and a stub-only host yields the
+    // public fallback rather than an unreachable address.
+    expect(discoverDns(host({ [ETC]: 'nameserver 127.0.0.53\n' }))).toEqual({
+      primary: '1.1.1.1',
+      fallback: '1.1.1.1',
+    });
+  });
+
+  test('returns the public fallback on a host with no resolver files at all', () => {
+    expect(discoverDns(host({}))).toEqual({ primary: '1.1.1.1', fallback: '1.1.1.1' });
+  });
+
+  // The recurrence gate for celilo#1239 (bead ce-dxr8). The dns-client-config
+  // aspect of a deployed dns_internal provider rewrites /etc/resolv.conf to
+  // name that provider. Discovery reading the file back adopts the fleet's own
+  // resolver as the fleet's upstream and writes it into dns.primary /
+  // dns.fallback, where it reaches every LXC's permanent birth nameserver
+  // line. Proven to fail against the accepting implementation first.
+  describe('refusing resolvers celilo itself installed', () => {
+    // technitium's advertised dns_internal addresses: its dmz zone address and
+    // (once advertised) the firewall ingress `internal`-zone systems use.
+    const FLEET = ['10.0.10.13', '192.168.0.151'];
+
+    test('refuses a resolv.conf naming a deployed dns_internal provider address', () => {
+      const servers = discoverDns(host({ [ETC]: `nameserver ${FLEET[0]}\n` }), {
+        fleetResolverIps: FLEET,
+      });
+      expect(servers).toEqual({ primary: '1.1.1.1', fallback: '1.1.1.1' });
+    });
+
+    test('keeps the real upstreams beside a refused fleet resolver', () => {
+      const servers = discoverDns(host({ [ETC]: `nameserver ${FLEET[0]}\nnameserver 9.9.9.9\n` }), {
+        fleetResolverIps: FLEET,
+      });
+      expect(servers).toEqual({ primary: '9.9.9.9', fallback: '1.1.1.1' });
+    });
+
+    test('falls through to /etc/resolv.conf when the systemd file holds only fleet resolvers', () => {
+      const servers = discoverDns(
+        host({
+          [SYSTEMD]: `nameserver ${FLEET[0]}\n`,
+          [ETC]: 'nameserver 9.9.9.9\n',
+        }),
+        { fleetResolverIps: FLEET },
+      );
+      expect(servers).toEqual({ primary: '9.9.9.9', fallback: '1.1.1.1' });
+    });
+
+    test('matches a capability address that carries a CIDR suffix', () => {
+      // Capability data advertises `192.168.0.151/24` (server.ip resolves from
+      // target_ip); the resolv.conf entry is the bare address.
+      const servers = discoverDns(host({ [ETC]: `nameserver ${FLEET[1]}\n` }), {
+        fleetResolverIps: [`${FLEET[1]}/24`],
+      });
+      expect(servers).toEqual({ primary: '1.1.1.1', fallback: '1.1.1.1' });
+    });
+
+    test('accepts a private upstream that is not a fleet resolver', () => {
+      // A home-lab upstream on the operator's own network is legitimate. The
+      // refusal is scoped to resolvers celilo deployed, not to privateness.
+      expect(
+        discoverDns(host({ [ETC]: 'nameserver 192.168.0.1\n' }), { fleetResolverIps: FLEET }),
+      ).toEqual({ primary: '192.168.0.1', fallback: '1.1.1.1' });
+    });
+  });
+});

package/src/services/dns-discovery.ts

@@ -0,0 +1,134 @@
+/**
+ * Read the upstream DNS resolvers this box uses.
+ *
+ * This sits beside `network-discovery.ts` and for the same reason: celilo
+ * records facts about the management box itself, and a module is not the right
+ * place to read them. The network half moved first
+ * (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md),
+ * the fleet key followed (openspec/changes/hook-process-boundary, design D9b),
+ * and DNS is the last of the three. It came from
+ * `modules/celilo-mgmt/scripts/discovery.ts`, whose hook could only reach
+ * celilo by spawning the CLI — which a jailed hook cannot do at all
+ * (celilo#1225).
+ *
+ * Reading it here is not merely tidier, it is the only place it can be
+ * correct. celilo-mgmt is never deployed to a remote box (ruled 2026-09-02),
+ * so the host whose resolvers we want is always the host celilo runs on. A
+ * hook reading `/etc/resolv.conf` reads that same file, one process further
+ * out, for no benefit.
+ *
+ * Pure parsers are separated from the IO wrapper so they test without a host
+ * (Rule 2.3).
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+
+/**
+ * Where a host publishes its real upstream resolvers, most trustworthy first.
+ *
+ * systemd-resolved's file comes first deliberately: on such a host
+ * `/etc/resolv.conf` names the local 127.0.0.53 stub rather than the upstreams
+ * behind it, so reading only the second file discovers a loopback address and
+ * writes it into system config as the fleet's DNS.
+ */
+const RESOLVER_FILES = ['/run/systemd/resolve/resolv.conf', '/etc/resolv.conf'] as const;
+
+/** Where we land when the host offers no usable upstream of its own. */
+const PUBLIC_FALLBACK = '1.1.1.1';
+
+export interface DnsServers {
+  primary: string;
+  fallback: string;
+}
+
+/** Extract `nameserver` IPs from resolv.conf-format text, in file order. */
+export function parseNameservers(resolvConf: string): string[] {
+  return resolvConf
+    .split('\n')
+    .map((l) => l.trim())
+    .filter((l) => l.startsWith('nameserver '))
+    .map((l) => l.split(/\s+/)[1])
+    .filter(Boolean);
+}
+
+/**
+ * Choose primary and fallback from discovered nameservers.
+ *
+ * Loopback entries are dropped rather than used. A stub resolver's address is
+ * a real answer to "what does this host query" and the wrong answer to "what
+ * should the fleet query", because nothing else on the network can reach it.
+ */
+export function chooseDns(nameservers: string[]): DnsServers {
+  const upstream = nameservers.filter((ns) => !ns.startsWith('127.') && ns !== '::1');
+  return {
+    primary: upstream[0] ?? PUBLIC_FALLBACK,
+    fallback: upstream[1] ?? PUBLIC_FALLBACK,
+  };
+}
+
+/** Reads a resolver file, or returns null when it is not present. */
+export type ResolverFileReader = (path: string) => string | null;
+
+const readResolverFile: ResolverFileReader = (path) =>
+  existsSync(path) ? readFileSync(path, 'utf-8') : null;
+
+/** Options for {@link discoverDns}. */
+export interface DiscoverDnsOptions {
+  /**
+   * Addresses of resolvers celilo itself deployed, as advertised in the
+   * capability data of providers that declare the `fleet_resolver` marker
+   * (`server.ip`, `server.internal_ip`). Entries may carry a CIDR suffix.
+   */
+  fleetResolverIps?: readonly string[];
+}
+
+/** Strip a CIDR suffix: `192.168.0.151/24` → `192.168.0.151`. */
+export function bareIp(address: string): string {
+  return address.split('/')[0];
+}
+
+/**
+ * Drop nameservers that are resolvers celilo itself installed.
+ *
+ * A deployed dns_internal provider's base-module aspect rewrites this box's
+ * `/etc/resolv.conf` to name that provider. Reading the file back afterwards
+ * adopts the fleet's own resolver as the fleet's upstream and writes it into
+ * `dns.primary` / `dns.fallback` (celilo#1239), where it reaches every LXC's
+ * permanent birth nameserver line. A resolver answering the fleet is not an
+ * upstream for the fleet; refusing it here is what keeps discovery honest
+ * once the aspect has run.
+ */
+export function excludeFleetResolvers(
+  nameservers: string[],
+  fleetResolverIps: readonly string[] = [],
+): string[] {
+  if (fleetResolverIps.length === 0) return nameservers;
+  const fleet = new Set(fleetResolverIps.map(bareIp));
+  return nameservers.filter((ns) => !fleet.has(ns));
+}
+
+/**
+ * Read this box's upstream resolvers.
+ *
+ * Loopback stubs and resolvers celilo itself installed ({@link DiscoverDnsOptions.fleetResolverIps})
+ * are refused, so a file holding only refused entries falls through to the
+ * next one. `/etc/resolv.conf` is the last resort and whatever survives the
+ * refusal is accepted, including the public fallback, because there is
+ * nothing further to consult.
+ */
+export function discoverDns(
+  read: ResolverFileReader = readResolverFile,
+  options: DiscoverDnsOptions = {},
+): DnsServers {
+  for (const path of RESOLVER_FILES) {
+    const contents = read(path);
+    if (contents === null) continue;
+    const servers = chooseDns(
+      excludeFleetResolvers(parseNameservers(contents), options.fleetResolverIps),
+    );
+    if (servers.primary !== PUBLIC_FALLBACK || path === '/etc/resolv.conf') {
+      return servers;
+    }
+  }
+  return { primary: PUBLIC_FALLBACK, fallback: PUBLIC_FALLBACK };
+}

package/src/services/fleet-checks.ts

@@ -29,7 +29,11 @@ import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capabilit
 import type { ModuleManifest } from '../manifest/schema';
 
 import { getServiceCredentials, listContainerServices } from './container-service';
-import { getModuleSystems, listAllModuleSystems } from './deployed-systems';
+import {
+  CONTROL_PLANE_MODULE_ID,
+  getModuleSystems,
+  listAllModuleSystems,
+} from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
 import {
   SUPERVISOR_SCOPES,
@@ -43,7 +47,7 @@ import { describePausedModule, listPausedModules } from './module-pause';
 import { resolveSubscription } from './module-subscriptions';
 
 /** The module that IS celilo's control plane. */
-const CONTROL_PLANE_MODULE = 'celilo-mgmt';
+const CONTROL_PLANE_MODULE = CONTROL_PLANE_MODULE_ID;
 
 /**
  * Zones reachable from the operator's LAN. A celilo placement zone other

package/src/services/fleet-key.test.ts

@@ -5,10 +5,11 @@
  */
 
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
-import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
-import { ensureFleetKey, getFleetSshDir } from './fleet-key';
+import { ensureFleetKey, findFleetPrivateKey, getFleetSshDir } from './fleet-key';
 
 describe('ensureFleetKey', () => {
   let dataDir: string;
@@ -45,3 +46,66 @@ describe('ensureFleetKey', () => {
     expect(second.publicKey).toBe(first.publicKey);
   });
 });
+
+describe('findFleetPrivateKey', () => {
+  let dataDir: string;
+  let homeDir: string;
+  let realHome: string | undefined;
+
+  beforeEach(() => {
+    dataDir = mkdtempSync(join(tmpdir(), 'celilo-fleet-find-'));
+    homeDir = mkdtempSync(join(tmpdir(), 'celilo-fleet-home-'));
+    process.env.CELILO_DB_PATH = join(dataDir, 'celilo.db');
+    realHome = process.env.HOME;
+    process.env.HOME = homeDir;
+  });
+
+  afterEach(() => {
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.HOME = realHome;
+    rmSync(dataDir, { recursive: true, force: true });
+    rmSync(homeDir, { recursive: true, force: true });
+  });
+
+  /** A second, unrelated keypair in $HOME/.ssh, as every real box has. */
+  function plantHomeKey(): string {
+    const dir = join(homeDir, '.ssh');
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const path = join(dir, 'id_ed25519');
+    execFileSync('ssh-keygen', ['-t', 'ed25519', '-N', '', '-f', path, '-C', 'someone-else'], {
+      stdio: 'pipe',
+    });
+    return readFileSync(`${path}.pub`, 'utf-8').trim();
+  }
+
+  it('finds the key celilo minted, even when $HOME holds a different one', () => {
+    // This is celilo#1240. `ensureFleetKey` mints into the data dir and
+    // `machine add` searched $HOME only, so a fleet whose key celilo minted
+    // could not add a machine without --ssh-key-file. The e2e management
+    // container has both directories populated with DIFFERENT keys, which is
+    // why it failed there and not on a developer's box.
+    const homePublicKey = plantHomeKey();
+    const fleet = ensureFleetKey();
+    expect(fleet.publicKey).not.toBe(homePublicKey);
+
+    const found = findFleetPrivateKey(fleet.publicKey);
+
+    expect(found).toBe(join(dataDir, '.ssh', 'id_ed25519'));
+  });
+
+  it('still finds a key that only lives in $HOME', () => {
+    // The auto-detected case: `initializeSystem` reads $HOME/.ssh when
+    // ssh.public_key is unset, and that fleet must keep working.
+    const homePublicKey = plantHomeKey();
+
+    expect(findFleetPrivateKey(homePublicKey)).toBe(join(homeDir, '.ssh', 'id_ed25519'));
+  });
+
+  it('returns null when no private key matches the recorded public half', () => {
+    plantHomeKey();
+    const orphan =
+      'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA nobody';
+
+    expect(findFleetPrivateKey(orphan)).toBeNull();
+  });
+});

package/src/services/fleet-key.ts

@@ -73,3 +73,57 @@ export function ensureFleetKey(): FleetKey {
 
   return { publicKey: readFileSync(publicKeyPath, 'utf-8').trim(), created: true };
 }
+
+/**
+ * Every directory a fleet private key may sit in, in priority order.
+ *
+ * Exported so an error message names exactly what was searched. The message
+ * used to say "~/.ssh/" while the search had already been widened, which is a
+ * smaller version of the bug this whole helper exists to prevent.
+ */
+export function fleetKeySearchDirs(): string[] {
+  return [getFleetSshDir(), join(process.env.HOME ?? '~', '.ssh')];
+}
+
+/**
+ * The private key whose public half is `publicKey`, or null.
+ *
+ * Searches celilo's own key directory FIRST and `$HOME/.ssh` second, and the
+ * order is the point. Two different keys are normally present: the one
+ * `ensureFleetKey` minted, which is what celilo authorizes on managed
+ * machines, and whatever keypair the operator or the base image left in the
+ * home directory. Only the first can open a machine celilo provisioned.
+ *
+ * This lives beside `getFleetSshDir` for the reason that helper exists. The
+ * caller that searched for the key hand-derived its own `join(HOME, '.ssh')`
+ * and therefore could not see a minted key at all (celilo#1240).
+ *
+ * Matching ignores the trailing comment, because ssh-keygen writes
+ * `user@host` there and the recorded public half carries `celilo-fleet`.
+ */
+export function findFleetPrivateKey(publicKey: string): string | null {
+  const [type, material] = publicKey.trim().split(/\s+/);
+  if (!type || !material) return null;
+
+  const byType: Record<string, string[]> = {
+    'ssh-ed25519': ['id_ed25519'],
+    'ssh-rsa': ['id_rsa'],
+    'ecdsa-sha2-nistp256': ['id_ecdsa'],
+    'ecdsa-sha2-nistp384': ['id_ecdsa'],
+    'ecdsa-sha2-nistp521': ['id_ecdsa'],
+  };
+  const candidates = byType[type] ?? ['id_ed25519', 'id_rsa', 'id_ecdsa'];
+
+  for (const dir of fleetKeySearchDirs()) {
+    if (!existsSync(dir)) continue;
+    for (const name of candidates) {
+      const keyPath = join(dir, name);
+      const pubPath = `${keyPath}.pub`;
+      if (!existsSync(keyPath) || !existsSync(pubPath)) continue;
+      const [fileType, fileMaterial] = readFileSync(pubPath, 'utf-8').trim().split(/\s+/);
+      if (fileType === type && fileMaterial === material) return keyPath;
+    }
+  }
+
+  return null;
+}