@celilo/cli 0.8.2 → 0.9.1

package/src/services/deployed-systems.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
 import { existsSync } from 'node:fs';
 import { rm } from 'node:fs/promises';
+import { and, eq } from 'drizzle-orm';
 import { type DbClient, createDbClient } from '../db/client';
 import {
   containerServices,
@@ -10,7 +11,7 @@ import {
   moduleSystems,
   modules,
 } from '../db/schema';
-import { backfillModuleSystems, getModuleSystems } from './deployed-systems';
+import { backfillModuleSystems, getModuleSystems, upsertDeployedSystem } from './deployed-systems';
 
 const TEST_DB_PATH = './test-deployed-systems.db';
 
@@ -233,3 +234,74 @@ describe('backfillModuleSystems', () => {
     expect(getModuleSystems('namecheap', db)).toHaveLength(0);
   });
 });
+
+/**
+ * Canonical instance sizing (ISS-0150): sizing is seeded onto module_systems
+ * once at first provision and then OWNED by `celilo proxmox … resize` — a routine
+ * re-deploy must never reset a resized instance back to its manifest minimum.
+ */
+describe('upsertDeployedSystem sizing — seed-once (ISS-0150)', () => {
+  let db: DbClient;
+
+  beforeEach(() => {
+    db = createDbClient({ path: TEST_DB_PATH });
+    db.insert(modules)
+      .values({
+        id: 'm1',
+        name: 'm1',
+        version: '1.0.0',
+        manifestData: {},
+        sourcePath: '/tmp/m1',
+        state: 'VERIFIED',
+      })
+      .run();
+  });
+
+  afterEach(async () => {
+    db.$client.close();
+    for (const suffix of ['', '-shm', '-wal']) {
+      const p = `${TEST_DB_PATH}${suffix}`;
+      if (existsSync(p)) await rm(p);
+    }
+  });
+
+  const sizeArgs = (memory: number) => ({
+    name: 'main',
+    hostname: 'h',
+    ipv4Address: '10.0.0.5/24',
+    zone: 'app' as const,
+    infraType: 'container_service' as const,
+    vmid: 200,
+    cpu: 4,
+    memory,
+    disk: 80,
+  });
+
+  const row = () =>
+    db
+      .select()
+      .from(moduleSystems)
+      .where(and(eq(moduleSystems.moduleId, 'm1'), eq(moduleSystems.name, 'main')))
+      .get();
+
+  test('seeds sizing on first insert', () => {
+    upsertDeployedSystem(db, 'm1', sizeArgs(8192));
+    const r = row();
+    expect(r?.cpu).toBe(4);
+    expect(r?.memory).toBe(8192);
+    expect(r?.disk).toBe(80);
+  });
+
+  test('a re-deploy does NOT reset a resized instance to the manifest minimum', () => {
+    upsertDeployedSystem(db, 'm1', sizeArgs(8192)); // first provision: seed 8 GB
+    // Simulate `celilo proxmox vm resize` bumping the canonical size to 16 GB.
+    db.update(moduleSystems)
+      .set({ memory: 16384 })
+      .where(and(eq(moduleSystems.moduleId, 'm1'), eq(moduleSystems.name, 'main')))
+      .run();
+    // Re-deploy passes the manifest minimum (8 GB) again — must be ignored.
+    upsertDeployedSystem(db, 'm1', sizeArgs(8192));
+    expect(row()?.memory).toBe(16384);
+    expect(row()?.cpu).toBe(4);
+  });
+});

package/src/services/deployed-systems.ts

@@ -52,6 +52,52 @@ export function getModuleSystems(moduleId: string, db: DbClient): DeployedSystem
   return rows.map(rowToSystem).sort((a, b) => a.name.localeCompare(b.name));
 }
 
+/**
+ * A celilo-provisioned instance with its canonical size (ISS-0150). CLI-internal
+ * shape (NOT the `DeployedSystem` capability type) for the `celilo proxmox
+ * vm/ct …` surface, which needs the sizing columns the capability type omits.
+ */
+export interface ProvisionedSystem {
+  moduleId: string;
+  name: string;
+  hostname: string;
+  ipv4Address: string;
+  zone: NetworkZone;
+  serviceId: string | null;
+  vmid: number | null;
+  /** Canonical desired size (null until seeded / for non-Proxmox). */
+  cpu: number | null;
+  memory: number | null;
+  disk: number | null;
+}
+
+/**
+ * Every celilo-provisioned (container_service) system with a Proxmox vmid, across
+ * all modules, including its canonical sizing — the read model behind
+ * `celilo proxmox vm/ct list|show`. Ordered by vmid for stable output.
+ */
+export function getProvisionedSystems(db: DbClient): ProvisionedSystem[] {
+  return db
+    .select()
+    .from(moduleSystems)
+    .where(eq(moduleSystems.infraType, 'container_service'))
+    .all()
+    .filter((r) => r.vmid != null)
+    .map((r) => ({
+      moduleId: r.moduleId,
+      name: r.name,
+      hostname: r.hostname,
+      ipv4Address: r.ipv4Address,
+      zone: r.zone,
+      serviceId: r.serviceId,
+      vmid: r.vmid,
+      cpu: r.cpu,
+      memory: r.memory,
+      disk: r.disk,
+    }))
+    .sort((a, b) => (a.vmid ?? 0) - (b.vmid ?? 0));
+}
+
 /**
  * All container_service systems (Proxmox LXCs, droplets, …) whose zone is in
  * `zones`, across every module — the LXC complement to machine-pool's
@@ -93,6 +139,15 @@ export interface DeployedSystemInput {
   machineId?: string | null;
   serviceId?: string | null;
   vmid?: number | null;
+  /**
+   * Canonical deployed size (ISS-0150), seeded from the module's
+   * `requires.system` at first provision. Seed-once: written on INSERT only and
+   * preserved across re-deploys (omitted from the conflict update), so a later
+   * `celilo proxmox … resize` is not reset back to the manifest minimum.
+   */
+  cpu?: number | null;
+  memory?: number | null;
+  disk?: number | null;
 }
 
 /**
@@ -117,6 +172,11 @@ export function upsertDeployedSystem(
       machineId: system.machineId ?? null,
       serviceId: system.serviceId ?? null,
       vmid: system.vmid ?? null,
+      // Seed-once (ISS-0150): set on insert; deliberately omitted from the
+      // conflict update below so a resize survives re-deploys.
+      cpu: system.cpu ?? null,
+      memory: system.memory ?? null,
+      disk: system.disk ?? null,
       updatedAt: new Date(),
     })
     .onConflictDoUpdate({
@@ -129,6 +189,9 @@ export function upsertDeployedSystem(
         machineId: system.machineId ?? null,
         serviceId: system.serviceId ?? null,
         vmid: system.vmid ?? null,
+        // NOTE: cpu/memory/disk intentionally NOT updated here — sizing is
+        // canonical state owned by `celilo proxmox … resize`, not reset by a
+        // routine re-deploy (seed-once). See ISS-0150 / CLAUDE.md.
         updatedAt: new Date(),
       },
     })
@@ -207,6 +270,11 @@ export async function recordDeployedSystemForModule(
     machineId: infrastructure?.machineId ?? null,
     serviceId: infrastructure?.serviceId ?? null,
     vmid: Number.isNaN(vmid as number) ? null : vmid,
+    // Seed canonical size from requires.system (seed-once; preserved across
+    // re-deploys). Only meaningful for celilo-provisioned instances.
+    cpu: decl.resources.cpu ?? null,
+    memory: decl.resources.memory ?? null,
+    disk: decl.resources.disk ?? null,
   });
 
   return getModuleSystems(moduleId, db);
@@ -279,6 +347,10 @@ export function backfillModuleSystems(db: DbClient): string[] {
       machineId: infra.machineId ?? null,
       serviceId: infra.serviceId ?? null,
       vmid: vmid != null && !Number.isNaN(vmid) ? vmid : null,
+      // Seed canonical size from requires.system for upgraded deployments.
+      cpu: decl.resources.cpu ?? null,
+      memory: decl.resources.memory ?? null,
+      disk: decl.resources.disk ?? null,
     });
     backfilled.push(infra.moduleId);
   }

package/src/services/dns-internal-records.test.ts

@@ -3,11 +3,13 @@ import { mkdtempSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import type { DnsRecordRequest } from '@celilo/capabilities';
+import type { ViewOverride } from '@celilo/capabilities';
 import type { DbClient } from '../db/client';
 import { modules } from '../db/schema';
 import { setupTestDatabase } from '../test-utils/setup-test-db';
 import {
   listDnsInternalRecords,
+  listViewOverrides,
   recordDnsInternalRecord,
   removeDnsInternalRecord,
   withDnsInternalLedger,
@@ -70,10 +72,27 @@ describe('dns-internal-records ledger', () => {
     expect(rows.map((r) => r.host)).toEqual(['b.x']);
   });
 
+  it('listViewOverrides returns only fronted records (zone IP set), as host→zoneIp', () => {
+    recordDnsInternalRecord(db, { ...ctx(), host: 'plain.x', ip: '192.168.0.253' });
+    recordDnsInternalRecord(db, {
+      ...ctx(),
+      host: 'git.celilo.computer',
+      ip: '192.168.0.253',
+      zoneRoutableIp: '10.0.10.10',
+    });
+    const overrides = listViewOverrides(db, 'technitium');
+    expect(overrides).toEqual([{ host: 'git.celilo.computer', ip: '10.0.10.10' }]);
+  });
+
   describe('withDnsInternalLedger', () => {
-    function fakeProvider() {
+    function fakeProvider(withViews = false) {
       const calls: Array<['register' | 'delete', DnsRecordRequest]> = [];
-      const iface = {
+      const reconcileCalls: ViewOverride[][] = [];
+      const iface: {
+        registerRecord(req: DnsRecordRequest): Promise<void>;
+        deleteRecord(req: DnsRecordRequest): Promise<void>;
+        reconcileViews?(o: ViewOverride[]): Promise<void>;
+      } = {
         async registerRecord(req: DnsRecordRequest) {
           calls.push(['register', req]);
         },
@@ -81,7 +100,12 @@ describe('dns-internal-records ledger', () => {
           calls.push(['delete', req]);
         },
       };
-      return { iface, calls };
+      if (withViews) {
+        iface.reconcileViews = async (o: ViewOverride[]) => {
+          reconcileCalls.push(o);
+        };
+      }
+      return { iface, calls, reconcileCalls };
     }
 
     it('records A-record registrations and passes the call through', async () => {
@@ -109,6 +133,55 @@ describe('dns-internal-records ledger', () => {
       expect(listDnsInternalRecords(db)).toHaveLength(0);
     });
 
+    it('reconciles views from the full ledger set when a fronted record is registered', async () => {
+      const { iface, reconcileCalls } = fakeProvider(true);
+      const wrapped = withDnsInternalLedger(iface, ctx());
+      // A plain (non-fronted) record must NOT trigger a view reconcile.
+      await wrapped.registerRecord({ host: 'plain.x', type: 'A', value: '192.168.0.253' });
+      expect(reconcileCalls).toHaveLength(0);
+      // A fronted record (zoneRoutableValue set) triggers a reconcile from the
+      // COMPLETE fronted set in the ledger.
+      await wrapped.registerRecord({
+        host: 'git.celilo.computer',
+        type: 'A',
+        value: '192.168.0.253',
+        zoneRoutableValue: '10.0.10.10',
+      });
+      expect(reconcileCalls).toHaveLength(1);
+      expect(reconcileCalls[0]).toEqual([{ host: 'git.celilo.computer', ip: '10.0.10.10' }]);
+    });
+
+    it('reconciles views on delete (a removed host drops from the set)', async () => {
+      const { iface, reconcileCalls } = fakeProvider(true);
+      const wrapped = withDnsInternalLedger(iface, ctx());
+      await wrapped.registerRecord({
+        host: 'git.celilo.computer',
+        type: 'A',
+        value: '192.168.0.253',
+        zoneRoutableValue: '10.0.10.10',
+      });
+      await wrapped.deleteRecord({
+        host: 'git.celilo.computer',
+        type: 'A',
+        value: '192.168.0.253',
+      });
+      // Last reconcile reflects the now-empty fronted set.
+      expect(reconcileCalls.at(-1)).toEqual([]);
+    });
+
+    it('is a no-op reconcile path for providers without view support', async () => {
+      const { iface } = fakeProvider(false);
+      const wrapped = withDnsInternalLedger(iface, ctx());
+      // Must not throw despite a fronted registration when reconcileViews is absent.
+      await wrapped.registerRecord({
+        host: 'git.celilo.computer',
+        type: 'A',
+        value: '192.168.0.253',
+        zoneRoutableValue: '10.0.10.10',
+      });
+      expect(listViewOverrides(db, 'technitium')).toHaveLength(1);
+    });
+
     it('does not write the ledger if the underlying register throws', async () => {
       const iface = {
         async registerRecord(): Promise<void> {

package/src/services/dns-internal-records.ts

@@ -13,14 +13,16 @@
  * Row lifecycle is FK cascade — records die with their provider or consumer.
  */
 
-import type { DnsInternalCapability, DnsRecordRequest } from '@celilo/capabilities';
-import { and, eq } from 'drizzle-orm';
+import type { DnsInternalCapability, DnsRecordRequest, ViewOverride } from '@celilo/capabilities';
+import { and, eq, isNotNull } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import { dnsInternalRecords } from '../db/schema';
 
 export interface DnsInternalRecordRow {
   host: string;
   ip: string;
+  /** In-zone split-horizon answer (caddy's zone IP), or null for plain records. */
+  zoneRoutableIp: string | null;
   providerModuleId: string;
   consumerModuleId: string;
   registeredAt: Date;
@@ -34,6 +36,7 @@ export function listDnsInternalRecords(
     .select({
       host: dnsInternalRecords.host,
       ip: dnsInternalRecords.ip,
+      zoneRoutableIp: dnsInternalRecords.zoneRoutableIp,
       providerModuleId: dnsInternalRecords.providerModuleId,
       consumerModuleId: dnsInternalRecords.consumerModuleId,
       registeredAt: dnsInternalRecords.registeredAt,
@@ -44,16 +47,45 @@ export function listDnsInternalRecords(
     : query.all();
 }
 
+/**
+ * The COMPLETE set of source-based split-horizon view overrides a provider
+ * should serve — every ledger record that carries a zone-routable IP (ISS-0156).
+ * This is the desired state the resolver's view config is reconciled from; the
+ * provider's `reconcileViews` is the single writer that materializes it.
+ */
+export function listViewOverrides(db: DbClient, providerModuleId: string): ViewOverride[] {
+  return db
+    .select({ host: dnsInternalRecords.host, ip: dnsInternalRecords.zoneRoutableIp })
+    .from(dnsInternalRecords)
+    .where(
+      and(
+        eq(dnsInternalRecords.providerModuleId, providerModuleId),
+        isNotNull(dnsInternalRecords.zoneRoutableIp),
+      ),
+    )
+    .all()
+    .map((r) => ({ host: r.host, ip: r.ip as string }));
+}
+
 export function recordDnsInternalRecord(
   db: DbClient,
-  record: { providerModuleId: string; consumerModuleId: string; host: string; ip: string },
+  record: {
+    providerModuleId: string;
+    consumerModuleId: string;
+    host: string;
+    ip: string;
+    /** In-zone split-horizon answer (caddy's zone IP); null/absent for plain records. */
+    zoneRoutableIp?: string | null;
+  },
 ): void {
+  const zoneRoutableIp = record.zoneRoutableIp ?? null;
   db.insert(dnsInternalRecords)
     .values({
       providerModuleId: record.providerModuleId,
       consumerModuleId: record.consumerModuleId,
       host: record.host,
       ip: record.ip,
+      zoneRoutableIp,
       registeredAt: new Date(),
     })
     .onConflictDoUpdate({
@@ -61,6 +93,7 @@ export function recordDnsInternalRecord(
       set: {
         consumerModuleId: record.consumerModuleId,
         ip: record.ip,
+        zoneRoutableIp,
         registeredAt: new Date(),
       },
     })
@@ -93,6 +126,17 @@ export function withDnsInternalLedger(
   ctx: { db: DbClient; providerModuleId: string; consumerModuleId: string },
 ): DnsInternalCapability {
   const isA = (request: DnsRecordRequest) => request.type.toUpperCase() === 'A';
+
+  // Source-based split-horizon (ISS-0156): after the ledger changes, re-materialize
+  // the provider's view config from the COMPLETE desired set. The provider's
+  // reconcileViews is the single writer; driving it from the ledger here (which
+  // both the live public_web path and the deploy-time backfill flow through)
+  // keeps live and reconcile in agreement. No-op for providers without views.
+  const reconcileViews = async (): Promise<void> => {
+    if (typeof iface.reconcileViews !== 'function') return;
+    await iface.reconcileViews(listViewOverrides(ctx.db, ctx.providerModuleId));
+  };
+
   return {
     ...iface,
     async registerRecord(request: DnsRecordRequest): Promise<void> {
@@ -103,7 +147,10 @@ export function withDnsInternalLedger(
           consumerModuleId: ctx.consumerModuleId,
           host: request.host,
           ip: request.value,
+          zoneRoutableIp: request.zoneRoutableValue ?? null,
         });
+        // Only a fronted record (one carrying a zone-routable IP) changes views.
+        if (request.zoneRoutableValue) await reconcileViews();
       }
     },
     async deleteRecord(request: DnsRecordRequest): Promise<void> {
@@ -113,6 +160,8 @@ export function withDnsInternalLedger(
           providerModuleId: ctx.providerModuleId,
           host: request.host,
         });
+        // The removed host may have been a fronted override — recompute the set.
+        await reconcileViews();
       }
     },
   };

package/src/services/dns-provider-backfill.ts

@@ -18,7 +18,11 @@ import type { DnsInternalCapability, HookLogger } from '@celilo/capabilities';
 import { and, eq, inArray, or } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import { capabilities as capabilitiesTable, modules, webRoutes } from '../db/schema';
-import { loadCapabilityFunctions, resolveFirewallNatIp } from '../hooks/capability-loader';
+import {
+  loadCapabilityFunctions,
+  resolveCaddyZoneIp,
+  resolveFirewallNatIp,
+} from '../hooks/capability-loader';
 import { runNamedHook } from '../hooks/run-named-hook';
 import type { HookName } from '../hooks/types';
 import { getModuleSystems } from './deployed-systems';
@@ -137,13 +141,21 @@ export async function backfillWebRouteDns(
     return;
   }
 
+  // caddy's zone-routable IP — the in-zone split-horizon answer (ISS-0156). When
+  // it differs from the natIp, each fronted hostname carries it as
+  // `zoneRoutableValue`; the provider's reconcileViews (driven from the ledger by
+  // the registration wrapper) materializes the per-zone view overrides.
+  const caddyZoneIp = await resolveCaddyZoneIp(db);
+  const zoneRoutableValue = caddyZoneIp && caddyZoneIp !== natIp ? caddyZoneIp : undefined;
+  const viewNote = zoneRoutableValue ? ` (in-zone view → ${zoneRoutableValue})` : '';
+
   logger.info(
-    `Backfilling ${hostnames.length} web-route hostname(s) into '${moduleId}' at ${natIp}`,
+    `Backfilling ${hostnames.length} web-route hostname(s) into '${moduleId}' at ${natIp}${viewNote}`,
   );
   const failures: string[] = [];
   for (const host of hostnames) {
     try {
-      await dnsInternal.registerRecord({ host, type: 'A', value: natIp });
+      await dnsInternal.registerRecord({ host, type: 'A', value: natIp, zoneRoutableValue });
     } catch (err) {
       failures.push(`${host}: ${err instanceof Error ? err.message : String(err)}`);
     }

package/src/services/fleet-checks.test.ts

@@ -242,7 +242,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
         variables: {
           owns: [
             {
-              name: 'idp_dmz_ip',
+              name: 'idp_auth_url',
               type: 'string',
               required: true,
               source: 'capability',
@@ -254,21 +254,21 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
       });
 
     it('fails when the consumed capability has no deployed provider', () => {
-      insertModule('forgejo', consumer('$capability:idp.dmz_ip'));
+      insertModule('forgejo', consumer('$capability:idp.auth_url'));
       const f = checkCapabilityProviders(db);
       expect(f.status).toBe('fail');
       expect(f.detail.join(' ')).toContain("no deployed module provides 'idp'");
     });
 
     it('fails when the provider lacks the referenced field', () => {
-      insertModule('forgejo', consumer('$capability:idp.dmz_ip'));
+      insertModule('forgejo', consumer('$capability:idp.auth_url'));
       insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
       db.insert(capabilitiesTable)
         .values({
           moduleId: 'authentik',
           capabilityName: 'idp',
           version: '1.0.0',
-          data: { auth_url: 'x' },
+          data: { admin_email: 'x' },
         })
         .run();
       const f = checkCapabilityProviders(db);
@@ -277,14 +277,14 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
     });
 
     it('passes when the provider carries a concrete value', () => {
-      insertModule('forgejo', consumer('$capability:idp.dmz_ip'));
+      insertModule('forgejo', consumer('$capability:idp.auth_url'));
       insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
       db.insert(capabilitiesTable)
         .values({
           moduleId: 'authentik',
           capabilityName: 'idp',
           version: '1.0.0',
-          data: { dmz_ip: '10.0.10.10' },
+          data: { auth_url: 'https://auth.celilo.computer' },
         })
         .run();
       const f = checkCapabilityProviders(db);
@@ -292,16 +292,16 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
     });
 
     it('passes but flags a derived ref for the chain trace (ISS-0114)', () => {
-      // The workstream-B case: idp.dmz_ip is present but is itself a ref
-      // ($self:caddy_dmz_ip) — we can't verify it resolves without the walker.
-      insertModule('forgejo', consumer('$capability:idp.dmz_ip'));
+      // idp.auth_url is present but is itself a ref ($self:auth_url) — we can't
+      // verify it resolves without the walker.
+      insertModule('forgejo', consumer('$capability:idp.auth_url'));
       insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
       db.insert(capabilitiesTable)
         .values({
           moduleId: 'authentik',
           capabilityName: 'idp',
           version: '1.0.0',
-          data: { dmz_ip: '$self:caddy_dmz_ip' },
+          data: { auth_url: '$self:auth_url' },
         })
         .run();
       const f = checkCapabilityProviders(db);
@@ -442,11 +442,11 @@ describe('findBrokenCapabilityDerivations (shared predicate)', () => {
     variables: {
       owns: [
         {
-          name: 'idp_dmz_ip',
+          name: 'idp_auth_url',
           type: 'string',
           required: true,
           source: 'capability',
-          derive_from: '$capability:idp.dmz_ip',
+          derive_from: '$capability:idp.auth_url',
         },
       ],
       imports: [],
@@ -461,21 +461,23 @@ describe('findBrokenCapabilityDerivations (shared predicate)', () => {
   });
 
   it('flags empty-value when the field is present but empty', () => {
-    const problems = findBrokenCapabilityDerivations('forgejo', consumer, { idp: { dmz_ip: '' } });
+    const problems = findBrokenCapabilityDerivations('forgejo', consumer, {
+      idp: { auth_url: '' },
+    });
     expect(problems[0].reason).toBe('empty-value');
   });
 
   it('flags unresolved-ref when the resolved value is still a template', () => {
     const problems = findBrokenCapabilityDerivations('forgejo', consumer, {
-      idp: { dmz_ip: '$self:caddy_dmz_ip' },
+      idp: { auth_url: '$self:auth_url' },
     });
     expect(problems[0].reason).toBe('unresolved-ref');
-    expect(problems[0].value).toBe('$self:caddy_dmz_ip');
+    expect(problems[0].value).toBe('$self:auth_url');
   });
 
   it('returns nothing when the field resolves to a concrete value', () => {
     const problems = findBrokenCapabilityDerivations('forgejo', consumer, {
-      idp: { dmz_ip: '10.0.10.10' },
+      idp: { auth_url: 'https://auth.celilo.computer' },
     });
     expect(problems).toHaveLength(0);
   });

package/src/services/machine-detector.ts

@@ -27,23 +27,45 @@ export class DetectionError extends Error {
 export type CommandRunner = (command: string) => string;
 
 /**
- * Execute SSH command and return output
+ * Per-command SSH timeout. Detection opens a fresh SSH connection per
+ * attribute (hostname, cpu, memory, ...); under load (e.g. the e2e builder
+ * running many containers) a single handshake + command can exceed a tight
+ * budget, surfacing as `spawnSync ... ETIMEDOUT` and failing an
+ * otherwise-healthy `machine add`. 30s gives headroom without masking a
+ * genuinely-unreachable host (which fails fast on connection refusal).
+ */
+const SSH_TIMEOUT_MS = 30_000;
+/** Retry attempts for a detection SSH command. Detection is read-only
+ * (hostname / nproc / free / ...), so retrying a transient timeout is safe. */
+const SSH_ATTEMPTS = 3;
+
+/** Synchronous sleep — detection runs synchronously (execSync), so there's no
+ * async context to await in. */
+function sleepSync(ms: number): void {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * Execute SSH command and return output. Retries transient failures
+ * (connection timeouts under load) with linear backoff — safe because every
+ * detection command is read-only.
  */
 function sshExec(ip: string, user: string, keyPath: string, command: string): string {
-  try {
-    const output = execSync(
-      `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -i "${keyPath}" ${user}@${ip} "${command}"`,
-      {
+  const sshCmd = `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=10 -i "${keyPath}" ${user}@${ip} "${command}"`;
+  let lastMessage = 'Unknown error';
+  for (let attempt = 1; attempt <= SSH_ATTEMPTS; attempt++) {
+    try {
+      return execSync(sshCmd, {
         encoding: 'utf8',
         stdio: ['pipe', 'pipe', 'pipe'],
-        timeout: 10000, // 10 second timeout
-      },
-    );
-    return output.trim();
-  } catch (error) {
-    const message = error instanceof Error ? error.message : 'Unknown error';
-    throw new DetectionError(`SSH command failed: ${message}`);
+        timeout: SSH_TIMEOUT_MS,
+      }).trim();
+    } catch (error) {
+      lastMessage = error instanceof Error ? error.message : 'Unknown error';
+      if (attempt < SSH_ATTEMPTS) sleepSync(1000 * attempt);
+    }
   }
+  throw new DetectionError(`SSH command failed after ${SSH_ATTEMPTS} attempts: ${lastMessage}`);
 }
 
 /** CommandRunner that SSHes to a remote machine. */