@celilo/cli 0.4.0-alpha.1 → 0.4.0

package/src/services/dns-provider-backfill.test.ts

@@ -0,0 +1,150 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { DnsRecordRequest, HookLogger } from '@celilo/capabilities';
+import { closeDb, getDb } from '../db/client';
+import { runMigrations } from '../db/migrate';
+import { capabilities, moduleConfigs, modules, webRoutes } from '../db/schema';
+import { backfillWebRouteDns } from './dns-provider-backfill';
+
+const silentLogger: HookLogger = {
+  info() {},
+  warn() {},
+  error() {},
+  debug() {},
+} as unknown as HookLogger;
+
+describe('backfillWebRouteDns (ISS-0029)', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-webroute-backfill-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  function seedModule(id: string) {
+    getDb()
+      .insert(modules)
+      .values({
+        id,
+        name: id,
+        version: '1.0.0',
+        manifestData: { id, name: id, version: '1.0.0', celilo_contract: '1.0' },
+        sourcePath: `/tmp/${id}`,
+      })
+      .run();
+  }
+
+  function seedFirewall(natIp: string) {
+    seedModule('iptables');
+    getDb()
+      .insert(capabilities)
+      .values({ moduleId: 'iptables', capabilityName: 'firewall', version: '1.0.0', data: {} })
+      .run();
+    getDb()
+      .insert(moduleConfigs)
+      .values({
+        moduleId: 'iptables',
+        key: 'nat_ip',
+        value: natIp,
+        valueJson: JSON.stringify(natIp),
+      })
+      .run();
+  }
+
+  function seedRoute(hostname: string, path = '/') {
+    // web_routes.module_id is a FK → modules; ensure the owner exists.
+    getDb()
+      .insert(modules)
+      .values({
+        id: 'caddy',
+        name: 'caddy',
+        version: '1.0.0',
+        manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '1.0' },
+        sourcePath: '/tmp/caddy',
+      })
+      .onConflictDoNothing()
+      .run();
+    getDb()
+      .insert(webRoutes)
+      .values({
+        slug: `${hostname}${path}`,
+        moduleId: 'caddy',
+        type: 'reverse_proxy',
+        path,
+        hostname,
+      })
+      .run();
+  }
+
+  /** Stub capability loader returning a dns_internal that records its calls. */
+  function stubLoader(calls: DnsRecordRequest[]) {
+    return async () => ({
+      dns_internal: {
+        async registerRecord(req: DnsRecordRequest) {
+          calls.push(req);
+        },
+        async deleteRecord() {},
+      },
+    });
+  }
+
+  it('registers each DISTINCT web-route hostname at the firewall nat_ip', async () => {
+    seedFirewall('100.64.0.1');
+    seedRoute('apt.celilo.computer', '/');
+    seedRoute('apt.celilo.computer', '/-/publish'); // same host, different path → deduped
+    seedRoute('registry.lunacycle.net', '/');
+
+    const calls: DnsRecordRequest[] = [];
+    await backfillWebRouteDns('technitium', getDb(), silentLogger, stubLoader(calls));
+
+    expect(calls.map((c) => c.host).sort()).toEqual([
+      'apt.celilo.computer',
+      'registry.lunacycle.net',
+    ]);
+    expect(calls.every((c) => c.value === '100.64.0.1' && c.type === 'A')).toBe(true);
+  });
+
+  it('skips without throwing when no firewall nat_ip is available', async () => {
+    seedRoute('apt.celilo.computer');
+    const calls: DnsRecordRequest[] = [];
+    await backfillWebRouteDns('technitium', getDb(), silentLogger, stubLoader(calls));
+    expect(calls).toHaveLength(0);
+  });
+
+  it('is a no-op when there are no web routes', async () => {
+    seedFirewall('100.64.0.1');
+    const calls: DnsRecordRequest[] = [];
+    await backfillWebRouteDns('technitium', getDb(), silentLogger, stubLoader(calls));
+    expect(calls).toHaveLength(0);
+  });
+
+  it('aggregates per-host failures into one error', async () => {
+    seedFirewall('100.64.0.1');
+    seedRoute('a.celilo.computer');
+    seedRoute('b.celilo.computer');
+    const failing = async () => ({
+      dns_internal: {
+        async registerRecord(req: DnsRecordRequest) {
+          if (req.host === 'a.celilo.computer') throw new Error('boom');
+        },
+        async deleteRecord() {},
+      },
+    });
+    await expect(backfillWebRouteDns('technitium', getDb(), silentLogger, failing)).rejects.toThrow(
+      /failed for 1 host/,
+    );
+  });
+});

package/src/services/dns-provider-backfill.ts

@@ -14,10 +14,11 @@
  * D5 division of labour: celilo owns host inventory, the module owns DNS.
  */
 
-import type { HookLogger } from '@celilo/capabilities';
+import type { DnsInternalCapability, HookLogger } from '@celilo/capabilities';
 import { and, eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { capabilities as capabilitiesTable, modules } from '../db/schema';
+import { capabilities as capabilitiesTable, modules, webRoutes } from '../db/schema';
+import { loadCapabilityFunctions, resolveFirewallNatIp } from '../hooks/capability-loader';
 import { runNamedHook } from '../hooks/run-named-hook';
 import type { HookName } from '../hooks/types';
 import { getModuleSystems } from './deployed-systems';
@@ -73,3 +74,72 @@ export async function backfillProviderDns(
     throw new Error(`DNS backfill failed for ${failures.length} host(s): ${failures.join('; ')}`);
   }
 }
+
+/**
+ * Backfill split-horizon records for every PUBLISHED web-route hostname
+ * (ISS-0029). The per-system backfill above covers each deployed host by its
+ * bare hostname; this covers the FQDNs modules publish via public_web (e.g.
+ * `apt.celilo.computer`), which a late-deploying provider would otherwise never
+ * learn. Each hostname is registered at the firewall NAT IP — the same value
+ * public_web uses for the live registration — so backfill and live agree.
+ *
+ * Unlike the per-system path, this does NOT go through `on_system_event` (which
+ * concatenates `<hostname>.<zone>` and would corrupt an already-FQDN host).
+ * It calls the provider's `registerRecord` directly with the FQDN; the
+ * provider creates the split-horizon zone on demand (Phase 1). Attempts every
+ * hostname, then throws an aggregate error if any failed.
+ */
+export async function backfillWebRouteDns(
+  moduleId: string,
+  db: DbClient,
+  logger: HookLogger,
+  // Injectable so unit tests don't dynamically import a real provider module.
+  loadCaps: typeof loadCapabilityFunctions = loadCapabilityFunctions,
+): Promise<void> {
+  const hostnames = [
+    ...new Set(
+      db
+        .select({ hostname: webRoutes.hostname })
+        .from(webRoutes)
+        .all()
+        .map((r) => r.hostname),
+    ),
+  ];
+  if (hostnames.length === 0) return;
+
+  const natIp = await resolveFirewallNatIp(db);
+  if (!natIp) {
+    // No firewall NAT IP to point at — the live public_web path falls back to
+    // Caddy's IP per route; we don't replicate that here. Records land when the
+    // route is (re)published. Surface it rather than silently doing nothing.
+    logger.warn(
+      `web-route DNS backfill for '${moduleId}' skipped: no firewall nat_ip available (${hostnames.length} hostname(s) deferred to live registration)`,
+    );
+    return;
+  }
+
+  const caps = await loadCaps(moduleId, db, logger);
+  const dnsInternal = caps.dns_internal as DnsInternalCapability | undefined;
+  if (!dnsInternal) {
+    logger.warn(`web-route DNS backfill: dns_internal capability not loadable for '${moduleId}'`);
+    return;
+  }
+
+  logger.info(
+    `Backfilling ${hostnames.length} web-route hostname(s) into '${moduleId}' at ${natIp}`,
+  );
+  const failures: string[] = [];
+  for (const host of hostnames) {
+    try {
+      await dnsInternal.registerRecord({ host, type: 'A', value: natIp });
+    } catch (err) {
+      failures.push(`${host}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  if (failures.length > 0) {
+    throw new Error(
+      `web-route DNS backfill failed for ${failures.length} host(s): ${failures.join('; ')}`,
+    );
+  }
+}

package/src/services/e2e-guard.test.ts

@@ -0,0 +1,38 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { E2E_CONFLICT_FIX, E2E_CONFLICT_MESSAGE, runningE2eContainers } from './e2e-guard';
+
+describe('e2e-guard', () => {
+  let prev: string | undefined;
+  beforeEach(() => {
+    prev = process.env.CELILO_DB_PATH;
+  });
+  afterEach(() => {
+    process.env.CELILO_DB_PATH = prev;
+  });
+
+  it('skips the docker check when running against a temp test DB', () => {
+    // Integration tests point CELILO_DB_PATH at os.tmpdir() and must not trip
+    // over a developer's unrelated e2e stack.
+    process.env.CELILO_DB_PATH = join(tmpdir(), 'celilo-test.db');
+    expect(runningE2eContainers()).toEqual([]);
+  });
+
+  it('returns an array and never throws when docker is checked', () => {
+    // A non-temp path defeats the short-circuit, exercising the real docker
+    // path. Result depends on the host's docker state; the contract is "an
+    // array of celilo-e2e-* names, never an exception".
+    process.env.CELILO_DB_PATH = '/opt/celilo/celilo.db';
+    const result = runningE2eContainers();
+    expect(Array.isArray(result)).toBe(true);
+    for (const name of result) {
+      expect(name.startsWith('celilo-e2e-')).toBe(true);
+    }
+  });
+
+  it('exposes a deploy message that includes the remediation hint', () => {
+    expect(E2E_CONFLICT_MESSAGE).toContain(E2E_CONFLICT_FIX);
+    expect(E2E_CONFLICT_MESSAGE).toContain('mutually exclusive');
+  });
+});

package/src/services/e2e-guard.ts

@@ -0,0 +1,43 @@
+/**
+ * Live/e2e mutual-exclusion guard.
+ *
+ * A live deploy and the e2e simulator can't run at the same time: the e2e
+ * stack binds the same docker networks and hostnames a live deploy touches,
+ * so a deploy fired while an e2e network is up (e.g. a leftover `--keep` run)
+ * would collide. Both the deploy itself and the fast pre-flight check use this
+ * helper to refuse before touching any infrastructure.
+ */
+
+import { execSync } from 'node:child_process';
+import { tmpdir } from 'node:os';
+
+/** One-line remediation, shared by the deploy error and the preflight error. */
+export const E2E_CONFLICT_FIX = 'Stop e2e tests first: cele2e down';
+
+/** Flat error string the deploy returns when an e2e stack is up. */
+export const E2E_CONFLICT_MESSAGE = `Cannot deploy: e2e test containers are running.
+Live and e2e environments are mutually exclusive.
+${E2E_CONFLICT_FIX}`;
+
+/**
+ * Names of running `celilo-e2e-*` containers (empty = clear to deploy).
+ *
+ * Skipped (returns `[]`) when CELILO_DB_PATH points at a temp dir, since
+ * integration tests run against throwaway DBs and never touch docker — they
+ * must not trip over a developer's unrelated e2e stack. A docker failure
+ * (not installed / not running) is treated as "clear": there's nothing to
+ * conflict with.
+ */
+export function runningE2eContainers(): string[] {
+  if (process.env.CELILO_DB_PATH?.startsWith(tmpdir())) return [];
+  try {
+    const running = execSync('docker ps --format "{{.Names}}" 2>/dev/null', {
+      encoding: 'utf-8',
+      timeout: 5000,
+    });
+    return running.split('\n').filter((name) => name.startsWith('celilo-e2e-'));
+  } catch {
+    // docker not installed / not running — nothing to conflict with.
+    return [];
+  }
+}

package/src/services/module-deploy.ts

@@ -35,6 +35,7 @@ import { waitForSSH } from './deploy-ssh';
 import { executeTerraform, parseTerraformOutputs } from './deploy-terraform';
 import { validateAndPrepareDeployment } from './deploy-validation';
 import { getModuleSystems } from './deployed-systems';
+import { E2E_CONFLICT_MESSAGE, runningE2eContainers } from './e2e-guard';
 import { resolveInfrastructureVariables } from './infrastructure-variable-resolver';
 import { findMachineForModule } from './machine-pool';
 import { checkProxmoxReachable, formatProxmoxUnreachableError } from './proxmox-preflight';
@@ -341,31 +342,12 @@ async function deployModuleImpl(
     : null;
 
   try {
-    // Check for e2e test containers — live and e2e environments are mutually exclusive.
-    // Skip when using a test database (integration tests use os.tmpdir() paths).
-    const { tmpdir } = await import('node:os');
-    const usingTestDb = process.env.CELILO_DB_PATH?.startsWith(tmpdir());
-    if (!usingTestDb) {
-      try {
-        const { execSync } = await import('node:child_process');
-        const running = execSync('docker ps --format "{{.Names}}" 2>/dev/null', {
-          encoding: 'utf-8',
-          timeout: 5000,
-        });
-        const e2eContainers = running.split('\n').filter((n) => n.startsWith('celilo-e2e-'));
-        if (e2eContainers.length > 0) {
-          return {
-            success: false,
-            error:
-              'Cannot deploy: e2e test containers are running.\n' +
-              'Live and e2e environments are mutually exclusive.\n' +
-              'Stop e2e tests first: cele2e down',
-            phases,
-          };
-        }
-      } catch {
-        // docker ps failed — Docker may not be installed or running, that's fine
-      }
+    // Live and e2e environments are mutually exclusive — refuse if an e2e
+    // stack is up (shared docker networks/hostnames). See e2e-guard.ts.
+    // Pre-flight checks the same condition; this is the defense-in-depth
+    // guard for callers that skip pre-flight.
+    if (runningE2eContainers().length > 0) {
+      return { success: false, error: E2E_CONFLICT_MESSAGE, phases };
     }
 
     const validation = await validateAndPrepareDeployment(moduleId, db);
@@ -1292,7 +1274,7 @@ async function deployModuleImpl(
       // provider (deliveries bind at emit time). Non-providers skip this
       // entirely; their registration rides the system.created event below.
       // v2/EVENT_DRIVEN_HOOK_SUBSCRIPTIONS.md.
-      const { isDnsInternalProvider, backfillProviderDns } = await import(
+      const { isDnsInternalProvider, backfillProviderDns, backfillWebRouteDns } = await import(
         './dns-provider-backfill'
       );
       if (isDnsInternalProvider(moduleId, db)) {
@@ -1304,7 +1286,11 @@ async function deployModuleImpl(
         dnsGauge.start();
         try {
           const dnsLogger = createGaugeLogger(dnsGauge, moduleId, 'dns_backfill');
+          // Per-system records (bare hostnames) ...
           await backfillProviderDns(moduleId, db, dnsLogger);
+          // ... and published web-route FQDNs (apt.celilo.computer), which the
+          // per-system path doesn't cover (ISS-0029).
+          await backfillWebRouteDns(moduleId, db, dnsLogger);
           dnsGauge.stop(true);
         } catch (error) {
           dnsGauge.stop(false);

package/src/services/responder-probe.test.ts

@@ -0,0 +1,87 @@
+/**
+ * Recurrence gate for ISS-0025: a headless deploy interview must fail fast
+ * instead of hanging forever when no responder is listening. The deploy path
+ * routes every interview query through `busInterviewGuarded`, which calls
+ * `ensureResponderForInterview` — so this guard is the choke point to protect.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { defineEvents, openBus } from '@celilo/event-bus';
+import { ensureResponderForInterview } from './responder-probe';
+
+const NO_SCHEMAS = defineEvents({});
+
+function setTTY(value: boolean | undefined): void {
+  Object.defineProperty(process.stdin, 'isTTY', { value, configurable: true });
+}
+
+describe('ensureResponderForInterview (ISS-0025)', () => {
+  let dir: string;
+  let dbPath: string;
+  let origEnv: string | undefined;
+  let origTTY: boolean | undefined;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'responder-guard-'));
+    dbPath = join(dir, 'events.db');
+    origEnv = process.env.EVENT_BUS_DB;
+    process.env.EVENT_BUS_DB = dbPath;
+    origTTY = process.stdin.isTTY;
+  });
+  afterEach(() => {
+    if (origEnv === undefined) process.env.EVENT_BUS_DB = undefined;
+    else process.env.EVENT_BUS_DB = origEnv;
+    setTTY(origTTY);
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('throws an actionable error when non-TTY and no responder is listening', async () => {
+    setTTY(false);
+    // No responder on the bus → the probe times out → fail fast, not a hang.
+    await expect(ensureResponderForInterview('config.required.foo.bar')).rejects.toThrow(
+      /No responder is listening/,
+    );
+  });
+
+  it('names the blocked prompt and the remediations in the error', async () => {
+    setTTY(false);
+    let message = '';
+    try {
+      await ensureResponderForInterview('secret.required.caddy.api_token');
+    } catch (err) {
+      message = err instanceof Error ? err.message : String(err);
+    }
+    expect(message).toContain('secret.required.caddy.api_token');
+    expect(message).toContain('celilo events respond');
+  });
+
+  it('resolves when a responder answers the probe', async () => {
+    setTTY(false);
+    const responder = openBus({ dbPath, events: NO_SCHEMAS });
+    responder.watch('responder.probe', (event) => {
+      responder.emitRaw(
+        `${event.type}.reply`,
+        { kind: 'programmatic', emittedBy: 'test' },
+        { replyFor: event.id, emittedBy: 'test' },
+      );
+    });
+
+    await expect(ensureResponderForInterview('config.required.foo.bar')).resolves.toBeUndefined();
+
+    responder.close();
+  });
+
+  it('skips the probe entirely on a TTY (the terminal-responder will answer)', async () => {
+    setTTY(true);
+    // No responder running, but a TTY short-circuits before probing — the
+    // built-in terminal-responder is the responder.
+    await expect(ensureResponderForInterview('config.required.foo.bar')).resolves.toBeUndefined();
+  });
+});

package/src/services/responder-probe.ts

@@ -13,6 +13,7 @@
  */
 
 import { defineEvents, openBus } from '@celilo/event-bus';
+import { getEventBusPath } from '../config/paths';
 
 const NO_SCHEMAS = defineEvents({});
 
@@ -43,3 +44,31 @@ export async function probeForResponder(busDbPath: string, timeoutMs = 1500): Pr
     bus.close();
   }
 }
+
+/**
+ * Fail fast instead of hanging forever on a deploy interview (ISS-0025).
+ *
+ * `busInterview` waits indefinitely (`timeoutMs: 0`) for a responder's reply.
+ * On a TTY the deploy registers a terminal-responder, so a prompt will be
+ * answered — we skip the probe. Headless (non-TTY) with no responder listening,
+ * the prompt would hang forever; we probe once and throw an actionable error so
+ * a `module generate`-style fail-fast applies to deploys too. Call this
+ * immediately before emitting an interview query (see `busInterviewGuarded`).
+ *
+ * @param queryType the interview event type about to be emitted (e.g.
+ *   `config.required.<m>.<k>`), echoed in the error so the operator sees which
+ *   prompt is blocked.
+ */
+export async function ensureResponderForInterview(queryType: string): Promise<void> {
+  if (process.stdin.isTTY) return;
+  const available = await probeForResponder(getEventBusPath());
+  if (available) return;
+  throw new Error(
+    `No responder is listening and stdin isn't a TTY, so this interview prompt can't be answered (${queryType}).
+
+Either:
+  1. Run it in a terminal — the built-in prompt will ask, or
+  2. Start a responder in another shell:  celilo events respond
+     (or pre-stage answers:  celilo events respond --values <file>)`,
+  );
+}

package/src/services/restore-from-file.ts

@@ -233,15 +233,25 @@ export async function restoreFromArtifactFile(
       }
     }
 
-    // 7. Apply staged system files (celilo.db + master.key).
-    //    These were staged by celilo-mgmt's on_restore at
-    //    restore_dir/system/. The live process has the DB open, so
-    //    this swap must happen AFTER the hook returns (which is now).
+    // 7. Complete the operation BEFORE swapping the DB. The swap
+    //    (applyStagedSystemFiles) is the final, irreversible step: it closes
+    //    the live DB and overwrites celilo.db with the artifact's. Marking the
+    //    operation complete writes to module_operations — if done AFTER the
+    //    swap it reopens the freshly-staged artifact DB in THIS process and
+    //    throws SQLITE_IOERR (a fresh process opens the same file fine; it's a
+    //    same-process post-swap reopen artifact). The operation row lives in
+    //    the pre-swap DB the swap discards anyway, so completing it here —
+    //    while that DB is still open — is correct and avoids the reopen.
+    //    Nothing in-process must touch the DB after the swap; migrateRestoredDb
+    //    (the CLI's next step) is best-effort and tolerates a reopen failure.
+    completeOperation(opId);
+
+    // 8. Apply staged system files (celilo.db + master.key + fleet SSH key).
+    //    Staged by celilo-mgmt's on_restore at restore_dir/system/. The live
+    //    process had the DB open, so this swap happens AFTER the hook returns.
     const systemStaging = join(restoreDataDir, 'system');
     const apply = applyStagedSystemFiles(systemStaging);
 
-    completeOperation(opId);
-
     return {
       success: true,
       crossModuleApplied,

package/src/services/storage-providers/s3.test.ts

@@ -0,0 +1,101 @@
+/**
+ * Recurrence gate for ISS-0016: the S3 provider's upload() must send a
+ * self-describing, replayable Buffer body — NOT a one-shot Node read stream.
+ *
+ * The original bug passed `createReadStream(localPath)` as PutObject Body with
+ * no ContentLength; AWS SDK v3 fell back to aws-chunked streaming and failed
+ * with "The request body terminated unexpectedly" (and couldn't replay the
+ * stream across S3's retries/redirects). The verify path used a string body
+ * (length known) and so never exercised the broken path — every real S3 backup
+ * silently failed. These tests upload a REAL on-disk file and assert the body
+ * is a Buffer, so a regression back to a stream is caught here.
+ *
+ * No live S3 / MinIO harness exists, so we intercept S3Client.prototype.send
+ * and inspect the command the provider builds.
+ */
+
+import { type Mock, afterEach, describe, expect, it, spyOn } from 'bun:test';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { type S3StorageConfig, createS3StorageProvider } from './s3';
+
+const CONFIG: S3StorageConfig = {
+  bucket: 'test-bucket',
+  region: 'us-east-1',
+  endpoint: 'http://localhost:9000',
+  accessKeyId: 'test-key',
+  secretAccessKey: 'test-secret',
+};
+
+describe('s3 storage provider (ISS-0016)', () => {
+  let dir: string | undefined;
+  // biome-ignore lint/suspicious/noExplicitAny: send() is heavily overloaded; the test only reads .input.
+  let sendSpy: Mock<(command: any) => Promise<unknown>> | undefined;
+
+  afterEach(() => {
+    sendSpy?.mockRestore();
+    sendSpy = undefined;
+    if (dir) {
+      rmSync(dir, { recursive: true, force: true });
+      dir = undefined;
+    }
+  });
+
+  it('uploads a real on-disk file as a self-describing Buffer body, not a stream', async () => {
+    dir = mkdtempSync(join(tmpdir(), 's3-upload-'));
+    const file = join(dir, 'envelope.tar.gz');
+    // A multi-KB real file — the case that broke against live S3.
+    const bytes = Buffer.from('celilo backup envelope payload — '.repeat(2000));
+    writeFileSync(file, bytes);
+
+    // biome-ignore lint/suspicious/noExplicitAny: capturing the built command.
+    const sent: any[] = [];
+    sendSpy = spyOn(S3Client.prototype, 'send').mockImplementation(async (command: unknown) => {
+      sent.push(command);
+      return {};
+    });
+
+    const provider = createS3StorageProvider(CONFIG);
+    await provider.upload(file, 'celilo-mgmt/2026/envelope.tar.gz');
+
+    expect(sent).toHaveLength(1);
+    const command = sent[0];
+    expect(command).toBeInstanceOf(PutObjectCommand);
+
+    const body = command.input.Body;
+    // The regression gate: a Node read stream (the original bug) is not a Buffer.
+    expect(Buffer.isBuffer(body)).toBe(true);
+    expect(body instanceof Readable).toBe(false);
+    // Self-describing length == the whole file: the SDK derives ContentLength
+    // and can replay the body across retries/redirects.
+    expect(body.length).toBe(bytes.length);
+    expect(Buffer.compare(body, bytes)).toBe(0);
+    // Key is prefixed with the backup namespace.
+    expect(command.input.Key).toBe('celilo-backups/celilo-mgmt/2026/envelope.tar.gz');
+    expect(command.input.Bucket).toBe('test-bucket');
+  });
+
+  it('downloads a multi-chunk response body to disk intact (short-read safe)', async () => {
+    dir = mkdtempSync(join(tmpdir(), 's3-download-'));
+    const out = join(dir, 'restored.bin');
+    const chunks = [Buffer.from('chunk-1-'), Buffer.from('chunk-2-'), Buffer.from('chunk-3')];
+    const expected = Buffer.concat(chunks);
+
+    sendSpy = spyOn(S3Client.prototype, 'send').mockImplementation(async (command: unknown) => {
+      expect(command).toBeInstanceOf(GetObjectCommand);
+      const body = new Readable({ read() {} });
+      // Push several chunks separately to model short reads over the wire.
+      for (const chunk of chunks) body.push(chunk);
+      body.push(null);
+      return { Body: body };
+    });
+
+    const provider = createS3StorageProvider(CONFIG);
+    await provider.download('celilo-mgmt/2026/envelope.tar.gz', out);
+
+    expect(readFileSync(out)).toEqual(expected);
+  });
+});