@celilo/cli 0.21.0 → 0.23.0

package/src/services/module-pause.test.ts

@@ -0,0 +1,573 @@
+/**
+ * Unit tests for pause planning + execution.
+ *
+ * The planner is pure, so the interesting cases (ordering, refusals, walking
+ * through already-done members) need no database at all. Execution is tested
+ * against fake deps — the point is the state machine, not Proxmox.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { type DbClient, createDbClient } from '../db/client';
+import { ipAllocations, moduleConfigs, modules, secrets } from '../db/schema';
+import type { ModuleState } from '../db/schema';
+import type { ModuleManifest } from '../manifest/schema';
+import {
+  type ModuleSnapshot,
+  type PauseDeps,
+  type PausePlan,
+  PauseRefusedError,
+  actedOn,
+  executePause,
+  executeUnpause,
+  formatPausedDuration,
+  planPause,
+  planUnpause,
+} from './module-pause';
+
+function makeManifest(
+  id: string,
+  opts: { provides?: string[]; requires?: string[]; optional?: string[] } = {},
+): ModuleManifest {
+  return {
+    id,
+    name: id,
+    version: '1.0.0',
+    celilo_contract: '1.0',
+    provides: {
+      capabilities: (opts.provides ?? []).map((name) => ({
+        name,
+        version: '1.0.0',
+        data: {},
+        functions: [],
+      })),
+    },
+    requires: { capabilities: (opts.requires ?? []).map((name) => ({ name, version: '1.0.0' })) },
+    optional: opts.optional
+      ? { capabilities: opts.optional.map((name) => ({ name, version: '1.0.0' })) }
+      : undefined,
+  } as unknown as ModuleManifest;
+}
+
+function snap(
+  id: string,
+  state: ModuleState,
+  opts: { provides?: string[]; requires?: string[]; optional?: string[] } = {},
+): ModuleSnapshot {
+  return {
+    id,
+    state,
+    pausedAt: state === 'PAUSED' ? new Date('2026-08-01T00:00:00Z') : null,
+    pauseReason: state === 'PAUSED' ? 'swapping the edge router' : null,
+    manifest: makeManifest(id, opts),
+  };
+}
+
+/**
+ * The real shape of the greenwave → axon problem: a firewall provider with a
+ * chain of consumers, one of which reaches it only via `optional`.
+ *
+ *   greenwave ──provides firewall──> caddy (requires) ──provides public_web──> website
+ *                                    technitium (OPTIONAL dhcp_server)
+ */
+function fleet(states: Partial<Record<string, ModuleState>> = {}): ModuleSnapshot[] {
+  const state = (id: string, fallback: ModuleState = 'INSTALLED') => states[id] ?? fallback;
+  return [
+    snap('greenwave', state('greenwave'), { provides: ['firewall', 'dhcp_server'] }),
+    snap('caddy', state('caddy'), { provides: ['public_web'], requires: ['firewall'] }),
+    snap('website', state('website'), { requires: ['public_web'] }),
+    snap('technitium', state('technitium'), { optional: ['dhcp_server'] }),
+    snap('unrelated', state('unrelated')),
+  ];
+}
+
+describe('planPause — ordering', () => {
+  test('a plain pause is single-module, whatever depends on it', () => {
+    const plan = planPause({ moduleId: 'greenwave', fleet: fleet(), cascade: false });
+    expect(plan.steps.map((s) => s.moduleId)).toEqual(['greenwave']);
+  });
+
+  test('cascade pause covers the transitive consumers and pauses them FIRST', () => {
+    const plan = planPause({ moduleId: 'greenwave', fleet: fleet(), cascade: true });
+    const order = plan.steps.map((s) => s.moduleId);
+
+    expect(new Set(order)).toEqual(new Set(['greenwave', 'caddy', 'website', 'technitium']));
+    expect(order).not.toContain('unrelated');
+
+    // Consumers before the provider they depend on — the ordering IS the
+    // feature, so assert relative position rather than membership.
+    expect(order.indexOf('website')).toBeLessThan(order.indexOf('caddy'));
+    expect(order.indexOf('caddy')).toBeLessThan(order.indexOf('greenwave'));
+    expect(order.indexOf('technitium')).toBeLessThan(order.indexOf('greenwave'));
+  });
+
+  test('cascade unpause runs the same set in the opposite order — providers first', () => {
+    const paused = fleet({
+      greenwave: 'PAUSED',
+      caddy: 'PAUSED',
+      website: 'PAUSED',
+      technitium: 'PAUSED',
+    });
+    const order = planUnpause({ moduleId: 'greenwave', fleet: paused, cascade: true }).steps.map(
+      (s) => s.moduleId,
+    );
+
+    expect(order.indexOf('greenwave')).toBeLessThan(order.indexOf('caddy'));
+    expect(order.indexOf('caddy')).toBeLessThan(order.indexOf('website'));
+  });
+
+  test('an `optional` consumer is in the cascade — the graph counts it as an edge', () => {
+    const plan = planPause({ moduleId: 'greenwave', fleet: fleet(), cascade: true });
+    expect(plan.steps.map((s) => s.moduleId)).toContain('technitium');
+  });
+});
+
+describe('planPause — legal source states (task 3.7)', () => {
+  test.each([['INSTALLED'], ['VERIFIED'], ['ERROR']] as const)(
+    '%s is pausable',
+    (state: ModuleState) => {
+      const plan = planPause({
+        moduleId: 'greenwave',
+        fleet: fleet({ greenwave: state }),
+        cascade: false,
+      });
+      expect(plan.steps[0]).toEqual({ moduleId: 'greenwave', disposition: 'act' });
+    },
+  );
+
+  test('a never-deployed module is refused, and the message says why', () => {
+    expect(() =>
+      planPause({ moduleId: 'greenwave', fleet: fleet({ greenwave: 'IMPORTED' }), cascade: false }),
+    ).toThrow(/never been deployed/);
+  });
+
+  test('a module mid-transition is refused with a DIFFERENT message', () => {
+    expect(() =>
+      planPause({
+        moduleId: 'greenwave',
+        fleet: fleet({ greenwave: 'DEPLOYING' }),
+        cascade: false,
+      }),
+    ).toThrow(/strand/);
+  });
+
+  test('an in-flight operation refuses the pause and names the operation', () => {
+    expect(() =>
+      planPause({
+        moduleId: 'greenwave',
+        fleet: fleet(),
+        cascade: false,
+        inFlight: new Map([['greenwave', 'backup of greenwave (pid 4242)']]),
+      }),
+    ).toThrow(/backup of greenwave \(pid 4242\)/);
+  });
+
+  test('a cascade is refused WHOLE when any member cannot be paused', () => {
+    // caddy is mid-deploy. Pausing the rest and stopping would leave the
+    // operator to work out which half happened.
+    expect(() =>
+      planPause({ moduleId: 'greenwave', fleet: fleet({ caddy: 'DEPLOYING' }), cascade: true }),
+    ).toThrow(PauseRefusedError);
+  });
+
+  test('an unknown module is refused rather than silently planning nothing', () => {
+    expect(() => planPause({ moduleId: 'nope', fleet: fleet(), cascade: false })).toThrow(
+      /Module not found/,
+    );
+  });
+});
+
+describe('a cascade walks THROUGH members already in the target condition (task 4.7)', () => {
+  // The failure the operator specifically called out: `unpause --cascade caddy`
+  // when caddy is already unpaused must still reach the modules beyond it.
+  // A cascade that halted at the first already-done member would leave the
+  // consumers paused forever, and a half-finished cascade unresumable.
+  test('unpause --cascade on an ALREADY-UNPAUSED module still reaches its consumers', () => {
+    const plan = planUnpause({
+      moduleId: 'caddy',
+      fleet: fleet({ caddy: 'INSTALLED', website: 'PAUSED' }),
+      cascade: true,
+    });
+
+    const ids = plan.steps.map((s) => s.moduleId);
+    expect(ids).toContain('caddy');
+    expect(ids).toContain('website');
+
+    expect(plan.steps.find((s) => s.moduleId === 'caddy')?.disposition).toBe('skip_already');
+    expect(plan.steps.find((s) => s.moduleId === 'website')?.disposition).toBe('act');
+    expect(actedOn(plan)).toEqual(['website']);
+  });
+
+  test('pause --cascade over a partly paused set plans only the outstanding work', () => {
+    const plan = planPause({
+      moduleId: 'greenwave',
+      fleet: fleet({ website: 'PAUSED', technitium: 'PAUSED' }),
+      cascade: true,
+    });
+
+    expect(actedOn(plan).sort()).toEqual(['caddy', 'greenwave']);
+    // Still PRESENT in the plan — the report tells the operator what was
+    // already done rather than pretending it was not in scope.
+    expect(plan.steps.map((s) => s.moduleId)).toContain('website');
+  });
+
+  test('an already-paused member does not trip the source-state refusal', () => {
+    // PAUSED is not in PAUSABLE_STATES, so a naive validator would refuse the
+    // whole cascade the second time it ran — i.e. exactly when resuming.
+    expect(() =>
+      planPause({ moduleId: 'greenwave', fleet: fleet({ caddy: 'PAUSED' }), cascade: true }),
+    ).not.toThrow();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Execution
+// ---------------------------------------------------------------------------
+
+/**
+ * Execution runs against a REAL isolated SQLite database rather than a fake
+ * drizzle chain. A hand-rolled fake cannot honour a `where` clause, so it
+ * silently answered the wrong row for the read-before-redeploy that preserves
+ * the original `pausedAt` — the fake would have hidden precisely the bug that
+ * read exists to prevent. Everything else (deploy, bus, Proxmox) stays injected.
+ */
+interface FakeState {
+  unsubscribed: string[];
+  resubscribed: string[];
+  redeployed: string[];
+  operations: Array<{ moduleId: string; kind: string; outcome: 'completed' | 'failed' }>;
+  infraStopped: string[];
+}
+
+function makeDb(): DbClient {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-pause-'));
+  process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+  return createDbClient({ path: process.env.CELILO_DB_PATH });
+}
+
+function seed(db: DbClient, initial: Record<string, ModuleState>): void {
+  for (const [id, state] of Object.entries(initial)) {
+    db.insert(modules)
+      .values({
+        id,
+        name: id,
+        version: '1.0.0',
+        state,
+        manifestData: makeManifest(id) as unknown as Record<string, unknown>,
+        sourcePath: `/tmp/${id}`,
+        pausedAt: state === 'PAUSED' ? new Date('2026-08-01T00:00:00Z') : null,
+        pauseReason: state === 'PAUSED' ? 'original reason' : null,
+      })
+      .run();
+  }
+}
+
+function readRow(db: DbClient, id: string) {
+  return db.select().from(modules).where(eq(modules.id, id)).get();
+}
+
+function fakeDeps(
+  initial: Record<string, ModuleState>,
+  overrides: { failRedeployOf?: string } = {},
+): { deps: PauseDeps; state: FakeState; db: DbClient } {
+  const db = makeDb();
+  seed(db, initial);
+
+  const state: FakeState = {
+    unsubscribed: [],
+    resubscribed: [],
+    redeployed: [],
+    operations: [],
+    infraStopped: [],
+  };
+
+  const deps: PauseDeps = {
+    db,
+    unsubscribe: (id) => state.unsubscribed.push(id),
+    resubscribe: (id) => state.resubscribed.push(id),
+    redeploy: async (id) => {
+      state.redeployed.push(id);
+      if (overrides.failRedeployOf === id) {
+        // A real failed deploy leaves `state` somewhere else entirely — often
+        // ERROR. Model that, so the test proves the executor RESTORES PAUSED
+        // rather than merely never having left it.
+        db.update(modules).set({ state: 'ERROR' }).where(eq(modules.id, id)).run();
+        return { success: false, error: 'bad config' };
+      }
+      db.update(modules).set({ state: 'INSTALLED' }).where(eq(modules.id, id)).run();
+      return { success: true };
+    },
+    stopInfrastructure: async (id) => {
+      state.infraStopped.push(id);
+      return { stopped: true, detail: `stopped ${id}` };
+    },
+    startOperation: (moduleId, kind) => {
+      state.operations.push({ moduleId, kind, outcome: 'completed' });
+      return `${moduleId}:${kind}`;
+    },
+    completeOperation: () => {},
+    failOperation: (operationId) => {
+      const entry = state.operations.find((o) => `${o.moduleId}:${o.kind}` === operationId);
+      if (entry) entry.outcome = 'failed';
+    },
+    now: () => new Date('2026-08-12T00:00:00Z'),
+    log: () => {},
+  };
+
+  return { deps, state, db };
+}
+
+function planFor(action: 'pause' | 'unpause', ids: string[], stopInfra = false): PausePlan {
+  return {
+    action,
+    requested: ids[0],
+    cascade: ids.length > 1,
+    stopInfra,
+    steps: ids.map((id) => ({ moduleId: id, disposition: 'act' as const })),
+  };
+}
+
+describe('executePause', () => {
+  test('pauses, quiesces, and records the reason', async () => {
+    const { deps, state, db } = fakeDeps({ greenwave: 'INSTALLED' });
+    const report = await executePause(planFor('pause', ['greenwave']), deps, 'edge router swap');
+
+    expect(report.success).toBe(true);
+    expect(readRow(db, 'greenwave')?.state).toBe('PAUSED');
+    expect(readRow(db, 'greenwave')?.pauseReason).toBe('edge router swap');
+    expect(state.unsubscribed).toEqual(['greenwave']);
+  });
+
+  test('does NOT stop infrastructure by default (design D2)', async () => {
+    const { deps, state } = fakeDeps({ caddy: 'INSTALLED' });
+    await executePause(planFor('pause', ['caddy']), deps, null);
+    // Pausing caddy to swap the FIREWALL must not take every website down.
+    expect(state.infraStopped).toEqual([]);
+  });
+
+  test('--stop-infra stops it', async () => {
+    const { deps, state } = fakeDeps({ caddy: 'INSTALLED' });
+    await executePause(planFor('pause', ['caddy'], true), deps, null);
+    expect(state.infraStopped).toEqual(['caddy']);
+  });
+
+  test('re-pausing preserves the ORIGINAL pausedAt and reason (task 3.5)', async () => {
+    const { deps, db } = fakeDeps({ greenwave: 'PAUSED' });
+    const plan: PausePlan = {
+      action: 'pause',
+      requested: 'greenwave',
+      cascade: false,
+      stopInfra: false,
+      steps: [{ moduleId: 'greenwave', disposition: 'skip_already', note: 'already paused' }],
+    };
+
+    const report = await executePause(plan, deps, 'a NEW reason');
+
+    expect(report.success).toBe(true);
+    // The age must keep measuring the real outage, not reset on every retry.
+    expect(readRow(db, 'greenwave')?.pausedAt).toEqual(new Date('2026-08-01T00:00:00Z'));
+    expect(readRow(db, 'greenwave')?.pauseReason).toBe('original reason');
+  });
+});
+
+describe('executeUnpause', () => {
+  test('redeploys, clears the pause, and re-arms the subscriptions', async () => {
+    const { deps, state, db } = fakeDeps({ greenwave: 'PAUSED' });
+    const report = await executeUnpause(planFor('unpause', ['greenwave']), deps);
+
+    expect(report.success).toBe(true);
+    expect(state.redeployed).toEqual(['greenwave']);
+    expect(readRow(db, 'greenwave')?.state).toBe('INSTALLED');
+    expect(readRow(db, 'greenwave')?.pausedAt).toBeNull();
+    // Deploy does not register subscriptions — without this the module comes
+    // back deployed but permanently deaf.
+    expect(state.resubscribed).toEqual(['greenwave']);
+  });
+
+  test('a failed redeploy leaves the module PAUSED and quiesced (task 3.6)', async () => {
+    const { deps, state, db } = fakeDeps({ caddy: 'PAUSED' }, { failRedeployOf: 'caddy' });
+    const report = await executeUnpause(planFor('unpause', ['caddy']), deps);
+
+    expect(report.success).toBe(false);
+    // The fake deploy moved it to ERROR; the executor must put it back.
+    expect(readRow(db, 'caddy')?.state).toBe('PAUSED');
+    expect(readRow(db, 'caddy')?.pausedAt).toEqual(new Date('2026-08-01T00:00:00Z'));
+    expect(state.unsubscribed).toContain('caddy');
+    expect(state.resubscribed).toEqual([]);
+    expect(state.operations.find((o) => o.kind === 'unpause')?.outcome).toBe('failed');
+  });
+
+  test('a failed provider stops the cascade and leaves consumers paused, not mis-bound', async () => {
+    const { deps, state, db } = fakeDeps(
+      { greenwave: 'PAUSED', caddy: 'PAUSED', website: 'PAUSED' },
+      { failRedeployOf: 'greenwave' },
+    );
+
+    const report = await executeUnpause(
+      planFor('unpause', ['greenwave', 'caddy', 'website']),
+      deps,
+    );
+
+    expect(report.success).toBe(false);
+    // Redeploying a consumer against a provider that is not there is the exact
+    // mis-binding this design exists to prevent.
+    expect(state.redeployed).toEqual(['greenwave']);
+    expect(readRow(db, 'caddy')?.state).toBe('PAUSED');
+    expect(readRow(db, 'website')?.state).toBe('PAUSED');
+    expect(report.outcomes.filter((o) => o.result === 'skipped')).toHaveLength(2);
+  });
+
+  test('an already-unpaused member is skipped WITHOUT a redeploy (task 4.8)', async () => {
+    const { deps, state, db } = fakeDeps({ caddy: 'INSTALLED', website: 'PAUSED' });
+    const plan: PausePlan = {
+      action: 'unpause',
+      requested: 'caddy',
+      cascade: true,
+      stopInfra: false,
+      steps: [
+        { moduleId: 'caddy', disposition: 'skip_already', note: 'not paused' },
+        { moduleId: 'website', disposition: 'act' },
+      ],
+    };
+
+    const report = await executeUnpause(plan, deps);
+
+    expect(report.success).toBe(true);
+    expect(state.redeployed).toEqual(['website']);
+    expect(readRow(db, 'website')?.state).toBe('INSTALLED');
+  });
+});
+
+describe('formatPausedDuration', () => {
+  const now = new Date('2026-08-12T12:00:00Z');
+
+  test.each([
+    [new Date('2026-08-12T11:59:30Z'), 'just now'],
+    [new Date('2026-08-12T11:15:00Z'), '45m'],
+    [new Date('2026-08-12T09:00:00Z'), '3h'],
+    [new Date('2026-08-09T12:00:00Z'), '3d'],
+  ])('%s → %s', (pausedAt, expected) => {
+    expect(formatPausedDuration(pausedAt, now)).toBe(expected);
+  });
+
+  test('a null timestamp reads as unknown rather than as zero', () => {
+    // Rendering "just now" for a missing timestamp would make an old pause look
+    // fresh, which is the one direction that matters.
+    expect(formatPausedDuration(null, now)).toBe('unknown');
+  });
+});
+
+describe('a pause/unpause round trip preserves everything (task 1.4)', () => {
+  // Pausing is NOT a partial uninstall. If a round trip lost config, secrets, or
+  // the IPAM allocation, the provider swap would come back with a different IP
+  // and the whole exercise would be worse than the removal it replaced.
+  test('config, secret and IPAM/VMID rows are byte-identical before and after', async () => {
+    const { deps, db } = fakeDeps({ caddy: 'INSTALLED' });
+
+    db.insert(moduleConfigs)
+      .values({
+        moduleId: 'caddy',
+        key: 'domain',
+        value: 'example.test',
+        valueJson: '"example.test"',
+      })
+      .run();
+    db.insert(secrets)
+      .values({
+        moduleId: 'caddy',
+        name: 'api_key',
+        encryptedValue: 'ciphertext',
+        iv: 'iv-value',
+        authTag: 'tag-value',
+      })
+      .run();
+    db.insert(ipAllocations)
+      .values({ moduleId: 'caddy', vmid: 231, containerIp: '10.0.20.31/24', zone: 'dmz' })
+      .run();
+
+    const before = {
+      configs: db.select().from(moduleConfigs).where(eq(moduleConfigs.moduleId, 'caddy')).all(),
+      secrets: db.select().from(secrets).where(eq(secrets.moduleId, 'caddy')).all(),
+      ipam: db.select().from(ipAllocations).where(eq(ipAllocations.moduleId, 'caddy')).all(),
+    };
+
+    await executePause(planFor('pause', ['caddy']), deps, 'edge router swap');
+    expect(readRow(db, 'caddy')?.state).toBe('PAUSED');
+
+    await executeUnpause(planFor('unpause', ['caddy']), deps);
+    expect(readRow(db, 'caddy')?.state).toBe('INSTALLED');
+
+    expect(
+      db.select().from(moduleConfigs).where(eq(moduleConfigs.moduleId, 'caddy')).all(),
+    ).toEqual(before.configs);
+    expect(db.select().from(secrets).where(eq(secrets.moduleId, 'caddy')).all()).toEqual(
+      before.secrets,
+    );
+    // The allocation in particular: releasing it would hand the address to the
+    // next module and make the unpause land somewhere else entirely.
+    expect(
+      db.select().from(ipAllocations).where(eq(ipAllocations.moduleId, 'caddy')).all(),
+    ).toEqual(before.ipam);
+  });
+
+  test('the pause itself does not run on_uninstall', async () => {
+    // Pausing is not a partial uninstall: withdrawing the module's cross-module
+    // state is exactly what must NOT happen, or the consumers it registered for
+    // would be torn down by a pause meant to protect them.
+    const { deps, state } = fakeDeps({ caddy: 'INSTALLED' });
+    await executePause(planFor('pause', ['caddy']), deps, null);
+
+    expect(state.operations.map((o) => o.kind)).toEqual(['pause']);
+    expect(state.redeployed).toEqual([]);
+  });
+});
+
+describe('a cascade skips undeployed members instead of refusing (found by e2e)', () => {
+  // Regression: `pause --cascade greenwave` refused outright because
+  // `technitium` was IMPORTED — swept in from the manifest graph, never
+  // deployed. An undeployed module is not bound to the provider and has nothing
+  // to quiesce, so blocking on it wedges exactly the migration this feature
+  // exists to enable: one stray imported module stops the swap.
+  test('an IMPORTED consumer is skipped, and the rest of the cascade proceeds', () => {
+    const plan = planPause({
+      moduleId: 'greenwave',
+      fleet: fleet({ technitium: 'IMPORTED' }),
+      cascade: true,
+    });
+
+    const technitium = plan.steps.find((s) => s.moduleId === 'technitium');
+    expect(technitium?.disposition).toBe('skip_undeployed');
+    expect(actedOn(plan).sort()).toEqual(['caddy', 'greenwave', 'website']);
+  });
+
+  test.each([['IMPORTED'], ['VALIDATED'], ['CONFIGURED']] as const)(
+    'a %s consumer does not block the cascade',
+    (state: ModuleState) => {
+      expect(() =>
+        planPause({ moduleId: 'greenwave', fleet: fleet({ technitium: state }), cascade: true }),
+      ).not.toThrow();
+    },
+  );
+
+  test('but naming an undeployed module DIRECTLY is still refused', () => {
+    // The operator asked for something that cannot happen; that is worth saying.
+    expect(() =>
+      planPause({
+        moduleId: 'technitium',
+        fleet: fleet({ technitium: 'IMPORTED' }),
+        cascade: false,
+      }),
+    ).toThrow(/never been deployed/);
+  });
+
+  test('an in-flight cascade member still refuses — it WILL be bound', () => {
+    // Distinct from undeployed: a module mid-deploy is on its way to being
+    // bound to the provider, so pausing around it would strand the transition.
+    expect(() =>
+      planPause({ moduleId: 'greenwave', fleet: fleet({ caddy: 'DEPLOYING' }), cascade: true }),
+    ).toThrow(/strand/);
+  });
+});