@celilo/cli 1.11.0 → 1.13.0

package/src/services/backup-metadata.ts

@@ -4,7 +4,7 @@
  */
 
 import { randomUUID } from 'node:crypto';
-import { and, desc, eq, gt, isNotNull, like } from 'drizzle-orm';
+import { and, desc, eq, gt, gte, isNotNull, like } from 'drizzle-orm';
 import { getDb } from '../db/client';
 import { type Backup, type BackupStatus, type BackupType, backups } from '../db/schema';
 import type { BackupHistory } from './backup-schedule';
@@ -140,21 +140,29 @@ export function getBackup(id: string): Backup | null {
 export function listBackups(options?: {
   moduleId?: string;
   limit?: number;
+  /**
+   * Only attempts started at or after this instant (epoch ms).
+   *
+   * A WINDOW rather than a bigger `limit`, because the two answer different
+   * questions and only one of them is stable. "The last 500 attempts" spans a
+   * week for a module failing hourly and a year for one succeeding daily, so a
+   * caller asking for a fixed period cannot express it as a count. The console
+   * needs a period: it draws days, and a row missing from its answer renders as
+   * a day on which nothing ran.
+   */
+  since?: number;
 }): Backup[] {
   const db = getDb();
   const limit = options?.limit ?? 20;
 
-  if (options?.moduleId) {
-    return db
-      .select()
-      .from(backups)
-      .where(eq(backups.moduleId, options.moduleId))
-      .orderBy(desc(backups.startedAt))
-      .limit(limit)
-      .all();
-  }
+  const filters = [
+    options?.moduleId ? eq(backups.moduleId, options.moduleId) : undefined,
+    options?.since !== undefined ? gte(backups.startedAt, new Date(options.since)) : undefined,
+  ].filter((clause) => clause !== undefined);
 
-  return db.select().from(backups).orderBy(desc(backups.startedAt)).limit(limit).all();
+  const query = db.select().from(backups);
+  const filtered = filters.length > 0 ? query.where(and(...filters)) : query;
+  return filtered.orderBy(desc(backups.startedAt)).limit(limit).all();
 }
 
 /**

package/src/services/consumer-cleanup.ts

@@ -48,6 +48,21 @@ export interface CleanupTarget {
   skip?: CleanupSkipReason;
 }
 
+/**
+ * The only part of a manifest this plan reads.
+ *
+ * Narrower than `ModuleManifest` on purpose. A full manifest is a large,
+ * strictly-validated shape, and the console reads manifests straight out of the
+ * `modules` table where they are stored as opaque JSON. Asking for the whole
+ * shape would force either a full re-validation on every poll or a cast, and a
+ * cast at a trust boundary is the thing that eventually bites. `ModuleManifest`
+ * satisfies this structurally, so existing callers are unaffected.
+ */
+export interface ConsumedCapabilities {
+  requires?: { capabilities?: readonly { name: string }[] };
+  optional?: { capabilities?: readonly { name: string }[] };
+}
+
 export interface ProviderRow {
   moduleId: string;
   capabilityName: string;
@@ -76,7 +91,7 @@ export interface ProviderState {
  */
 export function planConsumerCleanup(
   consumer: string,
-  manifest: ModuleManifest,
+  manifest: ConsumedCapabilities,
   providerRows: ProviderRow[],
   providerStates: ProviderState[],
 ): CleanupTarget[] {
@@ -112,16 +127,27 @@ export function planConsumerCleanup(
     .sort((a, b) => a.providerId.localeCompare(b.providerId));
 }
 
+/**
+ * Every (module, capability) provider row.
+ *
+ * Exported because the console's closure walk needs the same edge this plan
+ * does, and reads it for many modules rather than one. Two copies of this query
+ * would be two definitions of what a provider is.
+ */
+export function loadCapabilityProviderRows(db: DbClient): ProviderRow[] {
+  return db
+    .select({ moduleId: capabilities.moduleId, capabilityName: capabilities.capabilityName })
+    .from(capabilities)
+    .all();
+}
+
 /** Read the plan's inputs out of the DB. */
 export function loadConsumerCleanupPlan(
   consumer: string,
   manifest: ModuleManifest,
   db: DbClient,
 ): CleanupTarget[] {
-  const providerRows = db
-    .select({ moduleId: capabilities.moduleId, capabilityName: capabilities.capabilityName })
-    .from(capabilities)
-    .all();
+  const providerRows = loadCapabilityProviderRows(db);
   const providerStates = db
     .select({ moduleId: modules.id, state: modules.state })
     .from(modules)

package/src/services/instance-ops.test.ts

@@ -0,0 +1,302 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { existsSync } from 'node:fs';
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { createModuleDeployWorker, isInstanceRequestRefusedError } from '@celilo/capabilities';
+import { type DbClient, createDbClient } from '../db/client';
+import type { ModuleManifest } from '../manifest/schema';
+import {
+  createInstanceOps,
+  refuseUndeclaredNetworks,
+  validateInstanceConfig,
+} from './instance-ops';
+import { deriveInstanceModuleId } from './module-instances';
+
+const manifest = (over: Record<string, unknown> = {}): ModuleManifest =>
+  ({
+    celilo_contract: '1.0',
+    id: 'lab',
+    name: 'BYOI Lab',
+    version: '1.0.0',
+    requires: { capabilities: [] },
+    provides: { capabilities: [] },
+    variables: { owns: [], imports: [] },
+    ...over,
+  }) as unknown as ModuleManifest;
+
+const ownedVar = (over: Record<string, unknown>) => ({
+  name: 'ssh_key',
+  type: 'string',
+  required: true,
+  source: 'user',
+  description: '',
+  ...over,
+});
+
+describe('validateInstanceConfig', () => {
+  test('accepts a config supplying every required user variable', () => {
+    const m = manifest({ variables: { owns: [ownedVar({})], imports: [] } });
+    expect(validateInstanceConfig(m, { ssh_key: 'ssh-ed25519 AAAA' })).toEqual([]);
+  });
+
+  test('reports a required variable that was not supplied', () => {
+    const m = manifest({ variables: { owns: [ownedVar({})], imports: [] } });
+    const problems = validateInstanceConfig(m, {});
+    expect(problems).toHaveLength(1);
+    expect(problems[0]).toContain("'ssh_key' is required");
+  });
+
+  test('a required variable with a default need not be supplied', () => {
+    const m = manifest({
+      variables: { owns: [ownedVar({ default: 'ssh-ed25519 DEFAULT' })], imports: [] },
+    });
+    expect(validateInstanceConfig(m, {})).toEqual([]);
+  });
+
+  test('refuses a key the submodule does not declare, and lists what it does', () => {
+    const m = manifest({ variables: { owns: [ownedVar({})], imports: [] } });
+    const problems = validateInstanceConfig(m, { ssh_key: 'k', nonsense: 1 });
+    expect(problems).toHaveLength(1);
+    expect(problems[0]).toContain("'nonsense' is not declared");
+    expect(problems[0]).toContain('ssh_key');
+  });
+
+  // A derived value silently overwritten at generate time looks like celilo
+  // ignoring the request, so it is refused rather than accepted and discarded.
+  test('refuses a declared variable celilo derives rather than the caller supplying', () => {
+    const m = manifest({
+      variables: {
+        owns: [ownedVar({}), ownedVar({ name: 'container_ip', source: 'infrastructure' })],
+        imports: [],
+      },
+    });
+    const problems = validateInstanceConfig(m, { ssh_key: 'k', container_ip: '10.0.0.5' });
+    expect(problems).toHaveLength(1);
+    expect(problems[0]).toContain('celilo derives it');
+  });
+
+  // Four failed instantiations to learn four missing keys is the experience
+  // this avoids.
+  test('reports every problem at once', () => {
+    const m = manifest({
+      variables: { owns: [ownedVar({}), ownedVar({ name: 'owner_sub' })], imports: [] },
+    });
+    expect(validateInstanceConfig(m, { junk: 1 })).toHaveLength(3);
+  });
+});
+
+describe('refuseUndeclaredNetworks', () => {
+  test('passes when every insisted network is declared', () => {
+    const m = manifest({ requires: { capabilities: [], networks: [{ name: 'quarantine' }] } });
+    expect(refuseUndeclaredNetworks(m, ['dmz', 'quarantine'])).toBeNull();
+  });
+
+  test('refuses and names the missing network', () => {
+    const m = manifest({ requires: { capabilities: [], networks: [{ name: 'quarantine' }] } });
+    const refusal = refuseUndeclaredNetworks(m, ['dmz']);
+    expect(refusal).toContain('quarantine');
+    expect(refusal).toContain('never creates a network');
+  });
+
+  test('a submodule insisting on nothing passes', () => {
+    expect(refuseUndeclaredNetworks(manifest(), [])).toBeNull();
+  });
+});
+
+describe('createInstanceOps against a real database', () => {
+  let root: string;
+  let db: DbClient;
+  let parentPath: string;
+
+  const SUBMODULE_YAML = `
+celilo_contract: "1.0"
+id: lab
+name: BYOI Lab
+version: 1.0.0
+variables:
+  owns:
+    - name: ssh_key
+      type: string
+      required: true
+      source: user
+      description: Public key baked into authorized_keys
+`;
+
+  async function setup(opts: { submoduleYaml?: string; declaredNetworks?: string[] } = {}) {
+    parentPath = join(root, 'modules', 'byoi');
+    const submodulePath = join(parentPath, 'submodules', 'lab');
+    await mkdir(join(submodulePath, 'ansible'), { recursive: true });
+    await writeFile(join(parentPath, 'manifest.yml'), 'id: byoi\n');
+    await writeFile(join(submodulePath, 'manifest.yml'), opts.submoduleYaml ?? SUBMODULE_YAML);
+
+    db.$client.run(
+      "INSERT INTO modules (id,name,version,manifest_data,source_path) VALUES ('byoi','byoi','1.0.0',?,?)",
+      [JSON.stringify({ id: 'byoi', submodules: ['lab'] }), parentPath],
+    );
+
+    return createInstanceOps({
+      parentId: 'byoi',
+      db,
+      declaredNetworks: opts.declaredNetworks ?? ['quarantine'],
+    });
+  }
+
+  const worker = (ops: ReturnType<typeof createInstanceOps>) =>
+    createModuleDeployWorker({ moduleId: 'byoi', ops });
+
+  beforeEach(async () => {
+    root = await mkdtemp(join(tmpdir(), 'celilo-instance-ops-'));
+    db = createDbClient({ path: join(root, 'celilo.db') });
+  });
+
+  afterEach(async () => {
+    db.$client.close();
+    await rm(root, { recursive: true, force: true });
+  });
+
+  test('records an instance, its config, and its symlink farm', async () => {
+    const ops = await setup();
+    const moduleId = await ops.create({
+      submodule: 'lab',
+      instanceKey: 'sub-abc',
+      config: { ssh_key: 'ssh-ed25519 AAAA' },
+      label: 'Sally',
+    });
+
+    expect(moduleId).toBe(deriveInstanceModuleId('byoi', 'lab', 'sub-abc'));
+
+    // A flat peer of every other module, never nested under the parent.
+    const instancePath = join(dirname(parentPath), moduleId);
+    expect(dirname(instancePath)).toBe(dirname(parentPath));
+    expect(existsSync(join(instancePath, 'generated'))).toBe(true);
+    expect(existsSync(join(instancePath, 'ansible'))).toBe(true);
+
+    const row = db.$client
+      .query('SELECT state, label, parent_id, submodule FROM module_instances')
+      .get() as Record<string, unknown>;
+    expect(row).toMatchObject({
+      state: 'pending',
+      label: 'Sally',
+      parent_id: 'byoi',
+      submodule: 'lab',
+    });
+
+    const config = db.$client
+      .query("SELECT value_json FROM module_configs WHERE module_id = ? AND key = 'ssh_key'")
+      .get(moduleId) as { value_json: string };
+    expect(JSON.parse(config.value_json)).toBe('ssh-ed25519 AAAA');
+  });
+
+  // A reconcile loop that fires twice must not produce a second system.
+  test('is idempotent on the caller key and repairs the farm on a repeat', async () => {
+    const ops = await setup();
+    const first = await ops.create({
+      submodule: 'lab',
+      instanceKey: 'sub-abc',
+      config: { ssh_key: 'k1' },
+    });
+
+    const instancePath = join(dirname(parentPath), first);
+    await rm(join(instancePath, 'ansible'), { force: true });
+
+    const second = await ops.create({
+      submodule: 'lab',
+      instanceKey: 'sub-abc',
+      config: { ssh_key: 'k2' },
+    });
+
+    expect(second).toBe(first);
+    expect(
+      (db.$client.query('SELECT count(*) n FROM module_instances').get() as { n: number }).n,
+    ).toBe(1);
+    // A retry REPAIRS rather than merely not-breaking.
+    expect(existsSync(join(instancePath, 'ansible'))).toBe(true);
+  });
+
+  test('refuses invalid config before writing anything', async () => {
+    const ops = await setup();
+
+    let caught: unknown;
+    try {
+      await ops.create({ submodule: 'lab', instanceKey: 'sub-abc', config: {} });
+    } catch (error) {
+      caught = error;
+    }
+    // Duck-typed, because the error crosses the identity boundary a module's
+    // bundled copy of @celilo/capabilities creates (celilo#173).
+    expect(isInstanceRequestRefusedError(caught)).toBe(true);
+
+    expect(
+      (db.$client.query('SELECT count(*) n FROM module_instances').get() as { n: number }).n,
+    ).toBe(0);
+  });
+
+  // D8, and at the REQUEST rather than three minutes into a terraform apply.
+  test('refuses an undeclared network before provisioning', async () => {
+    const ops = await setup({
+      submoduleYaml: `${SUBMODULE_YAML}
+requires:
+  networks:
+    - name: quarantine
+`,
+      declaredNetworks: ['dmz'],
+    });
+
+    await expect(
+      ops.create({ submodule: 'lab', instanceKey: 'k', config: { ssh_key: 'x' } }),
+    ).rejects.toThrow(/quarantine/);
+  });
+
+  test('a framework refusal reaches the caller as accepted:false, not a throw', async () => {
+    const capability = worker(await setup());
+
+    const result = await capability.instantiate({
+      submodule: 'lab',
+      instanceKey: 'sub-abc',
+      config: {},
+    });
+
+    expect(result.accepted).toBe(false);
+    if (!result.accepted) expect(result.reason).toContain("'ssh_key' is required");
+  });
+
+  test('destroy marks the instance and reports its id', async () => {
+    const ops = await setup();
+    const moduleId = await ops.create({
+      submodule: 'lab',
+      instanceKey: 'sub-abc',
+      config: { ssh_key: 'k' },
+    });
+
+    expect(await ops.markForDestruction('lab', 'sub-abc')).toBe(moduleId);
+
+    const row = db.$client.query('SELECT state FROM module_instances').get() as { state: string };
+    expect(row.state).toBe('destroying');
+  });
+
+  test('destroying something that was never there reports null', async () => {
+    const ops = await setup();
+    expect(await ops.markForDestruction('lab', 'never')).toBeNull();
+  });
+
+  test('list reports what was recorded, scoped to the caller', async () => {
+    const ops = await setup();
+    await ops.create({ submodule: 'lab', instanceKey: 'a', config: { ssh_key: 'k' } });
+    await ops.create({ submodule: 'lab', instanceKey: 'b', config: { ssh_key: 'k' } });
+
+    const listed = await ops.list();
+
+    expect(listed.map((i) => i.instanceKey).sort()).toEqual(['a', 'b']);
+    expect(listed.every((i) => i.state === 'pending')).toBe(true);
+    expect(listed.every((i) => i.failureReason === null)).toBe(true);
+  });
+
+  test('declaredSubmodules reads the parent manifest, so authority is not supplied', async () => {
+    const ops = await setup();
+    expect(ops.declaredSubmodules()).toEqual(['lab']);
+
+    const result = await worker(ops).instantiate({ submodule: 'not-mine', instanceKey: 'k' });
+    expect(result.accepted).toBe(false);
+  });
+});