@celilo/cli 1.8.0 → 1.9.0

package/src/hooks/broker.test.ts

@@ -0,0 +1,152 @@
+/**
+ * The broker: the capability surface crossing a process boundary.
+ *
+ * The claim under test is design D2's — that one generic proxy covers all
+ * twelve capabilities because every hook-facing method is already
+ * `(request: JSON) => Promise<JSON>`. So these tests fix the SHAPES a call can
+ * take (returns, throws, a structured throw the framework reads, an absent
+ * optional method, an unknown method) and say nothing about any particular
+ * capability. A per-method suite would prove the same thing thirty-seven times
+ * and go stale the moment a provider gained a method.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { execSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { MissingProviderInputError } from '@celilo/capabilities';
+import { capabilityShape } from './broker';
+import { executeHookScript } from './executor';
+import { createCapturingLogger } from './logger';
+import type { HookContext } from './types';
+
+const FIXTURES = join(__dirname, 'test-fixtures');
+
+function demoCapabilities(): Record<string, unknown> {
+  return {
+    demo: {
+      providerModuleId: 'demo-provider',
+      version: '1.0.0',
+      echo: async (request: unknown) => ({ echoed: request }),
+      returnsNothing: async () => undefined,
+      boom: async () => {
+        throw new Error('plain failure');
+      },
+      missingInput: async () => {
+        throw new MissingProviderInputError({
+          providerModuleId: 'caddy',
+          ensureId: 'hostnames',
+          value: 'foo.example.com',
+          humanContext: 'so the route resolves',
+        });
+      },
+    },
+  };
+}
+
+async function runCapabilityHook(): Promise<Record<string, unknown>> {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-broker-'));
+  try {
+    const context: HookContext = {
+      config: {},
+      secrets: {},
+      systems: [],
+      logger: createCapturingLogger().logger,
+      debug: false,
+      screenshotDir: dir,
+      capabilities: demoCapabilities(),
+    };
+    return await executeHookScript(
+      join(FIXTURES, 'capability-calling-hook.ts'),
+      context,
+      30_000,
+      30_000,
+    );
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+describe('capabilityShape', () => {
+  test('splits functions from data', () => {
+    const shape = capabilityShape(demoCapabilities());
+    expect(shape.demo.methods.sort()).toEqual(['boom', 'echo', 'missingInput', 'returnsNothing']);
+    expect(shape.demo.data).toEqual({ providerModuleId: 'demo-provider', version: '1.0.0' });
+  });
+
+  test('an unimplemented optional method is simply absent', () => {
+    // Not "present and throwing". `if (cap.registerTrustedSource)` is real
+    // code in the wireguard module and it has to keep answering correctly.
+    expect(capabilityShape(demoCapabilities()).demo.methods).not.toContain('sometimesAbsent');
+  });
+
+  test('symbol keys are dropped — they cannot cross JSON', () => {
+    const brand = Symbol('brand');
+    const shape = capabilityShape({ demo: { [brand]: 'x', ok: async () => 1 } });
+    expect(shape.demo.methods).toEqual(['ok']);
+    expect(shape.demo.data).toEqual({});
+  });
+
+  test('a value JSON cannot carry is left out of data rather than corrupted', () => {
+    const shape = capabilityShape({
+      demo: { nan: Number.NaN, inf: Number.POSITIVE_INFINITY, n: 1 },
+    });
+    expect(shape.demo.data).toEqual({ n: 1 });
+  });
+
+  test('a non-object capability entry is skipped, not crashed on', () => {
+    expect(capabilityShape({ broken: null, alsoBroken: 'string' })).toEqual({});
+  });
+});
+
+describe('capability calls across the boundary', () => {
+  test('every call shape survives the round trip', async () => {
+    const outputs = await runCapabilityHook();
+
+    expect(outputs.providerModuleId).toBe('demo-provider');
+    expect(outputs.version).toBe('1.0.0');
+    expect(outputs.optionalMethodAbsent).toBe(true);
+    expect(outputs.returned).toEqual({ echoed: { x: 1, nested: { y: [2, 3] } } });
+    expect(outputs.undefinedBecomesNull).toBeNull();
+
+    expect(outputs.plainThrow).toEqual({
+      isError: true,
+      name: 'Error',
+      message: 'plain failure',
+      hasStack: true,
+    });
+
+    // The one error the framework READS rather than displays. Lose these four
+    // fields and the cross-module ensure interview never runs — the deploy
+    // fails with a message where it should have asked a question.
+    expect(outputs.missingProviderInput).toEqual({
+      recognised: true,
+      providerModuleId: 'caddy',
+      ensureId: 'hostnames',
+      value: 'foo.example.com',
+      humanContext: 'so the route resolves',
+    });
+
+    // A method that is not in the shape is not on the proxy, so this fails on
+    // this side and never reaches the broker — the same TypeError a hook gets
+    // in-process today. The broker's own "no such method" guard is for a
+    // shape/map disagreement, which is a skew bug and not this path.
+    expect(outputs.unknownMethod).toContain('not a function');
+  }, 30_000);
+
+  test('the hook ran in a process of its own and left none behind', async () => {
+    await runCapabilityHook();
+
+    // By parent pid, not by name: the hook runner is a direct child of this
+    // process, and matching on the script name instead picks up whatever shell
+    // happens to have the filename in its own command line.
+    const orphans = execSync('ps -Ao ppid=,args=', { encoding: 'utf-8' })
+      .split('\n')
+      .filter(
+        (line) => Number.parseInt(line.trim(), 10) === process.pid && line.includes('hook-runner'),
+      );
+
+    expect(orphans).toEqual([]);
+  }, 30_000);
+});

package/src/hooks/broker.ts

@@ -0,0 +1,307 @@
+/**
+ * The hook broker.
+ *
+ * celilo keeps the database, the master key and the live capability objects.
+ * The hook runs in its own process and reaches them only by asking. This is
+ * the answering half; `hook-runner.ts` is the asking half.
+ *
+ * **It does not know what a capability is** (design D2). At handshake it sends
+ * a shape descriptor built by walking the object `loadCapabilityFunctions`
+ * already returns, and thereafter it dispatches `call` frames against that
+ * same object by name. That works because every hook-facing capability method
+ * is already `(request: JSON) => Promise<JSON>` — measured: twelve
+ * capabilities, thirty-seven methods, no callbacks, no streams, no handles —
+ * and because `wrapWithLogging` already treats a capability as an opaque table
+ * of async methods keyed by name. This is that walk with a process in the
+ * middle.
+ *
+ * Execution function (Rule 10.1) — owns the socket and performs the calls.
+ */
+
+import { mkdtempSync, rmSync } from 'node:fs';
+import { type Server, type Socket, createServer } from 'node:net';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  type CapabilityShape,
+  type HookError,
+  createLineReader,
+  encodeFrame,
+  parseChildFrame,
+  serializeError,
+  versionMismatch,
+} from './hook-protocol';
+import type { HookLogger } from './types';
+
+/**
+ * How long to wait after the child exits for its last frame to arrive. Short,
+ * because by this point the writer is already gone and the bytes are either in
+ * the buffer or they are never coming.
+ */
+const DRAIN_GRACE_MS = 1_000;
+
+/** What the hook returned, or what it threw. */
+export type HookOutcome =
+  | { ok: true; outputs: Record<string, unknown> }
+  | { ok: false; error: HookError };
+
+export interface BrokerOptions {
+  /** The live capability map, exactly as `loadCapabilityFunctions` built it. */
+  capabilities: Record<string, unknown>;
+  /** The serialisable half of `HookContext` — no `logger`, no `capabilities`. */
+  context: Record<string, unknown>;
+  /** Absolute path of the hook script, sent to the child rather than argv'd. */
+  scriptPath: string;
+  /** Where the hook's own log lines go. */
+  logger: HookLogger;
+  /** Called on every frame, so the idle timer measures the whole channel. */
+  onActivity: () => void;
+}
+
+export interface Broker {
+  /** Unix socket path, handed to the child in its environment. */
+  socketPath: string;
+  /**
+   * Resolves once the child's connection has closed, so its last frame has
+   * certainly been delivered.
+   *
+   * The child writes its terminal frame and then exits, and process exit is
+   * what the executor waits on — but the two are different channels. The exit
+   * can be observed while the frame is still sitting in the kernel buffer,
+   * unread, which would look exactly like a hook that exited without
+   * returning a result. Bounded, so a socket that never closes cannot hang
+   * celilo waiting for a process that has already gone.
+   */
+  drained(): Promise<void>;
+  /**
+   * The hook's terminal frame, or `undefined` if it never sent one — which is
+   * what a crash, a `process.exit` or a kill looks like from here. Read after
+   * the child has exited, so there is no race to lose.
+   */
+  outcome(): HookOutcome | undefined;
+  /** Every protocol-level complaint, for the error message when one matters. */
+  faults(): string[];
+  /** Refuse further calls. Idempotent. Called on kill and again on cleanup. */
+  stop(): void;
+  /** Close the socket and remove its directory. */
+  close(): void;
+}
+
+/**
+ * Walk a capability map into the descriptor the child rebuilds proxies from.
+ *
+ * Function-valued keys become `methods`; everything else is copied into `data`,
+ * which is where `stampProvider` puts `providerModuleId` — a hook reads it to
+ * name the provider that could not supply what it asked for.
+ *
+ * Own string keys only, the way `wrapWithLogging` walks. Every capability the
+ * loader builds is a plain object (a spread, an `Object.assign`, or
+ * `Object.create` plus own properties), so a prototype walk would find nothing
+ * a hook can call today — and if that ever stopped being true, auto-logging
+ * would break in the same breath, which is a louder signal than this would be.
+ * Symbol keys cannot cross JSON and are dropped.
+ *
+ * Planning function (Rule 10.4) — pure, so the descriptor is testable without
+ * a socket.
+ */
+export function capabilityShape(
+  capabilities: Record<string, unknown>,
+): Record<string, CapabilityShape> {
+  const shape: Record<string, CapabilityShape> = {};
+
+  for (const [name, capability] of Object.entries(capabilities)) {
+    if (!capability || typeof capability !== 'object') continue;
+
+    const methods: string[] = [];
+    const data: Record<string, unknown> = {};
+
+    for (const key of Reflect.ownKeys(capability)) {
+      if (typeof key !== 'string') continue;
+      const value = (capability as Record<string, unknown>)[key];
+      if (typeof value === 'function') {
+        methods.push(key);
+      } else if (isJsonSafe(value)) {
+        data[key] = value;
+      }
+    }
+
+    shape[name] = { methods, data };
+  }
+
+  return shape;
+}
+
+/** Everything `JSON.stringify` round-trips without inventing or losing a value. */
+function isJsonSafe(value: unknown): boolean {
+  if (value === null) return true;
+  const type = typeof value;
+  if (type === 'string' || type === 'boolean') return true;
+  if (type === 'number') return Number.isFinite(value as number);
+  if (type !== 'object') return false;
+  try {
+    return JSON.parse(JSON.stringify(value)) !== undefined;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Bind a Unix socket in a fresh per-run directory and answer one hook.
+ *
+ * The socket, not stdout: seventeen module script files spawn subprocesses,
+ * and a grandchild writing to fd 1 would corrupt the frame stream (design D3).
+ * The directory is short-named because `sun_path` is 104 bytes on macOS and
+ * the platform temp directory already spends half of it.
+ */
+export async function startBroker(options: BrokerOptions): Promise<Broker> {
+  const directory = mkdtempSync(join(tmpdir(), 'celilo-hook-'));
+  const socketPath = join(directory, 's');
+
+  let outcome: HookOutcome | undefined;
+  let stopped = false;
+  const faults: string[] = [];
+  let connection: Socket | undefined;
+  // Already resolved: no connection means nothing left to deliver.
+  let connectionClosed: Promise<void> = Promise.resolve();
+
+  const server: Server = createServer((socket) => {
+    if (connection) {
+      // One hook, one connection. A second is either a bug or a hook trying
+      // to hold the channel open past its own run.
+      faults.push('a second connection to the hook socket was refused');
+      socket.destroy();
+      return;
+    }
+    connection = socket;
+    connectionClosed = new Promise<void>((resolve) => socket.once('close', () => resolve()));
+    socket.setEncoding('utf-8');
+
+    const send = (frame: Parameters<typeof encodeFrame>[0]) => {
+      if (!socket.destroyed) socket.write(encodeFrame(frame));
+    };
+
+    const feed = createLineReader((line) => {
+      options.onActivity();
+
+      const parsed = parseChildFrame(line);
+      if (!parsed.ok) {
+        faults.push(`malformed frame from the hook: ${parsed.error}`);
+        return;
+      }
+
+      const frame = parsed.frame;
+      switch (frame.type) {
+        case 'ready': {
+          const mismatch = versionMismatch(frame.protocolVersion, 'the hook runner');
+          if (mismatch) {
+            faults.push(mismatch);
+            socket.destroy();
+            return;
+          }
+          send({
+            type: 'context',
+            protocolVersion: 1,
+            scriptPath: options.scriptPath,
+            context: options.context,
+          });
+          send({ type: 'capabilities', shape: capabilityShape(options.capabilities) });
+          return;
+        }
+
+        case 'log':
+          options.logger[frame.level](frame.message);
+          return;
+
+        case 'call':
+          void dispatch(frame.id, frame.capability, frame.method, frame.args, send);
+          return;
+
+        case 'result':
+          outcome = { ok: true, outputs: frame.outputs };
+          return;
+
+        case 'throw':
+          outcome = { ok: false, error: frame.error };
+          return;
+      }
+    });
+
+    socket.on('data', feed);
+    socket.on('error', (error) => faults.push(`hook socket error: ${error.message}`));
+  });
+
+  async function dispatch(
+    id: string,
+    capabilityName: string,
+    method: string,
+    args: unknown[],
+    send: (frame: Parameters<typeof encodeFrame>[0]) => void,
+  ): Promise<void> {
+    if (stopped) {
+      // The run is over — killed at its timeout, most likely. Refusing here is
+      // what makes the kill real: the whole failure celilo#1003 describes is a
+      // hook still registering DNS records after celilo moved on.
+      send({
+        type: 'throw',
+        id,
+        error: {
+          name: 'Error',
+          message: `Hook run has ended; refusing ${capabilityName}.${method}.`,
+        },
+      });
+      return;
+    }
+
+    const capability = options.capabilities[capabilityName];
+    const fn =
+      capability && typeof capability === 'object'
+        ? (capability as Record<string, unknown>)[method]
+        : undefined;
+
+    if (typeof fn !== 'function') {
+      send({
+        type: 'throw',
+        id,
+        error: {
+          name: 'TypeError',
+          message: `Capability '${capabilityName}' has no method '${method}'.`,
+        },
+      });
+      return;
+    }
+
+    try {
+      const value = await (fn as (...a: unknown[]) => unknown).apply(capability, args);
+      options.onActivity();
+      send({ type: 'return', id, value: value === undefined ? null : value });
+    } catch (error) {
+      options.onActivity();
+      send({ type: 'throw', id, error: serializeError(error) });
+    }
+  }
+
+  await new Promise<void>((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(socketPath, resolve);
+  });
+
+  return {
+    socketPath,
+    drained: () =>
+      Promise.race([
+        connectionClosed,
+        new Promise<void>((resolve) => setTimeout(resolve, DRAIN_GRACE_MS).unref()),
+      ]),
+    outcome: () => outcome,
+    faults: () => [...faults],
+    stop: () => {
+      stopped = true;
+    },
+    close: () => {
+      stopped = true;
+      connection?.destroy();
+      server.close();
+      rmSync(directory, { recursive: true, force: true });
+    },
+  };
+}

package/src/hooks/capability-loader-bindings.test.ts

@@ -0,0 +1,163 @@
+/**
+ * The recurrence gate for celilo#1072.
+ *
+ * celilo could name every provider a module MIGHT be bound to and none of the
+ * ones it IS. The live case: `tango-nexus` declares four optional capabilities
+ * (external_web, public_web, source_forge, registry_publish), is deployed
+ * against one off-fleet host, and nothing recorded which of the four was real.
+ *
+ * This models that shape with two capabilities the loader injects and one the
+ * consumer calls. The gate is the NEGATIVE half: the capability that was
+ * injected and never used must report zero bindings. Anything that records at
+ * resolution time — which is where the obvious fix goes — passes the positive
+ * assertion and fails this one.
+ *
+ * Two different flavours of over-recording are caught here, and the second is
+ * easy to miss when reading this file as being about one thing. The first is a
+ * consumer credited with a capability it never called. The second is the
+ * self-binding case: `dhcp-provider` running its OWN hook is also handed
+ * `notification` from `notify-provider`, because the loader hands over
+ * everything registered. Resolution-time recording books that as a binding
+ * `dhcp-provider` never made, so the "not bound to itself" test fails for a
+ * reason that has nothing to do with self-binding.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { HookLogger } from '@celilo/capabilities';
+import type { DbClient } from '../db/client';
+import { listCapabilityBindings } from '../services/capability-bindings';
+import { upsertModuleConfig } from '../services/module-config';
+import { cleanupTestDatabase, setupTestDatabase } from '../test-utils/database';
+import { loadCapabilityFunctions } from './capability-loader';
+
+const noopLogger: HookLogger = {
+  info() {},
+  warn() {},
+  error() {},
+  success() {},
+};
+
+const DHCP_MODULE = `
+export default function createDhcpServer(context) {
+  return {
+    async setDnsServers() {},
+    async getDnsServers() { return [context.config.marker]; },
+    async setDomainName() {},
+    async getDomainName() { return context.config.marker; },
+  };
+}
+`;
+
+const NOTIFICATION_MODULE = `
+export default function createNotification() {
+  return {
+    async send() { return { delivered: true }; },
+  };
+}
+`;
+
+function installProvider(
+  db: DbClient,
+  tempDir: string,
+  moduleId: string,
+  script: string,
+  source: string,
+  capabilityName: string,
+): void {
+  const modulePath = join(tempDir, moduleId);
+  const scriptsDir = join(modulePath, 'scripts');
+  mkdirSync(scriptsDir, { recursive: true });
+  writeFileSync(join(scriptsDir, script), source);
+  db.$client.run(
+    `INSERT INTO modules (id, name, version, source_path, manifest_data) VALUES ('${moduleId}', '${moduleId}', '1.0.0', '${modulePath}', '{}')`,
+  );
+  upsertModuleConfig(db, moduleId, 'marker', moduleId);
+  db.$client.run(
+    `INSERT INTO capabilities (module_id, capability_name, version, data, zones, registered_at) VALUES ('${moduleId}', '${capabilityName}', '1.0.0', '{}', NULL, unixepoch())`,
+  );
+}
+
+describe('capability bindings recorded from the loader', () => {
+  let db: DbClient;
+  let tempDir: string;
+
+  beforeEach(async () => {
+    db = await setupTestDatabase();
+    // `setupTestDatabase` does not run `PRAGMA foreign_keys = ON` and
+    // `db/client.ts:70` does, so every cascade in the schema is enforced in
+    // production and off in the suite (celilo#1074). Enabled here so the
+    // cascade assertion below measures the real behaviour. Delete this line
+    // when #1074 lands.
+    db.$client.run('PRAGMA foreign_keys = ON');
+    tempDir = join(tmpdir(), `celilo-binding-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(tempDir, { recursive: true });
+
+    installProvider(
+      db,
+      tempDir,
+      'dhcp-provider',
+      'dhcp-server-functions.ts',
+      DHCP_MODULE,
+      'dhcp_server',
+    );
+    installProvider(
+      db,
+      tempDir,
+      'notify-provider',
+      'notification.ts',
+      NOTIFICATION_MODULE,
+      'notification',
+    );
+    db.$client.run(
+      `INSERT INTO modules (id, name, version, source_path, manifest_data) VALUES ('consumer', 'consumer', '1.0.0', '${tempDir}', '{}')`,
+    );
+  });
+
+  afterEach(async () => {
+    await cleanupTestDatabase(db);
+  });
+
+  test('an injected capability the consumer never calls reports zero bindings', async () => {
+    const capabilities = await loadCapabilityFunctions('consumer', db, noopLogger);
+
+    // Both are injected — the loader hands over every registered capability
+    // regardless of what the consumer declared.
+    expect(capabilities.dhcp_server).toBeTruthy();
+    expect(capabilities.notification).toBeTruthy();
+
+    await (capabilities.dhcp_server as { getDomainName(): Promise<string> }).getDomainName();
+
+    const bindings = listCapabilityBindings(db, 'consumer');
+    expect(bindings.map((b) => b.capabilityName)).toEqual(['dhcp_server']);
+    expect(bindings[0].providerModuleId).toBe('dhcp-provider');
+  });
+
+  test('a redeploy re-asserts the binding rather than duplicating it', async () => {
+    for (let i = 0; i < 3; i++) {
+      const capabilities = await loadCapabilityFunctions('consumer', db, noopLogger);
+      await (capabilities.dhcp_server as { getDomainName(): Promise<string> }).getDomainName();
+    }
+
+    expect(listCapabilityBindings(db, 'consumer')).toHaveLength(1);
+  });
+
+  test('the binding dies with the consumer', async () => {
+    const capabilities = await loadCapabilityFunctions('consumer', db, noopLogger);
+    await (capabilities.dhcp_server as { getDomainName(): Promise<string> }).getDomainName();
+    expect(listCapabilityBindings(db, 'consumer')).toHaveLength(1);
+
+    db.$client.run(`DELETE FROM modules WHERE id = 'consumer'`);
+
+    expect(listCapabilityBindings(db, 'consumer')).toHaveLength(0);
+  });
+
+  test('a provider consuming its own capability is not bound to itself', async () => {
+    const capabilities = await loadCapabilityFunctions('dhcp-provider', db, noopLogger);
+    await (capabilities.dhcp_server as { getDomainName(): Promise<string> }).getDomainName();
+
+    expect(listCapabilityBindings(db, 'dhcp-provider')).toHaveLength(0);
+  });
+});