@celilo/cli 1.8.0 → 1.9.0

package/src/hooks/hook-protocol.ts

@@ -0,0 +1,275 @@
+/**
+ * The hook process boundary's wire protocol.
+ *
+ * A hook runs in its own `bun` process and reaches celilo — the database, the
+ * master key, the live capability objects — only through these frames. See
+ * `openspec/changes/hook-process-boundary/design.md`, D2 and D3.
+ *
+ * Newline-delimited JSON over a Unix socket, Zod-validated, in two
+ * discriminated unions. Modelled directly on `packages/core/src/protocol.ts`,
+ * which does the same job for the remote CLI.
+ *
+ * **Why a socket and not stdout.** Seventeen module script files spawn
+ * subprocesses. A grandchild writing raw bytes to fd 1 would corrupt the frame
+ * stream and no JS-level capture prevents it. The remote CLI gets away with
+ * stdout because it *translates* every unrecognised line into a log message;
+ * here the same line could be half a frame. The child's stdout and stderr stay
+ * exactly what they are — human output the parent forwards to the logger.
+ *
+ * **Where this lives.** Both ends are framework code: the broker in
+ * `broker.ts` and the runner shim in `hook-runner.ts`, both under
+ * `apps/celilo/src/hooks/`. Neither side imports the other, so the protocol
+ * needs no package of its own and `@celilo/capabilities` — which every MODULE
+ * imports — stays out of it. A hook script never sees these types.
+ */
+
+import { z } from 'zod';
+
+export const HOOK_PROTOCOL_VERSION = 1;
+
+/** Environment variable carrying the broker's socket path to the child. */
+export const HOOK_SOCKET_ENV = 'CELILO_HOOK_SOCKET';
+/** Environment variable carrying the parent's protocol version to the child. */
+export const HOOK_PROTOCOL_VERSION_ENV = 'CELILO_HOOK_PROTOCOL_VERSION';
+
+/**
+ * An error crossing the boundary.
+ *
+ * `fields` is what carries `MissingProviderInputError`'s `providerModuleId` /
+ * `ensureId` / `value` / `humanContext` (design D6). The framework READS that
+ * error rather than displaying it — `invokeHook` inspects it to drive the
+ * cross-module ensure interview — so those four fields have to survive the
+ * round trip intact. They do, because `isMissingProviderInputError` is
+ * duck-typed rather than `instanceof`: it was written that way for a module's
+ * bundled copy of `@celilo/capabilities` (celilo#173), and a process is one
+ * more of the same boundary.
+ */
+export const HookErrorSchema = z.object({
+  name: z.string(),
+  message: z.string(),
+  stack: z.string().optional(),
+  fields: z.record(z.unknown()).optional(),
+});
+export type HookError = z.infer<typeof HookErrorSchema>;
+
+// ── child → parent ────────────────────────────────────────────────────────
+
+/** Handshake. The parent checks the version and refuses a mismatch. */
+export const ReadyFrameSchema = z.object({
+  type: z.literal('ready'),
+  protocolVersion: z.number().int(),
+});
+
+/** A capability method call, correlated with its `return`/`throw` by `id`. */
+export const CallFrameSchema = z.object({
+  type: z.literal('call'),
+  id: z.string(),
+  capability: z.string(),
+  method: z.string(),
+  args: z.array(z.unknown()),
+});
+
+/** One `ctx.logger` call. Fire and forget — the hook does not wait on it. */
+export const LogFrameSchema = z.object({
+  type: z.literal('log'),
+  level: z.enum(['info', 'warn', 'error', 'success']),
+  message: z.string(),
+});
+
+/** The hook returned. Terminal. */
+export const ResultFrameSchema = z.object({
+  type: z.literal('result'),
+  outputs: z.record(z.unknown()),
+});
+
+/** The hook threw. Terminal. Distinct from a `throw` answering one `call`. */
+export const HookThrewFrameSchema = z.object({
+  type: z.literal('throw'),
+  error: HookErrorSchema,
+});
+
+export const ChildFrameSchema = z.discriminatedUnion('type', [
+  ReadyFrameSchema,
+  CallFrameSchema,
+  LogFrameSchema,
+  ResultFrameSchema,
+  HookThrewFrameSchema,
+]);
+export type ChildFrame = z.infer<typeof ChildFrameSchema>;
+
+// ── parent → child ────────────────────────────────────────────────────────
+
+/**
+ * Everything in `HookContext` that is plain data: the hook's inputs, `config`,
+ * `secrets`, `systems`, `debug`, `screenshotDir`. Not `logger` and not
+ * `capabilities` — the shim rebuilds both from frames.
+ */
+export const ContextFrameSchema = z.object({
+  type: z.literal('context'),
+  protocolVersion: z.number().int(),
+  scriptPath: z.string(),
+  context: z.record(z.unknown()),
+});
+
+/**
+ * The capability shape descriptor (design D2).
+ *
+ * The broker does not know what a capability is. It walks the object
+ * `loadCapabilityFunctions` returns the way `wrapWithLogging` does:
+ * function-valued keys become `methods`, everything else is copied into
+ * `data` — which is where `stampProvider`'s `providerModuleId` lives, and a
+ * hook reads it to name the provider in an error.
+ *
+ * An optional method the provider did not implement is simply absent from
+ * `methods`, so it is absent on the proxy, so `if (cap.registerTrustedSource)`
+ * keeps working with no special case.
+ */
+export const CapabilitiesFrameSchema = z.object({
+  type: z.literal('capabilities'),
+  shape: z.record(
+    z.object({
+      methods: z.array(z.string()),
+      data: z.record(z.unknown()),
+    }),
+  ),
+});
+
+/** A capability call returned. */
+export const ReturnFrameSchema = z.object({
+  type: z.literal('return'),
+  id: z.string(),
+  value: z.unknown(),
+});
+
+/** A capability call threw. */
+export const CallThrewFrameSchema = z.object({
+  type: z.literal('throw'),
+  id: z.string(),
+  error: HookErrorSchema,
+});
+
+export const ParentFrameSchema = z.discriminatedUnion('type', [
+  ContextFrameSchema,
+  CapabilitiesFrameSchema,
+  ReturnFrameSchema,
+  CallThrewFrameSchema,
+]);
+export type ParentFrame = z.infer<typeof ParentFrameSchema>;
+
+/** One capability's entry in the shape descriptor. */
+export type CapabilityShape = z.infer<typeof CapabilitiesFrameSchema>['shape'][string];
+
+// ── framing ───────────────────────────────────────────────────────────────
+
+export type ParseResult<T> = { ok: true; frame: T } | { ok: false; error: string };
+
+/** `JSON.stringify` plus the delimiter. One frame, one line. */
+export function encodeFrame(frame: ChildFrame | ParentFrame): string {
+  return `${JSON.stringify(frame)}\n`;
+}
+
+/**
+ * Parse one line into a frame.
+ *
+ * Returns the failure as a value rather than throwing it. A malformed line is
+ * a hook failure with a readable message, never a parse crash inside celilo:
+ * the whole point of the boundary is that the child cannot take the parent
+ * down, and a reader that throws would hand that back.
+ */
+function parseFrame<T>(schema: z.ZodType<T>, line: string): ParseResult<T> {
+  let json: unknown;
+  try {
+    json = JSON.parse(line);
+  } catch {
+    return { ok: false, error: `not JSON: ${truncate(line)}` };
+  }
+  const parsed = schema.safeParse(json);
+  if (!parsed.success) {
+    return {
+      ok: false,
+      error: `${parsed.error.issues[0]?.message ?? 'invalid'}: ${truncate(line)}`,
+    };
+  }
+  return { ok: true, frame: parsed.data };
+}
+
+export function parseChildFrame(line: string): ParseResult<ChildFrame> {
+  return parseFrame(ChildFrameSchema, line);
+}
+
+export function parseParentFrame(line: string): ParseResult<ParentFrame> {
+  return parseFrame(ParentFrameSchema, line);
+}
+
+function truncate(line: string): string {
+  return line.length > 120 ? `${line.slice(0, 120)}…` : line;
+}
+
+/**
+ * Split a byte stream into complete lines, holding the partial tail.
+ *
+ * Both ends need this and neither can assume a frame arrives in one chunk —
+ * a capability payload is easily larger than a socket read.
+ */
+export function createLineReader(onLine: (line: string) => void): (chunk: string) => void {
+  let buffer = '';
+  return (chunk: string) => {
+    buffer += chunk;
+    let newline = buffer.indexOf('\n');
+    while (newline !== -1) {
+      const line = buffer.slice(0, newline);
+      buffer = buffer.slice(newline + 1);
+      if (line.trim() !== '') onLine(line);
+      newline = buffer.indexOf('\n');
+    }
+  };
+}
+
+/**
+ * The handshake check, in one place so both ends produce the same sentence.
+ *
+ * Names both numbers: a mismatch is an install skew (a `.deb` upgraded while a
+ * module's bundled copy was not), and the operator needs to know which side is
+ * which to fix it.
+ */
+export function versionMismatch(theirs: number, side: string): string | null {
+  if (theirs === HOOK_PROTOCOL_VERSION) return null;
+  return `Hook protocol version mismatch: ${side} speaks ${theirs}, this process speaks ${HOOK_PROTOCOL_VERSION}.`;
+}
+
+// ── error envelopes ───────────────────────────────────────────────────────
+
+/** Fields `MissingProviderInputError` carries and the framework reads (D6). */
+const CARRIED_ERROR_FIELDS = ['providerModuleId', 'ensureId', 'value', 'humanContext'] as const;
+
+/** Turn a thrown value into something that survives JSON. */
+export function serializeError(error: unknown): HookError {
+  if (!(error instanceof Error)) {
+    return { name: 'Error', message: String(error) };
+  }
+  const source = error as unknown as Record<string, unknown>;
+  const fields: Record<string, unknown> = {};
+  for (const key of CARRIED_ERROR_FIELDS) {
+    if (source[key] !== undefined) fields[key] = source[key];
+  }
+  return {
+    name: error.name,
+    message: error.message,
+    stack: error.stack,
+    ...(Object.keys(fields).length > 0 ? { fields } : {}),
+  };
+}
+
+/**
+ * Rebuild a real `Error` so `try`/`catch` inside a hook behaves as it does
+ * in-process, and so the framework's duck-typed guards still recognise it.
+ */
+export function deserializeError(error: HookError): Error {
+  const rebuilt = new Error(error.message);
+  rebuilt.name = error.name;
+  if (error.stack) rebuilt.stack = error.stack;
+  for (const [key, value] of Object.entries(error.fields ?? {})) {
+    (rebuilt as unknown as Record<string, unknown>)[key] = value;
+  }
+  return rebuilt;
+}

package/src/hooks/hook-runner.ts

@@ -0,0 +1,231 @@
+/**
+ * The hook runner shim.
+ *
+ * This is the ONLY thing in celilo that `import()`s a module's hook script,
+ * and it runs in its own `bun` process with an allow-listed environment. It
+ * connects to the broker's socket, receives the context and the capability
+ * shape, rebuilds `HookContext` on this side, invokes the hook, and reports.
+ *
+ * It is spawned, never imported — `executeHookScript` runs
+ * `bun <this file> ` and talks to it over the socket named in
+ * `CELILO_HOOK_SOCKET`. Nothing here is exported for that reason.
+ *
+ * Execution function (Rule 10.1).
+ */
+
+import { connect } from 'node:net';
+import { isCompiledHook } from '@celilo/capabilities';
+import {
+  type CapabilityShape,
+  type ChildFrame,
+  HOOK_PROTOCOL_VERSION,
+  HOOK_SOCKET_ENV,
+  createLineReader,
+  encodeFrame,
+  parseParentFrame,
+  serializeError,
+  versionMismatch,
+} from './hook-protocol';
+import type { HookContext, HookLogger } from './types';
+
+const socketPath = process.env[HOOK_SOCKET_ENV];
+if (!socketPath) {
+  process.stderr.write(
+    `${HOOK_SOCKET_ENV} is not set; the hook runner is not spawnable directly.\n`,
+  );
+  process.exit(2);
+}
+
+const socket = connect(socketPath);
+socket.setEncoding('utf-8');
+
+/** Capability calls in flight, correlated with their answer by id. */
+const pending = new Map<string, { resolve: (v: unknown) => void; reject: (e: Error) => void }>();
+let nextCallId = 0;
+
+let contextData: Record<string, unknown> | undefined;
+let scriptPath: string | undefined;
+let shape: Record<string, CapabilityShape> | undefined;
+let started = false;
+
+function send(frame: ChildFrame): void {
+  socket.write(encodeFrame(frame));
+}
+
+/**
+ * Write the terminal frame and leave.
+ *
+ * The explicit exit is deliberate. A hook that leaves a timer or an open
+ * handle behind would otherwise hold this process open long past its answer,
+ * and the parent — which treats process exit as the one terminal event —
+ * would sit there until the timeout killed a hook that had already finished.
+ */
+function finish(frame: ChildFrame): void {
+  socket.end(encodeFrame(frame), () => process.exit(0));
+  // The flush callback does not fire if the peer is already gone.
+  setTimeout(() => process.exit(0), 2000).unref();
+}
+
+const logger: HookLogger = {
+  info: (message) => send({ type: 'log', level: 'info', message }),
+  warn: (message) => send({ type: 'log', level: 'warn', message }),
+  error: (message) => send({ type: 'log', level: 'error', message }),
+  success: (message) => send({ type: 'log', level: 'success', message }),
+};
+
+/**
+ * Rebuild `context.capabilities` from the shape descriptor.
+ *
+ * Each method forwards; each non-function property is copied verbatim, which
+ * is what keeps `providerModuleId` readable. A method the provider does not
+ * implement is simply absent from the descriptor and therefore absent here, so
+ * `if (capabilities.firewall.registerTrustedSource)` still answers correctly
+ * with no special case for optional methods.
+ */
+function buildCapabilities(descriptor: Record<string, CapabilityShape>): Record<string, unknown> {
+  const capabilities: Record<string, unknown> = {};
+
+  for (const [name, entry] of Object.entries(descriptor)) {
+    const proxy: Record<string, unknown> = { ...entry.data };
+    for (const method of entry.methods) {
+      proxy[method] = (...args: unknown[]) => callBroker(name, method, args);
+    }
+    capabilities[name] = proxy;
+  }
+
+  return capabilities;
+}
+
+function callBroker(capability: string, method: string, args: unknown[]): Promise<unknown> {
+  const id = `c${nextCallId++}`;
+  return new Promise((resolve, reject) => {
+    pending.set(id, { resolve, reject });
+    send({ type: 'call', id, capability, method, args });
+  });
+}
+
+/**
+ * The `defineHook` brand check, moved here from the executor unchanged
+ * (HOOK_API_V2 Phase 8 / D8). The brand is a `Symbol.for` key, so it survives
+ * the identity boundary between this copy of `@celilo/capabilities` and the
+ * one the module bundles (celilo#173) — which is the same reason the check
+ * could move at all.
+ */
+async function runHook(): Promise<void> {
+  if (started || !contextData || !shape || !scriptPath) return;
+  started = true;
+
+  try {
+    // celilo built this object and removed exactly two fields, so the cast
+    // says something true. The check is here because a shape error would
+    // otherwise surface deep inside somebody's hook as a missing property.
+    for (const field of ['config', 'secrets', 'systems', 'debug', 'screenshotDir'] as const) {
+      if (!(field in contextData)) throw new Error(`Hook context is missing '${field}'.`);
+    }
+
+    const context = {
+      ...contextData,
+      logger,
+      capabilities: buildCapabilities(shape),
+    } as unknown as HookContext;
+
+    const module = await import(scriptPath);
+
+    if (typeof module.default !== 'function') {
+      throw new Error(`Hook script must export a default function: ${scriptPath}`);
+    }
+
+    if (!isCompiledHook(module.default)) {
+      throw new Error(
+        `Hook script ${scriptPath} does not use defineHook(). As of HOOK_API_V2 Phase 8, all hook scripts must wrap their handler with defineHook from @celilo/capabilities so the executor can verify the brand and apply pre-flight checks. See reference/MODULE_DEVELOPMENT_GUIDE.md "Hooks" section for the migration pattern.`,
+      );
+    }
+
+    const result = await module.default(context);
+
+    if (result === null || result === undefined) {
+      finish({ type: 'result', outputs: {} });
+      return;
+    }
+
+    if (typeof result !== 'object' || Array.isArray(result)) {
+      throw new Error('Hook script must return an object (or nothing)');
+    }
+
+    finish({ type: 'result', outputs: result as Record<string, unknown> });
+  } catch (error) {
+    finish({ type: 'throw', error: serializeError(error) });
+  }
+}
+
+const feed = createLineReader((line) => {
+  const parsed = parseParentFrame(line);
+  if (!parsed.ok) {
+    // Deliberately WITHOUT the offending line, unlike the broker's mirror of
+    // this check. celilo builds these frames with `JSON.stringify`, so their
+    // content is no help in diagnosing a parse failure — and the `context`
+    // frame carries the module's secrets, which would then ride an error
+    // message out to the operator's terminal and any alert it raises. The
+    // other direction echoes the line because there the bytes ARE the
+    // diagnostic: a grandchild writing to the socket is the failure design D3
+    // exists to catch.
+    finish({
+      type: 'throw',
+      error: {
+        name: 'Error',
+        message: 'Malformed frame from celilo; the hook context could not be read.',
+      },
+    });
+    return;
+  }
+
+  const frame = parsed.frame;
+  switch (frame.type) {
+    case 'context': {
+      const mismatch = versionMismatch(frame.protocolVersion, 'celilo');
+      if (mismatch) {
+        finish({ type: 'throw', error: { name: 'Error', message: mismatch } });
+        return;
+      }
+      contextData = frame.context;
+      scriptPath = frame.scriptPath;
+      void runHook();
+      return;
+    }
+
+    case 'capabilities':
+      shape = frame.shape;
+      void runHook();
+      return;
+
+    case 'return': {
+      pending.get(frame.id)?.resolve(frame.value);
+      pending.delete(frame.id);
+      return;
+    }
+
+    case 'throw': {
+      // Rebuilt inline rather than via deserializeError so the hook's own
+      // `catch` sees a real Error carrying MissingProviderInputError's fields
+      // — `isMissingProviderInputError` is duck-typed and reads them.
+      const rebuilt = new Error(frame.error.message);
+      rebuilt.name = frame.error.name;
+      if (frame.error.stack) rebuilt.stack = frame.error.stack;
+      for (const [key, value] of Object.entries(frame.error.fields ?? {})) {
+        (rebuilt as unknown as Record<string, unknown>)[key] = value;
+      }
+      pending.get(frame.id)?.reject(rebuilt);
+      pending.delete(frame.id);
+      return;
+    }
+  }
+});
+
+socket.on('data', feed);
+socket.on('connect', () => send({ type: 'ready', protocolVersion: HOOK_PROTOCOL_VERSION }));
+socket.on('error', (error) => {
+  // The channel is the only way to report anything, so there is nowhere to
+  // send this. Exit non-zero and let the parent say what it saw.
+  process.stderr.write(`hook runner: socket error: ${error.message}\n`);
+  process.exit(3);
+});

package/src/hooks/hook-timeout.test.ts

@@ -0,0 +1,103 @@
+/**
+ * celilo#1003: a hook that times out must be KILLED, not abandoned.
+ *
+ * `module-lifecycle`'s spec has always required this:
+ *
+ * > WHEN a hook produces no output for longer than the idle timeout
+ * > THEN celilo SHALL terminate it rather than hang indefinitely
+ *
+ * The old executor raced the hook's promise against a timer and cancelled
+ * nothing, because a promise cannot be cancelled. The existing suite asserted
+ * the rejection, which held, and the requirement did not.
+ *
+ * So these tests assert the harm rather than the rejection: a marker file the
+ * hook writes only after its bound has passed. Both of them fail against the
+ * in-process executor and neither says anything about how the kill is
+ * implemented.
+ */
+
+import { afterEach, describe, expect, test } from 'bun:test';
+import { execSync } from 'node:child_process';
+import { existsSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { executeHookScript } from './executor';
+import { createCapturingLogger } from './logger';
+import type { HookContext } from './types';
+
+const FIXTURES = join(__dirname, 'test-fixtures');
+const dirs: string[] = [];
+
+afterEach(() => {
+  for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function scratch(): string {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-timeout-'));
+  dirs.push(dir);
+  return dir;
+}
+
+function contextFor(config: Record<string, unknown>): HookContext {
+  return {
+    config,
+    secrets: {},
+    systems: [],
+    logger: createCapturingLogger().logger,
+    debug: false,
+    screenshotDir: scratch(),
+    capabilities: {},
+  };
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Hook processes still parented to this one. By ppid rather than by script
+ * name: matching the name picks up whatever shell has the filename in its own
+ * command line, which is a false positive that looks exactly like a real leak.
+ */
+function survivingHookProcesses(): string[] {
+  return execSync('ps -Ao ppid=,args=', { encoding: 'utf-8' })
+    .split('\n')
+    .filter(
+      (line) => Number.parseInt(line.trim(), 10) === process.pid && line.includes('hook-runner'),
+    );
+}
+
+describe('hook timeout is a kill, not a race', () => {
+  test('a hook that outruns its total timeout stops doing work', async () => {
+    const marker = join(scratch(), 'kept-running');
+
+    await expect(
+      executeHookScript(
+        join(FIXTURES, 'runaway-hook.ts'),
+        contextFor({ sleep_ms: 1500, marker_path: marker }),
+        400,
+        400,
+      ),
+    ).rejects.toThrow(/timeout/i);
+
+    // Past when the abandoned hook would have written it.
+    await sleep(2000);
+    expect(existsSync(marker)).toBe(false);
+    expect(survivingHookProcesses()).toEqual([]);
+  }, 15_000);
+
+  test('a hook that declines SIGTERM is killed anyway', async () => {
+    const marker = join(scratch(), 'survived');
+
+    await expect(
+      executeHookScript(
+        join(FIXTURES, 'sigterm-ignoring-hook.ts'),
+        contextFor({ sleep_ms: 6000, marker_path: marker }),
+        400,
+        400,
+      ),
+    ).rejects.toThrow(/timeout/i);
+
+    await sleep(6500);
+    expect(existsSync(marker)).toBe(false);
+    expect(survivingHookProcesses()).toEqual([]);
+  }, 20_000);
+});