@celilo/cli 0.3.30 → 0.4.0-alpha.1

package/src/services/build-bus/fan-out.test.ts

@@ -0,0 +1,279 @@
+/**
+ * Fan-out + subscriber-store integration tests.
+ *
+ * Uses real localhost HTTP servers (Bun.serve) as subscriber stand-
+ * ins, not verdaccio — these tests exercise the webhook flow itself
+ * (signing, retry, parallel fan-out), not anything package-publish-
+ * specific. Each test spins up its own server on a random port and
+ * tears it down in afterEach.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { PublishEvent, Subscriber, WebhookEnvelope } from '@celilo/event-bus/build-bus';
+import { verifyEvent } from '@celilo/event-bus/build-bus';
+import { fanOut } from './fan-out';
+import {
+  addSubscriber,
+  loadSubscribers,
+  removeSubscriberByUrl,
+  subscriberStorePath,
+} from './subscriber-store';
+
+function buildEvent(overrides: Partial<PublishEvent> = {}): PublishEvent {
+  return {
+    eventId: '11111111-1111-4111-8111-111111111111',
+    timestamp: new Date().toISOString(),
+    registry: 'npm',
+    tag: 'latest',
+    package: { name: '@celilo/cli', version: '0.4.0' },
+    ...overrides,
+  };
+}
+
+interface TestServer {
+  url: string;
+  /** Latest envelope POST'd to this server. */
+  received: WebhookEnvelope[];
+  /** Latest x-celilo-signature header POSTed. */
+  signatures: string[];
+  stop(): void;
+}
+
+/**
+ * Spin up a tiny HTTP server with configurable response behavior.
+ * Returns its URL so the caller can register it as a subscriber.
+ */
+function startServer(
+  respond: (envelope: WebhookEnvelope, callCount: number) => Response,
+): TestServer {
+  const received: WebhookEnvelope[] = [];
+  const signatures: string[] = [];
+  let callCount = 0;
+  const server = Bun.serve({
+    port: 0,
+    fetch: async (req) => {
+      callCount++;
+      const body = await req.json();
+      received.push(body as WebhookEnvelope);
+      signatures.push(req.headers.get('x-celilo-signature') ?? '');
+      return respond(body as WebhookEnvelope, callCount);
+    },
+  });
+  return {
+    url: `http://localhost:${server.port}/`,
+    received,
+    signatures,
+    stop: () => server.stop(true),
+  };
+}
+
+describe('fanOut', () => {
+  test('delivers a signed envelope to a single subscriber on 200', async () => {
+    const server = startServer(() => new Response('ok', { status: 200 }));
+    try {
+      const subscriber: Subscriber = {
+        name: 'test-sub',
+        url: server.url,
+        secret: 'shared-secret',
+        match: {},
+      };
+      const event = buildEvent();
+      const results = await fanOut(event, [subscriber]);
+
+      expect(results).toHaveLength(1);
+      expect(results[0].ok).toBe(true);
+      expect(results[0].status).toBe(200);
+      expect(results[0].attempts).toBe(1);
+
+      // The receiver saw the envelope and the signature header.
+      expect(server.received).toHaveLength(1);
+      expect(server.received[0].event.eventId).toBe(event.eventId);
+      expect(server.signatures[0]).toMatch(/^sha256=[0-9a-f]{64}$/);
+
+      // The signature actually verifies against the secret.
+      expect(verifyEvent(server.received[0].event, server.signatures[0], 'shared-secret')).toBe(
+        true,
+      );
+      // ...and doesn't verify against a different secret.
+      expect(verifyEvent(server.received[0].event, server.signatures[0], 'wrong')).toBe(false);
+    } finally {
+      server.stop();
+    }
+  });
+
+  test('skips subscribers whose match rule does not fire', async () => {
+    const server = startServer(() => new Response('ok'));
+    try {
+      const event = buildEvent({ registry: 'npm', tag: 'latest' });
+      const results = await fanOut(event, [
+        {
+          url: server.url,
+          secret: 'k',
+          match: { tag: 'alpha' }, // doesn't match event.tag=latest
+        },
+      ]);
+      expect(results).toHaveLength(0);
+      expect(server.received).toHaveLength(0);
+    } finally {
+      server.stop();
+    }
+  });
+
+  test('retries on 5xx and eventually succeeds', async () => {
+    const server = startServer((_e, n) =>
+      n < 3 ? new Response('boom', { status: 503 }) : new Response('ok'),
+    );
+    try {
+      const results = await fanOut(buildEvent(), [{ url: server.url, secret: 'k', match: {} }], {
+        maxAttempts: 5,
+        baseBackoffMs: 1,
+      });
+      expect(results[0].ok).toBe(true);
+      expect(results[0].attempts).toBe(3);
+    } finally {
+      server.stop();
+    }
+  });
+
+  test('does NOT retry on 4xx (client error is permanent)', async () => {
+    const server = startServer(() => new Response('bad request', { status: 400 }));
+    try {
+      const results = await fanOut(buildEvent(), [{ url: server.url, secret: 'k', match: {} }], {
+        maxAttempts: 5,
+        baseBackoffMs: 1,
+      });
+      expect(results[0].ok).toBe(false);
+      expect(results[0].status).toBe(400);
+      expect(results[0].attempts).toBe(1);
+      expect(results[0].error).toContain('no retry on 4xx');
+    } finally {
+      server.stop();
+    }
+  });
+
+  test('gives up after maxAttempts of 5xx', async () => {
+    const server = startServer(() => new Response('boom', { status: 503 }));
+    try {
+      const results = await fanOut(buildEvent(), [{ url: server.url, secret: 'k', match: {} }], {
+        maxAttempts: 3,
+        baseBackoffMs: 1,
+      });
+      expect(results[0].ok).toBe(false);
+      expect(results[0].status).toBe(503);
+      expect(results[0].attempts).toBe(3);
+    } finally {
+      server.stop();
+    }
+  });
+
+  test('fans out to multiple subscribers in parallel', async () => {
+    const serverA = startServer(() => new Response('ok'));
+    const serverB = startServer(() => new Response('ok'));
+    try {
+      const event = buildEvent();
+      const results = await fanOut(event, [
+        { url: serverA.url, secret: 'a', match: {} },
+        { url: serverB.url, secret: 'b', match: {} },
+      ]);
+
+      expect(results).toHaveLength(2);
+      expect(results.every((r) => r.ok)).toBe(true);
+      expect(serverA.received).toHaveLength(1);
+      expect(serverB.received).toHaveLength(1);
+
+      // Each subscriber got its OWN signature (per-target secret).
+      expect(serverA.signatures[0]).not.toBe(serverB.signatures[0]);
+      expect(verifyEvent(serverA.received[0].event, serverA.signatures[0], 'a')).toBe(true);
+      expect(verifyEvent(serverB.received[0].event, serverB.signatures[0], 'b')).toBe(true);
+    } finally {
+      serverA.stop();
+      serverB.stop();
+    }
+  });
+
+  test('records network failure as ok=false', async () => {
+    const results = await fanOut(
+      buildEvent(),
+      [{ url: 'http://127.0.0.1:1/no-such-port', secret: 'k', match: {} }],
+      {
+        maxAttempts: 1,
+        timeoutMs: 200,
+      },
+    );
+    expect(results[0].ok).toBe(false);
+    expect(results[0].error).toBeDefined();
+  });
+});
+
+describe('subscriber-store', () => {
+  let storeDir: string;
+  let originalEnv: string | undefined;
+
+  beforeEach(() => {
+    storeDir = mkdtempSync(join(tmpdir(), 'celilo-build-bus-store-'));
+    originalEnv = process.env.CELILO_BUILD_BUS_SUBSCRIBERS_PATH;
+    process.env.CELILO_BUILD_BUS_SUBSCRIBERS_PATH = join(storeDir, 'subscribers.json');
+  });
+
+  afterEach(() => {
+    process.env.CELILO_BUILD_BUS_SUBSCRIBERS_PATH = originalEnv;
+    rmSync(storeDir, { recursive: true, force: true });
+  });
+
+  test('subscriberStorePath honors the env override', () => {
+    expect(subscriberStorePath()).toBe(join(storeDir, 'subscribers.json'));
+  });
+
+  test('loadSubscribers returns [] when the file is missing', () => {
+    expect(loadSubscribers()).toEqual([]);
+  });
+
+  test('round-trip: add → load → remove', () => {
+    const sub: Subscriber = {
+      name: 'lunacycle',
+      url: 'https://lunacycle.lab/build-bus',
+      secret: 'hex-secret-here',
+      match: { registry: 'npm', packagePattern: '@celilo/*' },
+    };
+
+    const addResult = addSubscriber(sub);
+    expect(addResult.replaced).toBeUndefined();
+
+    const after = loadSubscribers();
+    expect(after).toHaveLength(1);
+    expect(after[0]).toEqual(sub);
+
+    const removed = removeSubscriberByUrl(sub.url);
+    expect(removed).toEqual(sub);
+    expect(loadSubscribers()).toEqual([]);
+  });
+
+  test('adding a subscriber with an existing URL replaces it (idempotent rotation)', () => {
+    const original: Subscriber = {
+      url: 'https://x.test/',
+      secret: 'old',
+      match: { registry: 'npm' },
+    };
+    addSubscriber(original);
+
+    const rotated: Subscriber = {
+      url: 'https://x.test/',
+      secret: 'new',
+      match: { registry: 'celilo-registry' },
+    };
+    const result = addSubscriber(rotated);
+
+    expect(result.replaced).toEqual(original);
+    const after = loadSubscribers();
+    expect(after).toHaveLength(1);
+    expect(after[0].secret).toBe('new');
+    expect(after[0].match.registry).toBe('celilo-registry');
+  });
+
+  test('removeSubscriberByUrl returns undefined when URL not registered', () => {
+    expect(removeSubscriberByUrl('https://not-here.test/')).toBeUndefined();
+  });
+});

package/src/services/build-bus/fan-out.ts

@@ -0,0 +1,161 @@
+/**
+ * Webhook fan-out for the build bus ([[v2/BUILD_BUS.md]] Phase 2).
+ *
+ * Given a PublishEvent and a subscriber list, this module:
+ *   1. Filters subscribers whose match rule fires for the event.
+ *   2. Builds a signed WebhookEnvelope per subscriber (each gets
+ *      its own HMAC signature with the subscriber's per-target
+ *      secret).
+ *   3. Fires concurrent HTTP POSTs with exponential-backoff retry.
+ *   4. Returns per-subscriber DeliveryResults so the caller can
+ *      surface failures to the operator.
+ *
+ * Best-effort by design — fan-out failures do NOT fail the publish.
+ * The spec calls for delivery to be at-least-once with operator-
+ * visible failure (§9), not a publish blocker.
+ */
+
+import {
+  type DeliveryResult,
+  type PublishEvent,
+  type Subscriber,
+  type WebhookEnvelope,
+  signEvent,
+  subscribersFor,
+} from '@celilo/event-bus/build-bus';
+
+export interface FanOutOptions {
+  /** Total attempts (1 = no retry). Default: 4. */
+  maxAttempts?: number;
+  /** Per-attempt request timeout. Default: 10s. */
+  timeoutMs?: number;
+  /** First retry waits this long; subsequent doubles. Default: 500ms. */
+  baseBackoffMs?: number;
+  /**
+   * Override the fetch implementation. Tests inject a stub so they
+   * don't have to actually open sockets. Production uses the global
+   * fetch.
+   */
+  fetch?: typeof fetch;
+  /**
+   * "Now" injection for deterministic durationMs in tests. Default:
+   * Date.now.
+   */
+  now?: () => number;
+}
+
+const DEFAULT_MAX_ATTEMPTS = 4;
+const DEFAULT_TIMEOUT_MS = 10_000;
+const DEFAULT_BASE_BACKOFF_MS = 500;
+
+/**
+ * Fan out a single event to every matching subscriber. Returns one
+ * DeliveryResult per matched subscriber. Subscribers whose match
+ * rules don't fire are silently filtered out and don't appear in
+ * the result.
+ */
+export async function fanOut(
+  event: PublishEvent,
+  subscribers: Subscriber[],
+  opts: FanOutOptions = {},
+): Promise<DeliveryResult[]> {
+  const matched = subscribersFor(event, subscribers);
+  return Promise.all(matched.map((s) => deliverOne(event, s, opts)));
+}
+
+async function deliverOne(
+  event: PublishEvent,
+  subscriber: Subscriber,
+  opts: FanOutOptions,
+): Promise<DeliveryResult> {
+  const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const baseBackoffMs = opts.baseBackoffMs ?? DEFAULT_BASE_BACKOFF_MS;
+  const fetchFn = opts.fetch ?? fetch;
+  const now = opts.now ?? Date.now;
+
+  const envelope: WebhookEnvelope = {
+    event,
+    signature: signEvent(event, subscriber.secret),
+  };
+  const body = JSON.stringify(envelope);
+  const startedAt = now();
+
+  let lastError: string | undefined;
+  let lastStatus: number | undefined;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const response = await fetchFn(subscriber.url, {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-celilo-signature': envelope.signature,
+          'x-celilo-event-id': event.eventId,
+        },
+        body,
+        signal: controller.signal,
+      });
+      lastStatus = response.status;
+      if (response.ok) {
+        clearTimeout(timer);
+        return {
+          subscriber,
+          ok: true,
+          status: response.status,
+          attempts: attempt,
+          durationMs: now() - startedAt,
+        };
+      }
+      lastError = `HTTP ${response.status}`;
+      // 4xx is a client error — won't recover by retrying. Don't burn
+      // attempts on it (saves time + makes the operator-visible
+      // error actionable).
+      if (response.status >= 400 && response.status < 500) {
+        clearTimeout(timer);
+        return {
+          subscriber,
+          ok: false,
+          status: response.status,
+          attempts: attempt,
+          error: `HTTP ${response.status} (no retry on 4xx)`,
+          durationMs: now() - startedAt,
+        };
+      }
+    } catch (err) {
+      lastError = err instanceof Error ? err.message : String(err);
+    } finally {
+      clearTimeout(timer);
+    }
+
+    if (attempt < maxAttempts) {
+      const backoff = baseBackoffMs * 2 ** (attempt - 1);
+      await new Promise((r) => setTimeout(r, backoff));
+    }
+  }
+
+  return {
+    subscriber,
+    ok: false,
+    status: lastStatus,
+    attempts: maxAttempts,
+    error: lastError ?? 'unknown delivery failure',
+    durationMs: now() - startedAt,
+  };
+}
+
+/**
+ * Render a single delivery result for the operator. Operator
+ * surface in `celilo publish` summary + `celilo subscribers status`.
+ */
+export function formatDeliveryResult(result: DeliveryResult): string {
+  const target = result.subscriber.name
+    ? `${result.subscriber.name} (${result.subscriber.url})`
+    : result.subscriber.url;
+  if (result.ok) {
+    return `✓ ${target} — ${result.status} in ${result.durationMs}ms (${result.attempts} attempt${result.attempts === 1 ? '' : 's'})`;
+  }
+  return `✗ ${target} — ${result.error} (${result.attempts} attempt${result.attempts === 1 ? '' : 's'}, ${result.durationMs}ms)`;
+}

package/src/services/build-bus/hook-dispatch-mgmt.test.ts

@@ -0,0 +1,157 @@
+/**
+ * End-to-end test: build-bus hook dispatcher → modules/celilo-mgmt's
+ * on_upstream_publish rules → self-update script (with the actual
+ * `bun add -g` replaced by a noop so the test doesn't mutate the
+ * operator's global install).
+ *
+ * Confirms the manifest's match rules pick the right hook for a
+ * given event AND that the dispatcher passes the expected env vars
+ * to the script.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { readFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { PublishEvent } from '@celilo/event-bus/build-bus';
+import { parse as parseYaml } from 'yaml';
+import type { ModuleHookContext, UpstreamHookEntry } from './hook-dispatch';
+import { startHookDispatcher } from './hook-dispatcher';
+
+const REPO_ROOT = join(__dirname, '..', '..', '..', '..', '..');
+
+function loadCeliloMgmtHooks(): UpstreamHookEntry[] {
+  // Parse the real modules/celilo-mgmt/manifest.yml so this test
+  // tracks the actual deployed manifest — drift here is a test
+  // failure rather than silent breakage.
+  const manifestPath = join(REPO_ROOT, 'modules', 'celilo-mgmt', 'manifest.yml');
+  const parsed = parseYaml(readFileSync(manifestPath, 'utf-8')) as {
+    hooks?: { on_upstream_publish?: UpstreamHookEntry[] };
+  };
+  return parsed.hooks?.on_upstream_publish ?? [];
+}
+
+function buildEvent(overrides: Partial<PublishEvent> = {}): PublishEvent {
+  return {
+    eventId: 'evt-1',
+    timestamp: new Date().toISOString(),
+    registry: 'npm',
+    tag: 'latest',
+    package: { name: '@celilo/cli', version: '0.4.0' },
+    ...overrides,
+  };
+}
+
+describe('hook-dispatcher → modules/celilo-mgmt', () => {
+  let tmpModuleDir: string;
+
+  beforeEach(() => {
+    // Use the real manifest's hooks but point them at a noop script
+    // in a temp dir so we don't actually shell out to `bun add -g`.
+    tmpModuleDir = mkdtempSync(join(tmpdir(), 'celilo-mgmt-hook-test-'));
+    mkdirSync(join(tmpModuleDir, 'scripts'), { recursive: true });
+    const scriptPath = join(tmpModuleDir, 'scripts', 'on_upstream_publish.sh');
+    // Record the env we received so the test can verify the dispatcher
+    // passed the right vars. Sentinel file lives in tmpModuleDir.
+    const logPath = join(tmpModuleDir, 'invocations.log');
+    writeFileSync(
+      scriptPath,
+      `#!/usr/bin/env bash
+echo "name=$CELILO_EVENT_PACKAGE_NAME version=$CELILO_EVENT_PACKAGE_VERSION tag=$CELILO_EVENT_TAG" >> ${logPath}
+exit 0
+`,
+    );
+    chmodSync(scriptPath, 0o755);
+  });
+
+  afterEach(() => {
+    rmSync(tmpModuleDir, { recursive: true, force: true });
+  });
+
+  function buildModuleContext(): ModuleHookContext {
+    const hooks = loadCeliloMgmtHooks();
+    // Rewrite the script path to point at the temp noop. The match
+    // rules from the real manifest are preserved — that's what we're
+    // testing.
+    return {
+      moduleId: 'celilo-mgmt',
+      sourcePath: tmpModuleDir,
+      hooks,
+    };
+  }
+
+  test('the real manifest declares two on_upstream_publish hooks (cli + e2e)', () => {
+    const hooks = loadCeliloMgmtHooks();
+    expect(hooks).toHaveLength(2);
+    const names = hooks.map((h) => h.name).sort();
+    expect(names).toEqual(['self-update-celilo-cli', 'self-update-celilo-e2e']);
+  });
+
+  test('a @celilo/cli@latest event fires the celilo-cli hook only', async () => {
+    const dispatcher = await startHookDispatcher({
+      loadModules: () => [buildModuleContext()],
+    });
+    const results = await dispatcher.handleEvent(
+      buildEvent({ package: { name: '@celilo/cli', version: '0.5.0' } }),
+    );
+    await dispatcher.stop();
+
+    expect(results).toHaveLength(1);
+    expect(results[0].hookName).toBe('self-update-celilo-cli');
+    expect(results[0].exitCode).toBe(0);
+
+    // The log written by the noop script should show the env it received.
+    const log = readFileSync(join(tmpModuleDir, 'invocations.log'), 'utf-8');
+    expect(log).toContain('name=@celilo/cli version=0.5.0 tag=latest');
+  });
+
+  test('a @celilo/e2e@latest event fires the celilo-e2e hook only', async () => {
+    const dispatcher = await startHookDispatcher({
+      loadModules: () => [buildModuleContext()],
+    });
+    const results = await dispatcher.handleEvent(
+      buildEvent({ package: { name: '@celilo/e2e', version: '0.8.0' } }),
+    );
+    await dispatcher.stop();
+
+    expect(results).toHaveLength(1);
+    expect(results[0].hookName).toBe('self-update-celilo-e2e');
+  });
+
+  test('an alpha-tag event for @celilo/cli does NOT fire (tag must be "latest")', async () => {
+    const dispatcher = await startHookDispatcher({
+      loadModules: () => [buildModuleContext()],
+    });
+    const results = await dispatcher.handleEvent(
+      buildEvent({ tag: 'alpha', package: { name: '@celilo/cli', version: '0.5.0-alpha.0' } }),
+    );
+    await dispatcher.stop();
+    expect(results).toEqual([]);
+  });
+
+  test('a non-celilo package (e.g. @celilo/event-bus) does NOT fire', async () => {
+    const dispatcher = await startHookDispatcher({
+      loadModules: () => [buildModuleContext()],
+    });
+    const results = await dispatcher.handleEvent(
+      buildEvent({ package: { name: '@celilo/event-bus', version: '0.2.0' } }),
+    );
+    await dispatcher.stop();
+    expect(results).toEqual([]);
+  });
+
+  test('a non-npm registry event (e.g. celilo-registry module publish) does NOT fire', async () => {
+    const dispatcher = await startHookDispatcher({
+      loadModules: () => [buildModuleContext()],
+    });
+    const results = await dispatcher.handleEvent(
+      buildEvent({
+        registry: 'celilo-registry',
+        package: { name: '@celilo/cli', version: '0.5.0' },
+      }),
+    );
+    await dispatcher.stop();
+    expect(results).toEqual([]);
+  });
+});