@devindex/api-kit 0.1.0

package/events/drivers/bullmq.js

@@ -0,0 +1,112 @@
+import {
+  loadBullmq,
+  queueName,
+  redisConnection,
+} from '../../internal/bullmq.js';
+import { noopLogger } from '../../internal/logger.js';
+import { subscriberStream } from '../internal.js';
+
+/**
+ * Distributed backend on BullMQ, one queue per subscriber stream. Redis is only
+ * its store.
+ *
+ * @param {object} options
+ * @param {string} options.redisUrl
+ * @param {string} [options.prefix='app']
+ * @param {object} [options.logger]
+ * @return {object} A backend: add, work and stop.
+ */
+export function bullmqBackend({ redisUrl, prefix = 'app', logger = noopLogger } = {}) {
+  const queueConnection = redisConnection(redisUrl);
+  const workerConnection = redisConnection(redisUrl, { worker: true });
+  const queuePromises = new Map();
+  const workers = new Set();
+
+  /** Get or create the per-stream BullMQ queue with its retention policy. */
+  function getQueue(stream) {
+    let promise = queuePromises.get(stream);
+    if (!promise) {
+      promise = (async () => {
+        const { Queue } = await loadBullmq();
+        return new Queue(queueName(prefix, 'event', stream), {
+          connection: queueConnection,
+          defaultJobOptions: {
+            removeOnComplete: { age: 24 * 60 * 60, count: 1_000 },
+            removeOnFail: { age: 7 * 24 * 60 * 60 },
+          },
+        });
+      })();
+      queuePromises.set(stream, promise);
+    }
+    return promise;
+  }
+
+  return {
+    async add(stream, event, payload, { key, dedupId, eventId, attempts, backoff, delay = 0 }) {
+      const queue = await getQueue(stream);
+      const job = await queue.add(event, { payload, key, eventId }, {
+        attempts,
+        backoff,
+        deduplication: { id: dedupId },
+        ...(delay > 0 ? { delay } : {}),
+      });
+      return job.id;
+    },
+
+    async work(event, subscriber, definition) {
+      const { Worker } = await loadBullmq();
+      const stream = subscriberStream(event, subscriber);
+      const queue = await getQueue(stream);
+      const worker = new Worker(queue.name, async (job, _token, signal) => {
+        if (job.name !== event) {
+          throw new Error(`event "${job.name}" reached the "${subscriber}" worker for "${event}"`);
+        }
+        const attempt = job.attemptsMade + 1;
+        const totalAttempts = job.opts.attempts ?? 1;
+        const log = logger.child({ event, subscriber, key: job.data.key, deliveryId: job.id });
+        try {
+          return await definition.handler(job.data.payload, {
+            event,
+            subscriber,
+            eventId: job.data.eventId,
+            key: job.data.key,
+            attempt,
+            attemptsLeft: totalAttempts - attempt,
+            signal,
+            log,
+          });
+        } catch (error) {
+          // A retry that BullMQ will re-run is a warning; only the last attempt is
+          // an error, so expected transient retries don't trip error-rate alerts.
+          const final = attempt >= totalAttempts;
+          log[final ? 'error' : 'warn']({ err: error, attempt, final }, 'event delivery failed');
+          throw error;
+        }
+      }, { connection: workerConnection, concurrency: definition.concurrency });
+      workers.add(worker);
+      worker.on('error', (error) => logger.error({ err: error, event, subscriber }, 'event worker error'));
+      worker.on('stalled', (jobId) => {
+        logger.error({ event, subscriber, deliveryId: jobId }, 'event delivery stalled and will be delivered again');
+      });
+
+      // Surface a worker that cannot reach Redis at start(), not silently later.
+      await worker.waitUntilReady();
+    },
+
+    // BullMQ owns the drain deadline once worker.close() starts, so the bus's
+    // timeoutMs is accepted for a uniform signature but not used here.
+    async stop() {
+      // Workers first: they stop taking deliveries and let the running ones finish.
+      const workerResults = await Promise.allSettled([...workers].map((worker) => worker.close()));
+      const queueResults = await Promise.allSettled([...queuePromises.values()]);
+      const closeResults = await Promise.allSettled(
+        queueResults.flatMap((result) => (result.status === 'fulfilled' ? [result.value.close()] : [])),
+      );
+      for (const result of [...workerResults, ...queueResults, ...closeResults]) {
+        if (result.status === 'rejected') logger.error({ err: result.reason }, 'event bus close failed');
+      }
+      workers.clear();
+      queuePromises.clear();
+    },
+  };
+}

package/events/drivers/memory.js

@@ -0,0 +1,169 @@
+import { randomUUID } from 'node:crypto';
+import { noopLogger } from '../../internal/logger.js';
+import { backoffDelay } from '../../internal/retry.js';
+import { subscriberStream } from '../internal.js';
+
+/**
+ * In-process backend for tests and local runs. Honors delay, attempts, backoff,
+ * deduplication and concurrency, and keeps nothing once the process exits.
+ *
+ * @param {object} [options]
+ * @param {object} [options.logger]
+ * @return {object} A backend: add, work and stop.
+ */
+export function memoryBackend({ logger = noopLogger } = {}) {
+  const consumers = new Map();
+  const lanes = new Map();
+  const dedup = new Map();
+  const timers = new Set();
+  const inFlight = new Set();
+  const shutdown = new AbortController();
+  let closed = false;
+
+  /** Get or create the per-stream queue: its backlog, head pointer and running count. */
+  function lane(stream) {
+    let entry = lanes.get(stream);
+    if (!entry) {
+      entry = { waiting: [], head: 0, running: 0 };
+      lanes.set(stream, entry);
+    }
+    return entry;
+  }
+
+  /** Resolve after `ms`, or early if the bus is stopped, without leaking the timer. */
+  function sleep(ms) {
+    if (ms === 0) return Promise.resolve();
+    return new Promise((resolve) => {
+      const done = () => {
+        clearTimeout(timer);
+        timers.delete(timer);
+        shutdown.signal.removeEventListener('abort', done);
+        resolve();
+      };
+      const timer = setTimeout(done, ms);
+      timers.add(timer);
+      shutdown.signal.addEventListener('abort', done, { once: true });
+    });
+  }
+
+  /** Deliver one job to its subscriber, retrying with backoff until it succeeds or exhausts attempts. */
+  async function run(consumer, job) {
+    const { definition, event, subscriber } = consumer;
+    const log = logger.child({ event, subscriber, key: job.key, deliveryId: job.id });
+    try {
+      for (let attempt = 1; attempt <= definition.attempts; attempt += 1) {
+        try {
+          await definition.handler(job.payload, {
+            event,
+            subscriber,
+            eventId: job.eventId,
+            key: job.key,
+            attempt,
+            attemptsLeft: definition.attempts - attempt,
+            signal: shutdown.signal,
+            log,
+          });
+          return;
+        } catch (error) {
+          // A retry that will run again is a warning; only the last failure is an
+          // error, so expected transient retries don't trip error-rate alerts.
+          const final = attempt === definition.attempts || shutdown.signal.aborted;
+          log[final ? 'error' : 'warn']({ err: error, attempt, final }, 'event delivery failed');
+          if (final) return;
+          await sleep(backoffDelay(definition.backoff, attempt));
+          if (shutdown.signal.aborted) return;
+        }
+      }
+    } finally {
+      if (dedup.get(job.dedupId) === job.id) dedup.delete(job.dedupId);
+    }
+  }
+
+  /** Start as many queued deliveries for `stream` as concurrency allows, re-pumping as each finishes. */
+  function pump(stream) {
+    const consumer = consumers.get(stream);
+    if (closed || !consumer) return;
+    const entry = lane(stream);
+    while (entry.running < consumer.definition.concurrency && entry.head < entry.waiting.length) {
+      const job = entry.waiting[entry.head];
+      entry.waiting[entry.head] = undefined; // drop the payload reference for GC
+      entry.head += 1;
+      // `shift()` reindexes the whole backlog (O(n^2) under a burst); a head
+      // pointer keeps dequeue O(1), compacted so it can't grow unbounded.
+      if (entry.head > 1024 && entry.head * 2 >= entry.waiting.length) {
+        entry.waiting = entry.waiting.slice(entry.head);
+        entry.head = 0;
+      }
+      entry.running += 1;
+      const promise = run(consumer, job).finally(() => {
+        entry.running -= 1;
+        inFlight.delete(promise);
+        pump(stream);
+      });
+      inFlight.add(promise);
+    }
+  }
+
+  return {
+    async add(stream, event, payload, { key, dedupId, eventId, delay = 0 }) {
+      if (closed) throw new Error('cannot publish on a stopped event bus');
+      // Dedup is held only while the delivery is in flight, so the same key is
+      // free again once the delivery settles.
+      const held = dedup.get(dedupId);
+      if (held) return held;
+
+      const deliveryId = randomUUID();
+      dedup.set(dedupId, deliveryId);
+      const job = { id: deliveryId, dedupId, key, payload, eventId };
+
+      if (delay > 0) {
+        const timer = setTimeout(() => {
+          timers.delete(timer);
+          if (closed) return;
+          lane(stream).waiting.push(job);
+          pump(stream);
+        }, delay);
+        timers.add(timer);
+      } else {
+        lane(stream).waiting.push(job);
+        pump(stream);
+      }
+      return deliveryId;
+    },
+
+    async work(event, subscriber, definition) {
+      // Draining here picks up anything published before the worker existed.
+      const stream = subscriberStream(event, subscriber);
+      consumers.set(stream, { definition, event, subscriber });
+      pump(stream);
+    },
+
+    async stop({ timeoutMs = 30_000 } = {}) {
+      if (closed) return;
+      closed = true;
+      shutdown.abort();
+      for (const timer of timers) clearTimeout(timer);
+      timers.clear();
+      for (const entry of lanes.values()) {
+        entry.waiting.length = 0;
+        entry.head = 0;
+      }
+
+      // A handler that ignores `signal` would otherwise keep the process alive
+      // forever, so drain in-flight work under a deadline instead of unbounded.
+      const drained = await Promise.race([
+        Promise.allSettled([...inFlight]).then(() => true),
+        new Promise((resolve) => {
+          setTimeout(() => resolve(false), timeoutMs).unref?.();
+        }),
+      ]);
+      if (!drained) {
+        logger.warn({ timeoutMs, pending: inFlight.size }, 'event bus stop timed out; abandoning in-flight deliveries');
+      }
+
+      lanes.clear();
+      dedup.clear();
+      inFlight.clear();
+    },
+  };
+}

package/events/index.js

@@ -0,0 +1,172 @@
+import { randomUUID } from 'node:crypto';
+import { noopLogger } from '../internal/logger.js';
+import {
+  assertDuration,
+  assertHandler,
+  assertKey,
+  assertName,
+} from '../internal/validation.js';
+import {
+  DEFAULTS,
+  dedupId,
+  normalizeBackoff,
+  resolveDefinition,
+} from '../internal/retry.js';
+import { assertSubscriber, subscriberStream } from './internal.js';
+import { memoryBackend } from './drivers/memory.js';
+import { bullmqBackend } from './drivers/bullmq.js';
+
+// Named for what delivers the events, not for its store: the distributed backend
+// is BullMQ (Redis is only where it keeps state), so the semantics a caller gets —
+// stalled recovery, deduplication, retention — are BullMQ's, and the name says so.
+const BACKENDS = { memory: memoryBackend, bullmq: bullmqBackend };
+
+/**
+ * Creates a volatile local event bus or a distributed BullMQ event bus. Each
+ * published event fans out to every named subscriber independently.
+ *
+ * The lifecycle — declaration, fan-out, dedup keys, state — lives here; each
+ * backend only add()s, work()s and stop()s.
+ *
+ * @param {object} [options]
+ * @param {'memory'|'bullmq'} [options.driver='memory']
+ * @param {string} [options.redisUrl] - Required by the BullMQ driver.
+ * @param {string} [options.prefix='app']
+ * @param {object} [options.logger]
+ * @param {object} [options.defaults] - Per-subscriber attempts, backoff and concurrency.
+ * @return {object} The bus, not yet started; call start() first.
+ */
+export function createEventBus({
+  driver = 'memory',
+  redisUrl,
+  prefix = 'app',
+  logger = noopLogger,
+  defaults = {},
+} = {}) {
+  const createBackend = BACKENDS[driver];
+  if (!createBackend) {
+    throw new Error(`unknown event bus driver "${driver}", expected ${Object.keys(BACKENDS).join(' or ')}`);
+  }
+  const merged = Object.freeze({
+    attempts: defaults.attempts ?? DEFAULTS.attempts,
+    backoff: normalizeBackoff(defaults.backoff ?? DEFAULTS.backoff),
+    concurrency: defaults.concurrency ?? DEFAULTS.concurrency,
+  });
+
+  const subscribers = new Map();
+  const backend = createBackend({ redisUrl, prefix, logger });
+  let state = 'idle';
+
+  /**
+   * Declares a subscriber before start(); the (event, subscriber) pair must be unique.
+   *
+   * @param {string} event
+   * @param {string} subscriber - Names an independent, durable delivery stream for the event.
+   * @param {(payload: *, context: object) => (void|Promise<void>)} handler
+   * @param {object} [options] - Per-subscriber overrides: `attempts`, `backoff`, `concurrency`.
+   * @return {void}
+   */
+  function subscribe(event, subscriber, handler, options = {}) {
+    if (state !== 'idle') throw new Error('subscribers must be declared before start()');
+    const eventName = assertName(event, 'event');
+    const subscriberName = assertSubscriber(subscriber);
+    assertHandler(subscriberStream(eventName, subscriberName), handler);
+    let group = subscribers.get(eventName);
+    if (!group) {
+      group = new Map();
+      subscribers.set(eventName, group);
+    }
+    if (group.has(subscriberName)) {
+      throw new Error(`subscriber "${subscriberName}" is already declared for event "${eventName}"`);
+    }
+    group.set(subscriberName, { handler, ...resolveDefinition(merged, options) });
+  }
+
+  /**
+   * Publishes an event to every declared subscriber.
+   *
+   * @param {string} event
+   * @param {*} payload - Passed to each subscriber handler as its first argument.
+   * @param {object} options
+   * @param {string} options.key - Required, non-empty idempotency key.
+   * @param {number} [options.delay=0] - Milliseconds before delivery becomes runnable.
+   * @return {Promise<{eventId: string, event: string, key: string, deliveries: ReadonlyArray<{subscriber: string, deliveryId: string}>}>}
+   */
+  async function publish(event, payload, { key: givenKey, delay = 0 } = {}) {
+    if (state !== 'started') throw new Error('the event bus must be started before publish()');
+    const eventName = assertName(event, 'event');
+    const key = assertKey(givenKey);
+    assertDuration('delay', delay);
+    const eventId = randomUUID();
+    const group = subscribers.get(eventName);
+    if (!group || group.size === 0) {
+      // Publisher and subscriber are decoupled: a publish with no listeners is a
+      // valid no-op, not an error.
+      logger.debug({ event: eventName, eventId }, 'event published with no subscribers');
+      return Object.freeze({ eventId, event: eventName, key, deliveries: Object.freeze([]) });
+    }
+
+    // Fan-out: each subscriber has its own stream, dedup identity and delivery.
+    const deliveries = await Promise.all([...group].map(async ([subscriber, definition]) => {
+      const stream = subscriberStream(eventName, subscriber);
+      const deliveryId = await backend.add(stream, eventName, payload, {
+        key,
+        dedupId: dedupId(stream, key),
+        eventId,
+        attempts: definition.attempts,
+        backoff: definition.backoff,
+        delay,
+      });
+      return Object.freeze({ subscriber, deliveryId });
+    }));
+    return Object.freeze({ eventId, event: eventName, key, deliveries: Object.freeze(deliveries) });
+  }
+
+  async function start() {
+    if (state === 'started') return;
+    if (state === 'stopped') throw new Error('a stopped event bus cannot be restarted');
+    if (state !== 'idle') throw new Error('the event bus is already changing state');
+    state = 'starting';
+    const definitions = [...subscribers].flatMap(([event, group]) =>
+      [...group].map(([subscriber, definition]) => ({ event, subscriber, definition })));
+    // allSettled lets every worker register before a failure surfaces, so a
+    // rejection mid-start can't leak the workers its siblings already created.
+    const results = await Promise.allSettled(
+      definitions.map(({ event, subscriber, definition }) => backend.work(event, subscriber, definition)),
+    );
+    const failed = results.find((result) => result.status === 'rejected');
+    if (failed) {
+      state = 'stopping';
+      await backend.stop({});
+      state = 'stopped';
+      throw failed.reason;
+    }
+    state = 'started';
+  }
+
+  /**
+   * Stops intake and drains in-flight deliveries.
+   *
+   * @param {object} [options]
+   * @param {number} [options.timeoutMs=30000] - Drain deadline before abandoning
+   *   in-flight deliveries (memory driver; the BullMQ driver lets BullMQ own the deadline).
+   * @return {Promise<void>}
+   */
+  async function stop(options = {}) {
+    if (state === 'stopped') return;
+    state = 'stopping';
+    await backend.stop(options);
+    state = 'stopped';
+  }
+
+  return Object.freeze({
+    driver,
+    subscribe,
+    publish,
+    start,
+    stop,
+    get state() {
+      return state;
+    },
+  });
+}

package/events/internal.js

@@ -0,0 +1,14 @@
+import { assertName } from '../internal/validation.js';
+
+export function assertSubscriber(name) {
+  return assertName(name, 'subscriber');
+}
+
+/**
+ * Identity of one subscriber's durable stream for an event.
+ *
+ * @return {string} Key backing both the Redis queue name and the dedup id.
+ */
+export function subscriberStream(event, subscriber) {
+  return `${event}::${subscriber}`;
+}

package/http/createApp.js

@@ -0,0 +1,95 @@
+import Fastify from 'fastify';
+import decorators from './plugins/decorators.js';
+import errorHandler from './plugins/errorHandler.js';
+import rawBody from './plugins/rawBody.js';
+import requestContext from './plugins/requestContext.js';
+import requestId, { genReqId } from './plugins/requestId.js';
+import { LOG_TYPE, withType } from '../log/index.js';
+import corsPlugin from '@fastify/cors';
+import helmetPlugin from '@fastify/helmet';
+
+// CSP is a browser directive for rendered documents, inert on JSON responses,
+// and its default breaks any HTML tooling bolted onto the API (Swagger, error
+// pages). An app serving HTML re-enables it with its own directives.
+const HELMET_DEFAULTS = Object.freeze({ global: true, contentSecurityPolicy: false });
+
+/**
+ * Builds the complete Fastify stack without binding a port, so tests can drive
+ * it with `app.inject()`. Async context is opt-in: pass a `context` store to
+ * propagate the correlation id below the HTTP layer; omitted, no ALS runs.
+ *
+ * @param {object} [options]
+ * @param {object} [options.logger] - A pino instance, base and untyped — the kit stamps `type:lifecycle` on the instance and `type:request` per request. Omitted disables Fastify's logging.
+ * @param {object} [options.context] - A context store from `./context`. Omitted disables ALS.
+ * @param {object|false} [options.helmet] - `@fastify/helmet` options. On by default with CSP off; pass `false` to disable. Re-enable CSP for HTML endpoints.
+ * @param {object|false} [options.cors] - `@fastify/cors` options. On by default reflecting the caller's origin; pass `false` to disable. Override `origin` for a credentialed API.
+ * @param {Function} [options.routes] - The app's route plugin, registered last.
+ * @param {Array<Function|[Function, object]>} [options.plugins] - Extra plugins, in order.
+ * @param {string[]} [options.requestProperties] - Names decorated as null request slots, filled per request.
+ * @param {boolean} [options.captureRawBody=false] - Keep the exact bytes on `req.rawBody`.
+ * @param {Array} [options.ajvPlugins] - ajv plugins, e.g. `[ajvFormats]` for `format` support.
+ * @param {object} [options.ajvOptions] - Merged over ajv's `customOptions`.
+ * @param {Function} [options.genReqId] - Correlation id strategy. Defaults to the kit's.
+ * @param {object} [options.fastify] - Merged last into the Fastify constructor options.
+ * @return {Promise<import('fastify').FastifyInstance>} The built instance, not listening.
+ */
+export async function createApp({
+  logger,
+  context,
+  helmet,
+  cors,
+  routes,
+  plugins = [],
+  requestProperties = [],
+  captureRawBody = false,
+  ajvPlugins = [],
+  ajvOptions = {},
+  genReqId: genReqIdOption = genReqId,
+  fastify: fastifyOptions = {},
+} = {}) {
+  const app = Fastify({
+    ...(logger
+      ? {
+          loggerInstance: logger,
+          // Deriving from `parent` is what keeps Fastify's req/res serializers on
+          // the line; a child built off any other logger dumps the raw socket.
+          childLoggerFactory(parent, bindings, opts) {
+            return parent.child({ ...bindings, type: LOG_TYPE.REQUEST }, opts);
+          },
+        }
+      : { logger: false }),
+    genReqId: genReqIdOption,
+    ajv: {
+      customOptions: ajvOptions,
+      plugins: ajvPlugins,
+    },
+    ...fastifyOptions,
+  });
+
+  // `app.log` is Fastify's wrapper around `logger`, the layer holding the req/res
+  // serializers. Re-binding it here, after the routes took their own logger from
+  // the options above, reaches only the instance's lines — server listening, plugin
+  // warnings; at construction it would leave `type` twice on every request line.
+  if (logger) app.log = withType(app.log, LOG_TYPE.LIFECYCLE);
+
+  await app.register(decorators, { properties: requestProperties });
+  await app.register(requestContext, { context });
+  await app.register(requestId);
+  if (captureRawBody) await app.register(rawBody);
+  await app.register(errorHandler);
+
+  if (helmet !== false) {
+    await app.register(helmetPlugin, helmet && helmet !== true ? { ...HELMET_DEFAULTS, ...helmet } : HELMET_DEFAULTS);
+  }
+  if (cors !== false) {
+    await app.register(corsPlugin, cors && cors !== true ? { origin: true, ...cors } : { origin: true });
+  }
+
+  for (const entry of plugins) {
+    const [plugin, options] = Array.isArray(entry) ? entry : [entry];
+    await app.register(plugin, options);
+  }
+  if (routes) await app.register(routes);
+
+  return app;
+}

package/http/index.js

@@ -0,0 +1,13 @@
+export { createApp } from './createApp.js';
+export { default as decorators } from './plugins/decorators.js';
+export { default as errorHandler, classifyError, STATUS_BY_CODE } from './plugins/errorHandler.js';
+export { default as rawBody } from './plugins/rawBody.js';
+export { default as requestContext } from './plugins/requestContext.js';
+export {
+  default as requestId,
+  genReqId,
+  isUuid,
+  REQUEST_ID_HEADER,
+  UUID_PATTERN,
+} from './plugins/requestId.js';
+export * from './schema.js';

package/http/plugins/decorators.js

@@ -0,0 +1,9 @@
+import fp from 'fastify-plugin';
+
+/**
+ * Declares request-scoped properties as empty (null) slots, filled per request.
+ */
+export default fp(async function decorators(app, { properties = [] } = {}) {
+  // A shared non-null default would bleed state across requests.
+  for (const name of properties) app.decorateRequest(name, null);
+});