@devindex/api-kit 0.1.0

package/http/plugins/errorHandler.js

@@ -0,0 +1,124 @@
+import fp from 'fastify-plugin';
+import { ERROR_CODE, isDomainError } from '../../errors/index.js';
+
+export const STATUS_BY_CODE = Object.freeze({
+  [ERROR_CODE.VALIDATION_ERROR]: 422,
+  [ERROR_CODE.UNAUTHORIZED]: 401,
+  [ERROR_CODE.FORBIDDEN]: 403,
+  [ERROR_CODE.NOT_FOUND]: 404,
+  [ERROR_CODE.METHOD_NOT_ALLOWED]: 405,
+  [ERROR_CODE.CONFLICT]: 409,
+  [ERROR_CODE.LIMIT_REACHED]: 402,
+  [ERROR_CODE.PAYLOAD_TOO_LARGE]: 413,
+  [ERROR_CODE.TOO_MANY_REQUESTS]: 429,
+  [ERROR_CODE.UNAVAILABLE]: 503,
+  [ERROR_CODE.DOMAIN_ERROR]: 400,
+  [ERROR_CODE.INTERNAL_ERROR]: 500,
+});
+
+// A Fastify/plugin error carrying only an HTTP status must not be labelled a
+// validation failure: the envelope's `code` follows the status, so a 429 reads
+// TOO_MANY_REQUESTS and a 403 reads FORBIDDEN instead of all collapsing to one.
+const CODE_BY_STATUS = Object.freeze({
+  400: ERROR_CODE.VALIDATION_ERROR,
+  401: ERROR_CODE.UNAUTHORIZED,
+  402: ERROR_CODE.LIMIT_REACHED,
+  403: ERROR_CODE.FORBIDDEN,
+  404: ERROR_CODE.NOT_FOUND,
+  405: ERROR_CODE.METHOD_NOT_ALLOWED,
+  409: ERROR_CODE.CONFLICT,
+  413: ERROR_CODE.PAYLOAD_TOO_LARGE,
+  422: ERROR_CODE.VALIDATION_ERROR,
+  429: ERROR_CODE.TOO_MANY_REQUESTS,
+});
+
+// Own-property only: an app code spelled `constructor` or `toString` would
+// otherwise resolve up the prototype chain and hand Fastify a function as status.
+function statusForCode(code) {
+  return Object.hasOwn(STATUS_BY_CODE, code) ? STATUS_BY_CODE[code] : 400;
+}
+
+function schemaDetails(validation = []) {
+  return validation.map((issue) => ({
+    path: issue.instancePath || issue.params?.missingProperty || '',
+    message: issue.message,
+  }));
+}
+
+/**
+ * Maps an error to its response shape. A domain error's own `status` wins over
+ * `STATUS_BY_CODE`, which is what lets an app code carry a status the kit's
+ * table has never heard of.
+ *
+ * @return {{status: number, code: string, message: string, details: Array<object>}|null}
+ *   Null when the error is unrecognized; treat it as a 500.
+ */
+export function classifyError(error) {
+  if (isDomainError(error)) {
+    return {
+      status: error.status ?? statusForCode(error.code),
+      code: error.code,
+      message: error.message,
+      details: error.details ?? [],
+    };
+  }
+
+  if (error?.validation) {
+    return {
+      // Same code as a domain ValidationError, so the same status (STATUS_BY_CODE).
+      status: STATUS_BY_CODE[ERROR_CODE.VALIDATION_ERROR],
+      code: ERROR_CODE.VALIDATION_ERROR,
+      message: 'Request validation failed',
+      details: schemaDetails(error.validation),
+    };
+  }
+
+  if (Number.isInteger(error?.statusCode) && error.statusCode >= 400 && error.statusCode < 500) {
+    return {
+      status: error.statusCode,
+      code: CODE_BY_STATUS[error.statusCode] ?? ERROR_CODE.DOMAIN_ERROR,
+      message: error.message,
+      details: [],
+    };
+  }
+
+  return null;
+}
+
+function envelope(known, requestId) {
+  return {
+    error: {
+      code: known.code,
+      message: known.message,
+      details: known.details ?? [],
+      requestId,
+    },
+  };
+}
+
+export default fp(async function errorHandler(app) {
+  app.setErrorHandler((error, req, reply) => {
+    const known = classifyError(error);
+    if (!known) {
+      req.log.error({ err: error }, 'unhandled request error');
+      reply.status(500).send(envelope({
+        code: ERROR_CODE.INTERNAL_ERROR,
+        message: 'Internal server error',
+        details: [],
+      }, req.id));
+      return;
+    }
+
+    const level = known.status >= 500 ? 'error' : 'info';
+    req.log[level]({ err: error, code: known.code }, 'request failed');
+    reply.status(known.status).send(envelope(known, req.id));
+  });
+
+  app.setNotFoundHandler((req, reply) => {
+    reply.status(404).send(envelope({
+      code: ERROR_CODE.NOT_FOUND,
+      message: `Route ${req.method} ${req.url} not found`,
+      details: [],
+    }, req.id));
+  });
+});

package/http/plugins/rawBody.js

@@ -0,0 +1,23 @@
+import fp from 'fastify-plugin';
+
+/** Optional exact-body capture for webhook signature verification. */
+export default fp(async function rawBody(app) {
+  if (!app.hasRequestDecorator('rawBody')) app.decorateRequest('rawBody', null);
+  app.addContentTypeParser(
+    'application/json',
+    { parseAs: 'buffer' },
+    (req, body, done) => {
+      req.rawBody = body;
+      if (body.length === 0) {
+        done(null, undefined);
+        return;
+      }
+      try {
+        done(null, JSON.parse(body.toString('utf8')));
+      } catch (error) {
+        error.statusCode = 400;
+        done(error);
+      }
+    },
+  );
+});

package/http/plugins/requestContext.js

@@ -0,0 +1,9 @@
+import fp from 'fastify-plugin';
+
+/** Makes request/correlation ids available below the HTTP layer via ALS. */
+export default fp(async function requestContext(app, { context } = {}) {
+  if (!context) return;
+  app.addHook('onRequest', (req, _reply, done) => {
+    context.run({ requestId: req.id, correlationId: req.id }, done);
+  });
+});

package/http/plugins/requestId.js

@@ -0,0 +1,21 @@
+import { randomUUID } from 'node:crypto';
+import fp from 'fastify-plugin';
+
+export const REQUEST_ID_HEADER = 'x-request-id';
+export const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+
+export function isUuid(value) {
+  return typeof value === 'string' && UUID_PATTERN.test(value);
+}
+
+/** Only a valid UUID is trusted as an inbound correlation id. */
+export function genReqId(req) {
+  const inbound = req.headers[REQUEST_ID_HEADER];
+  return isUuid(inbound) ? inbound : randomUUID();
+}
+
+export default fp(async function requestId(app) {
+  app.addHook('onSend', async (req, reply) => {
+    reply.header(REQUEST_ID_HEADER, req.id);
+  });
+});

package/http/schema.js

@@ -0,0 +1,16 @@
+export function objectSchema(properties, required = []) {
+  return { type: 'object', properties, required };
+}
+
+export function pageQuery({ maxLimit = 100, defaultLimit = 20 } = {}) {
+  return objectSchema({
+    limit: { type: 'integer', minimum: 1, maximum: maxLimit, default: defaultLimit },
+    offset: { type: 'integer', minimum: 0, default: 0 },
+  });
+}
+
+export const stringSchema = (options = {}) => ({ type: 'string', minLength: 1, ...options });
+export const email = Object.freeze({ type: 'string', format: 'email' });
+export const dateTime = Object.freeze({ type: 'string', format: 'date-time' });
+export const dateKey = Object.freeze({ type: 'string', pattern: '^\\d{4}-\\d{2}-\\d{2}$' });
+export const clock = Object.freeze({ type: 'string', pattern: '^([01]\\d|2[0-3]):[0-5]\\d$' });

package/internal/bullmq.js

@@ -0,0 +1,53 @@
+import { createHash } from 'node:crypto';
+
+export async function loadBullmq() {
+  try {
+    return await import('bullmq');
+  } catch (error) {
+    if (['ERR_MODULE_NOT_FOUND', 'MODULE_NOT_FOUND'].includes(error?.code)) {
+      throw new Error(
+        'BullMQ support requires `bullmq` and `ioredis`: install both packages',
+        { cause: error },
+      );
+    }
+    throw error;
+  }
+}
+
+export function redisConnection(redisUrl, { worker = false } = {}) {
+  if (!redisUrl) throw new Error('a Redis connection requires `redisUrl`');
+  let url;
+  try {
+    url = new URL(redisUrl);
+  } catch (error) {
+    throw new Error(`invalid redisUrl: ${redisUrl}`, { cause: error });
+  }
+  if (!['redis:', 'rediss:'].includes(url.protocol)) {
+    throw new Error(`redisUrl must use redis:// or rediss://, got ${url.protocol}`);
+  }
+  const db = url.pathname.length > 1 ? Number(url.pathname.slice(1)) : 0;
+  if (!Number.isInteger(db) || db < 0) throw new Error(`invalid Redis database in ${redisUrl}`);
+
+  return {
+    host: url.hostname,
+    port: Number(url.port || 6379),
+    db,
+    ...(url.username ? { username: decodeURIComponent(url.username) } : {}),
+    ...(url.password ? { password: decodeURIComponent(url.password) } : {}),
+    ...(url.protocol === 'rediss:' ? { tls: {} } : {}),
+    // BullMQ workers refuse to start unless this is null; producers fail fast.
+    maxRetriesPerRequest: worker ? null : 1,
+  };
+}
+
+function safeSegment(value) {
+  const source = String(value);
+  // Slugifying is lossy; the hash suffix keeps the queue name unique.
+  const readable = source.toLowerCase().replace(/[^a-z0-9_-]+/g, '-').replace(/^-|-$/g, '') || 'work';
+  const hash = createHash('sha256').update(source).digest('hex').slice(0, 10);
+  return `${readable.slice(0, 40)}-${hash}`;
+}
+
+export function queueName(prefix, kind, name) {
+  return `${safeSegment(prefix)}-${kind}-${safeSegment(name)}`;
+}

package/internal/logger.js

@@ -0,0 +1,6 @@
+const LOG_LEVELS = ['fatal', 'error', 'warn', 'info', 'debug', 'trace'];
+
+export const noopLogger = Object.freeze({
+  ...Object.fromEntries(LOG_LEVELS.map((level) => [level, () => {}])),
+  child: () => noopLogger,
+});

package/internal/retry.js

@@ -0,0 +1,42 @@
+import { createHash } from 'node:crypto';
+import { assertDuration, assertPositiveInt } from './validation.js';
+
+export const DEFAULTS = Object.freeze({
+  attempts: 3,
+  backoff: { type: 'exponential', delay: 1_000 },
+  concurrency: 1,
+});
+
+export function dedupId(name, key) {
+  // NUL separates the fields so ("ab","c") and ("a","bc") can't collide.
+  return createHash('sha256').update(`${name}\0${key}`).digest('hex');
+}
+
+export function normalizeBackoff(backoff) {
+  if (typeof backoff === 'number') return assertDuration('backoff', backoff);
+  if (!backoff || !['fixed', 'exponential'].includes(backoff.type)) {
+    throw new TypeError('backoff must be milliseconds or `{ type: "fixed"|"exponential", delay }`');
+  }
+  return Object.freeze({
+    type: backoff.type,
+    delay: assertDuration('backoff.delay', backoff.delay),
+  });
+}
+
+// setTimeout clamps anything past ~24.8 days to 1ms, turning a slow backoff into
+// a hot retry loop; keep the delay under that ceiling so large `attempts` stay spaced.
+const MAX_BACKOFF_MS = 2_147_483_647;
+
+export function backoffDelay(backoff, attempt) {
+  if (typeof backoff === 'number') return backoff;
+  if (backoff.type !== 'exponential') return backoff.delay;
+  return Math.min(MAX_BACKOFF_MS, backoff.delay * 2 ** (attempt - 1));
+}
+
+export function resolveDefinition(defaults, options = {}) {
+  return Object.freeze({
+    attempts: assertPositiveInt('attempts', options.attempts ?? defaults.attempts),
+    backoff: normalizeBackoff(options.backoff ?? defaults.backoff),
+    concurrency: assertPositiveInt('concurrency', options.concurrency ?? defaults.concurrency),
+  });
+}

package/internal/validation.js

@@ -0,0 +1,34 @@
+export function assertName(name, subject = 'job') {
+  if (typeof name !== 'string' || name.trim().length === 0) {
+    throw new TypeError(`a ${subject} needs a non-empty name`);
+  }
+  return name.trim();
+}
+
+export function assertKey(key) {
+  if (typeof key !== 'string' || key.length === 0) {
+    throw new TypeError(`a \`key\` must be a non-empty string, got ${typeof key === 'string' ? "''" : typeof key}`);
+  }
+  return key;
+}
+
+export function assertHandler(name, handler) {
+  if (typeof handler !== 'function') {
+    throw new TypeError(`the handler for "${name}" must be a function`);
+  }
+  return handler;
+}
+
+export function assertPositiveInt(label, value) {
+  if (!Number.isInteger(value) || value < 1) {
+    throw new TypeError(`${label} must be an integer >= 1, got ${value}`);
+  }
+  return value;
+}
+
+export function assertDuration(label, value) {
+  if (!Number.isInteger(value) || value < 0) {
+    throw new TypeError(`${label} must be an integer >= 0 milliseconds, got ${value}`);
+  }
+  return value;
+}

package/jobs/drivers/bullmq.js

@@ -0,0 +1,122 @@
+import {
+  loadBullmq,
+  queueName,
+  redisConnection,
+} from '../../internal/bullmq.js';
+import { noopLogger } from '../../internal/logger.js';
+
+/**
+ * Distributed backend on BullMQ, one queue per job name. Redis is only its store.
+ *
+ * @param {object} options
+ * @param {string} options.redisUrl
+ * @param {string} [options.prefix='app']
+ * @param {object} [options.logger]
+ * @return {object} A backend: add, work, stop and idle.
+ */
+export function bullmqBackend({ redisUrl, prefix = 'app', logger = noopLogger } = {}) {
+  const queueConnection = redisConnection(redisUrl);
+  const workerConnection = redisConnection(redisUrl, { worker: true });
+  const queuePromises = new Map();
+  const workers = new Set();
+
+  function getQueue(name) {
+    let promise = queuePromises.get(name);
+    if (!promise) {
+      promise = (async () => {
+        const { Queue } = await loadBullmq();
+        return new Queue(queueName(prefix, 'job', name), {
+          connection: queueConnection,
+          defaultJobOptions: {
+            removeOnComplete: { age: 24 * 60 * 60, count: 1_000 },
+            removeOnFail: { age: 7 * 24 * 60 * 60 },
+          },
+        });
+      })();
+      queuePromises.set(name, promise);
+    }
+    return promise;
+  }
+
+  return {
+    async add(name, payload, { key, dedupId, attempts, backoff, delay = 0 }) {
+      const queue = await getQueue(name);
+      const job = await queue.add(name, { payload, key }, {
+        attempts,
+        backoff,
+        deduplication: { id: dedupId },
+        ...(delay > 0 ? { delay } : {}),
+      });
+      return job.id;
+    },
+
+    async work(name, definition) {
+      const { Worker } = await loadBullmq();
+      const queue = await getQueue(name);
+      const worker = new Worker(queue.name, async (job, _token, signal) => {
+        if (job.name !== name) throw new Error(`job "${job.name}" reached the worker for "${name}"`);
+        const attempt = job.attemptsMade + 1;
+        const totalAttempts = job.opts.attempts ?? 1;
+        const log = logger.child({ job: name, key: job.data.key, jobId: job.id });
+        try {
+          return await definition.handler(job.data.payload, {
+            name,
+            jobId: job.id,
+            key: job.data.key,
+            attempt,
+            attemptsLeft: totalAttempts - attempt,
+            signal,
+            log,
+          });
+        } catch (error) {
+          // A retry that BullMQ will re-run is a warning; only the last attempt is
+          // an error, so expected transient retries don't trip error-rate alerts.
+          const final = attempt >= totalAttempts;
+          log[final ? 'error' : 'warn']({ err: error, attempt, final }, 'job attempt failed');
+          throw error;
+        }
+      }, { connection: workerConnection, concurrency: definition.concurrency });
+      workers.add(worker);
+      worker.on('error', (error) => logger.error({ err: error, job: name }, 'worker error'));
+      worker.on('stalled', (jobId) => {
+        logger.error({ job: name, jobId }, 'job stalled and will be delivered again');
+      });
+
+      // Surface a worker that cannot reach Redis at start(), not silently later.
+      await worker.waitUntilReady();
+    },
+
+    async idle() {
+      while (true) {
+        const queues = await Promise.all([...queuePromises.values()]);
+        const remaining = await Promise.all(
+          queues.map((queue) => queue.getJobCountByTypes(
+            'active',
+            'waiting',
+            'delayed',
+            'prioritized',
+            'waiting-children',
+          )),
+        );
+        if (remaining.every((count) => count === 0)) return;
+        await new Promise((resolve) => setTimeout(resolve, 50));
+      }
+    },
+
+    // BullMQ owns the drain deadline once worker.close() starts, so the queue's
+    // timeoutMs is accepted for a uniform signature but not used here.
+    async stop() {
+      // Workers first: they stop taking jobs and let the running ones finish.
+      const workerResults = await Promise.allSettled([...workers].map((worker) => worker.close()));
+      const queueResults = await Promise.allSettled([...queuePromises.values()]);
+      const closeResults = await Promise.allSettled(
+        queueResults.flatMap((result) => (result.status === 'fulfilled' ? [result.value.close()] : [])),
+      );
+      for (const result of [...workerResults, ...queueResults, ...closeResults]) {
+        if (result.status === 'rejected') logger.error({ err: result.reason }, 'job queue close failed');
+      }
+      workers.clear();
+      queuePromises.clear();
+    },
+  };
+}

package/jobs/drivers/memory.js

@@ -0,0 +1,178 @@
+import { randomUUID } from 'node:crypto';
+import { noopLogger } from '../../internal/logger.js';
+import { backoffDelay } from '../../internal/retry.js';
+
+/**
+ * In-process backend for tests and local runs. Honors delay, attempts, backoff,
+ * deduplication and concurrency, and keeps nothing once the process exits.
+ *
+ * @param {object} [options]
+ * @param {object} [options.logger]
+ * @return {object} A backend: add, work, stop and idle.
+ */
+export function memoryBackend({ logger = noopLogger } = {}) {
+  const consumers = new Map();
+  const lanes = new Map();
+  const dedup = new Map();
+  const timers = new Set();
+  const inFlight = new Set();
+  const shutdown = new AbortController();
+  let closed = false;
+
+  /** Get or create the per-name queue: its backlog, head pointer and running count. */
+  function lane(name) {
+    let entry = lanes.get(name);
+    if (!entry) {
+      entry = { waiting: [], head: 0, running: 0 };
+      lanes.set(name, entry);
+    }
+    return entry;
+  }
+
+  /** Resolve after `ms`, or early if the queue is stopped, without leaking the timer. */
+  function sleep(ms) {
+    if (ms === 0) return Promise.resolve();
+    return new Promise((resolve) => {
+      const done = () => {
+        clearTimeout(timer);
+        timers.delete(timer);
+        shutdown.signal.removeEventListener('abort', done);
+        resolve();
+      };
+      const timer = setTimeout(done, ms);
+      timers.add(timer);
+      shutdown.signal.addEventListener('abort', done, { once: true });
+    });
+  }
+
+  /** Run one job through its handler, retrying with backoff until it succeeds or exhausts attempts. */
+  async function run(name, definition, job) {
+    const log = logger.child({ job: name, key: job.key, jobId: job.id });
+    try {
+      for (let attempt = 1; attempt <= definition.attempts; attempt += 1) {
+        try {
+          await definition.handler(job.payload, {
+            name,
+            jobId: job.id,
+            key: job.key,
+            attempt,
+            attemptsLeft: definition.attempts - attempt,
+            signal: shutdown.signal,
+            log,
+          });
+          return;
+        } catch (error) {
+          // A retry that will run again is a warning; only the last failure is an
+          // error, so expected transient retries don't trip error-rate alerts.
+          const final = attempt === definition.attempts || shutdown.signal.aborted;
+          log[final ? 'error' : 'warn']({ err: error, attempt, final }, 'job attempt failed');
+          if (final) return;
+          await sleep(backoffDelay(definition.backoff, attempt));
+          if (shutdown.signal.aborted) return;
+        }
+      }
+    } finally {
+      if (dedup.get(job.dedupId) === job.id) dedup.delete(job.dedupId);
+    }
+  }
+
+  /** Start as many queued jobs for `name` as concurrency allows, re-pumping as each finishes. */
+  function pump(name) {
+    const definition = consumers.get(name);
+    // A producer enqueues names this process never declared; without a local
+    // worker there is nothing here to run them.
+    if (closed || !definition) return;
+    const entry = lane(name);
+    while (entry.running < definition.concurrency && entry.head < entry.waiting.length) {
+      const job = entry.waiting[entry.head];
+      entry.waiting[entry.head] = undefined; // drop the payload reference for GC
+      entry.head += 1;
+      // `shift()` reindexes the whole backlog (O(n^2) under a burst); a head
+      // pointer keeps dequeue O(1), compacted so it can't grow unbounded.
+      if (entry.head > 1024 && entry.head * 2 >= entry.waiting.length) {
+        entry.waiting = entry.waiting.slice(entry.head);
+        entry.head = 0;
+      }
+      entry.running += 1;
+      const promise = run(name, definition, job).finally(() => {
+        entry.running -= 1;
+        inFlight.delete(promise);
+        pump(name);
+      });
+      inFlight.add(promise);
+    }
+  }
+
+  return {
+    async add(name, payload, { key, dedupId, delay = 0 }) {
+      if (closed) throw new Error('cannot enqueue on a stopped job queue');
+      // Dedup is held only while the job is in flight, so the same key is free
+      // again once the job settles.
+      const held = dedup.get(dedupId);
+      if (held) return held;
+
+      const jobId = randomUUID();
+      dedup.set(dedupId, jobId);
+      const job = { id: jobId, dedupId, key, payload };
+
+      if (delay > 0) {
+        const timer = setTimeout(() => {
+          timers.delete(timer);
+          if (closed) return;
+          lane(name).waiting.push(job);
+          pump(name);
+        }, delay);
+        timers.add(timer);
+      } else {
+        lane(name).waiting.push(job);
+        pump(name);
+      }
+      return jobId;
+    },
+
+    async work(name, definition) {
+      // Draining here picks up anything enqueued before the worker existed.
+      consumers.set(name, definition);
+      pump(name);
+    },
+
+    async stop({ timeoutMs = 30_000 } = {}) {
+      if (closed) return;
+      closed = true;
+      shutdown.abort();
+      for (const timer of timers) clearTimeout(timer);
+      timers.clear();
+      for (const entry of lanes.values()) {
+        entry.waiting.length = 0;
+        entry.head = 0;
+      }
+
+      // A handler that ignores `signal` would otherwise keep the process alive
+      // forever, so drain in-flight work under a deadline instead of unbounded.
+      const drained = await Promise.race([
+        Promise.allSettled([...inFlight]).then(() => true),
+        new Promise((resolve) => {
+          setTimeout(() => resolve(false), timeoutMs).unref?.();
+        }),
+      ]);
+      if (!drained) {
+        logger.warn({ timeoutMs, pending: inFlight.size }, 'job queue stop timed out; abandoning in-flight jobs');
+      }
+
+      lanes.clear();
+      dedup.clear();
+      inFlight.clear();
+    },
+
+    /** Resolves once nothing is delayed, queued or running. */
+    async idle() {
+      while (
+        timers.size
+        || inFlight.size
+        || [...lanes.values()].some((entry) => entry.head < entry.waiting.length)
+      ) {
+        await new Promise((resolve) => setTimeout(resolve, 5));
+      }
+    },
+  };
+}