@oneunit/redis 0.0.0-stage → 1.0.0

package/src/logger.ts

@@ -0,0 +1,159 @@
+const LOGGER_KEYS = ["error", "warn", "info", "debug"] as const;
+
+export interface Logger {
+  error(message: unknown, extra?: unknown): void;
+  warn(message: unknown, extra?: unknown): void;
+  info(message: unknown, extra?: unknown): void;
+  debug(message: unknown, extra?: unknown): void;
+  child?(bindings?: Record<string, unknown>): Logger;
+}
+
+export const silentLogger: Logger = Object.freeze({
+  error() {},
+  warn() {},
+  info() {},
+  debug() {},
+});
+
+export const consoleLogger: Logger = Object.freeze({
+  error(message: unknown, extra?: unknown) {
+    write(console.error, message, extra);
+  },
+  warn(message: unknown, extra?: unknown) {
+    write(console.warn, message, extra);
+  },
+  info(message: unknown, extra?: unknown) {
+    write(console.info, message, extra);
+  },
+  debug(message: unknown, extra?: unknown) {
+    write(console.debug, message, extra);
+  },
+});
+
+/**
+ * Does this logger take `(bindings, message)` instead of `(message, extra)`?
+ *
+ * pino and its drop-in forks accept `log.info(bindings, message)`, where the
+ * first argument is merged into the record and the second is the message.
+ * Those signatures are indistinguishable at the call site, so a caller has to
+ * be detected rather than chosen.
+ *
+ * `child()` was the first thing tried here and it is not a usable signal: this
+ * package's own `Logger` interface declares `child?()`, so every conforming
+ * logger is allowed to have one. Swapping arguments for those loggers silently
+ * moved the message into the bindings slot of every record.
+ *
+ * A pino instance exposes both `bindings()` and the `levels` map. `child()`
+ * loggers inherit `bindings()` from the same prototype, so a caller that hands
+ * us `logger.child({ service: "redis" })` is still recognised. Custom loggers
+ * have neither.
+ */
+function isBindingsFirst(logger: Logger): boolean {
+  const candidate = logger as unknown as Record<string, unknown>;
+
+  return (
+    typeof candidate.bindings === "function" &&
+    typeof candidate.levels === "object" &&
+    candidate.levels !== null
+  );
+}
+
+export function isLogger(value: unknown): value is Logger {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+
+  const hasLogFn = LOGGER_KEYS.some(
+    (key) => typeof (value as Record<string, unknown>)[key] === "function",
+  );
+  if (!hasLogFn) {
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Complete a caller-supplied logger, or keep "no logger" as no logging.
+ *
+ * `createLogger` is the public entry point and defaults to the console, which
+ * is right when someone asks for a logger. Internal call sites are different:
+ * every `createClient(url)` in the wild passes nothing and expects silence, so
+ * they need a logger that is either complete or absent. Without this, a logger
+ * that implements only some levels throws `logger?.info is not a function` from
+ * a connection event, where it is least likely to be caught.
+ */
+export function normalizeLogger(logger?: Logger | null): Logger | undefined {
+  if (!logger) {
+    return undefined;
+  }
+
+  return createLogger(logger);
+}
+
+export function createLogger(input?: Logger | null): Logger {
+  if (!input) {
+    return consoleLogger;
+  }
+
+  if (input === silentLogger || input === consoleLogger) {
+    return input;
+  }
+
+  return {
+    error(message: unknown, extra?: unknown) {
+      invoke(input, "error", message, extra);
+    },
+    warn(message: unknown, extra?: unknown) {
+      invoke(input, "warn", message, extra);
+    },
+    info(message: unknown, extra?: unknown) {
+      invoke(input, "info", message, extra);
+    },
+    debug(message: unknown, extra?: unknown) {
+      invoke(input, "debug", message, extra);
+    },
+  };
+}
+
+function write(
+  fn: (...args: unknown[]) => void,
+  message: unknown,
+  extra?: unknown,
+): void {
+  if (extra === undefined) {
+    fn(message);
+    return;
+  }
+  fn(message, extra);
+}
+
+function invoke(
+  logger: Logger,
+  level: "error" | "warn" | "info" | "debug",
+  message: unknown,
+  extra?: unknown,
+): void {
+  const fn =
+    typeof logger[level] === "function"
+      ? logger[level]
+      : typeof logger.info === "function"
+        ? logger.info
+        : undefined;
+
+  if (typeof fn !== "function") {
+    return;
+  }
+
+  if (extra === undefined) {
+    fn.call(logger, message);
+    return;
+  }
+
+  if (isBindingsFirst(logger)) {
+    fn.call(logger, extra, message);
+    return;
+  }
+
+  fn.call(logger, message, extra);
+}

package/src/pipeline/builder.ts

@@ -0,0 +1,307 @@
+import { type Redis as RedisClient, type ChainableCommander } from "ioredis";
+import { normalizeLogger, type Logger } from "../logger.js";
+import { redactError } from "../client/events.js";
+
+/**
+ * One command in a pipeline, plus a label for reporting failures.
+ *
+ * The label is what makes a failure identifiable. A pipeline result is a
+ * positional array, so a bare `results[7]` tells an operator nothing about which
+ * of several hundred commands went wrong; ioredis attaches the command itself to
+ * the error, but only for the commands it can describe, and the label is what
+ * this package logs.
+ */
+export interface PipelineStep {
+  label: string;
+  run: (pipeline: ChainableCommander) => void;
+}
+
+export interface PipelineOptions {
+  /**
+   * Milliseconds to wait for EXEC before giving up.
+   *
+   * ioredis queues commands while reconnecting and never flushes them until the
+   * connection is back, so EXEC against an unreachable server does not reject —
+   * it never settles. Without a bound a request handler awaiting a pipeline
+   * hangs for the whole outage and takes the process's request capacity with it.
+   */
+  timeout?: number;
+  logger?: Logger;
+  /**
+   * Reject if any single command in the batch failed.
+   *
+   * Off by default. EXEC resolves even when individual commands fail: each
+   * failure arrives as a `[error, null]` tuple in the results, and a pipeline
+   * that silently drops one of its writes looks exactly like a pipeline that
+   * succeeded. Turn this on where a partial batch is not acceptable.
+   */
+  throwOnError?: boolean;
+}
+
+/** Result of one command. Mirrors ioredis's own tuple, with the error redacted. */
+export interface PipelineStepResult {
+  label: string;
+  value: unknown;
+  error?: Error;
+}
+
+export interface PipelineResult {
+  results: PipelineStepResult[];
+  /** Wall-clock duration of EXEC, in milliseconds. */
+  durationMs: number;
+  /** How many commands failed. Zero when all succeeded. */
+  failed: number;
+}
+
+const DEFAULT_TIMEOUT_MS = 5000;
+
+/** Whether a value is promise-like, i.e. something a caller forgot to await. */
+function isThenable(value: unknown): boolean {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    typeof (value as { then?: unknown }).then === "function"
+  );
+}
+
+export class PipelineTimeoutError extends Error {
+  readonly steps: number;
+
+  constructor(timeout: number, steps: number) {
+    super(
+      `Redis pipeline of ${steps} command${steps === 1 ? "" : "s"} did not complete within ${timeout}ms`,
+    );
+    this.name = "PipelineTimeoutError";
+    this.steps = steps;
+  }
+}
+
+/**
+ * Thrown when `throwOnError` is set and a command failed.
+ *
+ * Carries the per-step results so the caller can see which command broke
+ * without re-running the batch. The `message` lists labels only — ioredis
+ * attaches command arguments to its errors, and a pipeline that batched an
+ * `AUTH` would otherwise put the password in an exception message.
+ */
+export class PipelineCommandError extends Error {
+  readonly results: PipelineStepResult[];
+
+  constructor(results: PipelineStepResult[]) {
+    const failed = results.filter((result) => result.error);
+    super(
+      `Redis pipeline: ${failed.length} of ${results.length} commands failed (${failed
+        .map((result) => result.label)
+        .join(", ")})`,
+    );
+    this.name = "PipelineCommandError";
+    this.results = results;
+  }
+}
+
+/**
+ * Thrown when a step does not queue exactly one command.
+ *
+ * `PipelineStep.run` is typed as returning `void`, and TypeScript allows any
+ * value to be returned from a `void` signature — so a step that queues two
+ * commands, or an `async` step whose command is only queued after `runPipeline`
+ * has already called `exec`, compiles without complaint. Both shift the
+ * positional pairing between `steps` and the result tuples, which is the one
+ * thing the label exists to prevent, so the batch is rejected rather than
+ * returned with a value against the wrong label.
+ */
+export class PipelineStepError extends Error {
+  readonly label: string;
+  readonly queued: number;
+
+  constructor(label: string, queued: number) {
+    super(
+      `Redis pipeline step "${label}" queued ${queued} command${
+        queued === 1 ? "" : "s"
+      }, expected exactly 1`,
+    );
+    this.name = "PipelineStepError";
+    this.label = label;
+    this.queued = queued;
+  }
+}
+
+/**
+ * Run a batch of commands in one round trip.
+ *
+ * Wraps `client.pipeline()` rather than reimplementing it. The value is in the
+ * four sharp edges this closes, all of which are silent in ioredis itself:
+ *
+ * 1. **A failed command does not fail the pipeline.** EXEC resolves with a
+ *    `[error, null]` tuple for the command that failed. Code that reads
+ *    `results.map(([, value]) => value)` gets `null` and carries on as if the
+ *    write landed. Every result here carries an explicit `error` field.
+ * 2. **EXEC can hang forever.** ioredis parks queued commands while
+ *    reconnecting. `timeout` bounds that.
+ * 3. **Command errors carry their arguments.** Redacted via `redactError`
+ *    before they reach a logger or an exception — on both the per-command
+ *    results and a rejected `exec()`.
+ * 4. **Results are positional.** Each step must queue exactly one command, or
+ *    every later label is paired with the wrong value. A step that does not
+ *    raises `PipelineStepError`.
+ */
+export async function runPipeline(
+  client: RedisClient,
+  steps: PipelineStep[],
+  options: PipelineOptions = {},
+): Promise<PipelineResult> {
+  const logger = normalizeLogger(options.logger);
+  const requested = options.timeout ?? DEFAULT_TIMEOUT_MS;
+  // setTimeout coerces a negative or NaN budget to 1ms and prints a warning for
+  // each; treat any invalid value as "no bound configured" instead.
+  const timeout =
+    Number.isFinite(requested) && requested > 0
+      ? requested
+      : DEFAULT_TIMEOUT_MS;
+
+  if (steps.length === 0) {
+    // No round trip is needed, and building a pipeline just to exec an empty
+    // one is a wasted allocation.
+    return { results: [], durationMs: 0, failed: 0 };
+  }
+
+  const pipeline = client.pipeline();
+
+  for (const step of steps) {
+    // Results are paired with steps by position, so the queue has to grow by
+    // exactly one per step. `length` is ioredis's own queue length, and it is
+    // read before and after rather than counted here so the check is against
+    // what was actually queued, not against what `run` appears to have done.
+    const before = pipeline.length;
+
+    let returned: unknown;
+
+    try {
+      returned = step.run(pipeline);
+    } catch (error) {
+      // A throwing `run` means the command was never queued, so the batch is
+      // already malformed. Fail the whole pipeline: silently dropping the
+      // command would shift every later result by one and return the wrong
+      // value against the wrong label.
+      logger?.error(`Redis pipeline step failed to queue: ${step.label}`, {
+        err: redactError(error),
+      });
+
+      throw error;
+    }
+
+    const queued = pipeline.length - before;
+
+    if (isThenable(returned)) {
+      // An `async` step is invisible to the count below: it queues nothing
+      // synchronously, so `exec()` runs first and the command it was going to
+      // queue lands in a pipeline that has already been sent. TypeScript
+      // permits returning a value from a `void`-typed signature, so this is
+      // not a compile error either.
+      //
+      // The step's promise is abandoned here, so its eventual rejection is
+      // nobody's to handle — which on Node 20 is a process-level crash that
+      // would mask the error naming the actual fault. Swallow it; the throw
+      // below is the real diagnosis.
+      void Promise.resolve(returned).catch(() => undefined);
+
+      logger?.error(
+        `Redis pipeline step returned a promise instead of queuing: ${step.label}`,
+      );
+
+      throw new PipelineStepError(step.label, queued);
+    }
+
+    if (queued !== 1) {
+      logger?.error(
+        `Redis pipeline step queued ${queued} commands: ${step.label}`,
+      );
+
+      throw new PipelineStepError(step.label, queued);
+    }
+  }
+
+  const started = performance.now();
+
+  let timer: NodeJS.Timeout | undefined;
+  const expiry = new Promise<"timeout">((resolve) => {
+    timer = setTimeout(() => resolve("timeout"), timeout);
+  });
+
+  let raw: Array<[Error | null, unknown]>;
+
+  try {
+    const outcome = await Promise.race([pipeline.exec(), expiry]);
+
+    if (outcome === "timeout") {
+      throw new PipelineTimeoutError(timeout, steps.length);
+    }
+
+    raw = outcome as Array<[Error | null, unknown]>;
+  } catch (error) {
+    if (error instanceof PipelineTimeoutError) {
+      throw error;
+    }
+
+    // A rejected `exec()` is a connection-level failure, and it bypasses the
+    // per-tuple redaction below entirely. The module documents that errors are
+    // redacted, so the rejection goes through the same path rather than
+    // reaching the caller — and from there a logger — as ioredis built it.
+    throw redactError(error);
+  } finally {
+    // A pending timeout keeps the event loop alive. Always clear it, including
+    // on the timeout path where it has already fired.
+    clearTimeout(timer);
+  }
+
+  const results: PipelineStepResult[] = raw.map(([error, value], index) => {
+    const step = steps[index];
+
+    return {
+      label: step?.label ?? `#${index}`,
+      value,
+      // Redacted here rather than at each call site so a caller that logs a
+      // whole result array cannot leak credentials by accident.
+      ...(error ? { error: redactError(error) as Error } : {}),
+    };
+  });
+
+  const failed = results.filter((result) => result.error).length;
+  const durationMs = Math.round(performance.now() - started);
+
+  if (failed > 0) {
+    logger?.warn(
+      `Redis pipeline: ${failed} of ${results.length} commands failed`,
+    );
+  }
+
+  logger?.debug?.(
+    `Redis pipeline completed ${results.length} command${
+      results.length === 1 ? "" : "s"
+    } in ${durationMs}ms`,
+  );
+
+  if (failed > 0 && options.throwOnError) {
+    throw new PipelineCommandError(results);
+  }
+
+  return { results, durationMs, failed };
+}
+
+/**
+ * Values of the successful results, in order.
+ *
+ * Throws if any command failed, because the alternative is handing back a
+ * sparse array whose length matches the batch but whose contents silently
+ * include a failed write. Use `results` directly when a partial batch is
+ * expected and worth handling.
+ */
+export function pipelineValues(results: PipelineStepResult[]): unknown[] {
+  const failed = results.find((result) => result.error);
+
+  if (failed) {
+    throw new PipelineCommandError(results);
+  }
+
+  return results.map((result) => result.value);
+}

package/src/pipeline/index.ts

@@ -0,0 +1,7 @@
+export * from "./builder.js";
+export type {
+  PipelineStep,
+  PipelineOptions,
+  PipelineResult,
+  PipelineStepResult,
+} from "./builder.js";

package/src/queue/events.ts

@@ -0,0 +1,158 @@
+import {
+  ConnectionClosedError,
+  QueueEvents,
+  type QueueEventsOptions,
+} from "bullmq";
+import { Queue } from "bullmq";
+import type { Logger } from "../logger.js";
+import type { Redis } from "ioredis";
+import { normalizeLogger } from "../logger.js";
+import { redactError } from "../client/events.js";
+
+export interface QueueEventsConfig {
+  queue: Queue;
+  logger?: Logger;
+  prefix?: string;
+  connection?: QueueEventsOptions["connection"];
+}
+
+/**
+ * QueueEvents that can be closed after a failed startup.
+ *
+ * BullMQ's own `close()` awaits `this.client` before disconnecting, and that
+ * getter resolves to the connection's `initializing` promise. When the
+ * connection never became ready that promise has already rejected with
+ * "Connection is closed.", so `close()` throws before it reaches
+ * `connection.close()` and the duplicated ioredis client is left running its
+ * reconnect loop. Nothing else in the process can stop it, because the caller
+ * has no reference to the duplicate, so the process never exits.
+ *
+ * `Queue` does not have this problem: `RedisConnection.close()` handles
+ * `status === "initializing"` itself.
+ *
+ * Disconnecting the duplicate first is safe in both directions. When the
+ * connection is healthy, `disconnect()` ends it and the subsequent
+ * `connection.close()` sees a client already at `end` and skips its own quit.
+ */
+class ManagedQueueEvents extends QueueEvents {
+  override async close(): Promise<void> {
+    const connection = this.connection as unknown as {
+      _client?: Redis;
+      close(force?: boolean): Promise<void>;
+    };
+
+    // A duplicate that is already gone does not need disconnecting, and
+    // `disconnect()` is a no-op at `end`.
+    connection._client?.disconnect();
+
+    try {
+      await super.close();
+    } catch (error) {
+      // `super.close()` propagates the rejection from the failed startup
+      // instead of reporting that the connection is now closed. The connection
+      // itself does know how to close from `initializing`, so drive it directly
+      // and only rethrow errors that are not about the connection being gone.
+      await connection.close(true).catch(() => undefined);
+
+      if (!isConnectionGone(error)) {
+        throw error;
+      }
+    }
+  }
+}
+
+/**
+ * True for the "this connection is already gone" family of errors.
+ *
+ * `ConnectionClosedError` is checked first and structurally, because BullMQ
+ * introduced it for exactly this reason — its own comment on the class says it
+ * exists so `isNotConnectionError` can "do a structural `instanceof` check
+ * rather than fragile message-substring matching". Matching the message cannot
+ * work here: only some of BullMQ's construction sites pass ioredis's
+ * `CONNECTION_CLOSED_ERROR_MSG`, and the others pass their own wording or no
+ * message at all, in which case the class default (`"Connection is closed"`,
+ * with no trailing period) applies. An exact string comparison therefore
+ * rethrows precisely the failures this function exists to absorb, and the
+ * caller gets an exception from teardown instead of a clean close.
+ *
+ * The string clauses stay as a fallback: they still cover a `bullmq` error that
+ * predates the class, and an error forwarded from another adapter. `instanceof`
+ * is identity-based, so a consumer with two copies of `bullmq` in one tree
+ * would miss the class check — the fallbacks catch the ioredis wording, and
+ * missing them is the safe direction to fail.
+ */
+function isConnectionGone(error: unknown): boolean {
+  if (error instanceof ConnectionClosedError) {
+    return true;
+  }
+
+  if (!(error instanceof Error)) {
+    return false;
+  }
+
+  return (
+    error.message === "Connection is closed." ||
+    error.message.includes("ECONNREFUSED") ||
+    (error as NodeJS.ErrnoException).code === "ECONNREFUSED"
+  );
+}
+
+export function attachQueueEvents(config: QueueEventsConfig): QueueEvents {
+  const { queue, connection } = config;
+
+  // BullMQ swallows a throwing event listener and re-emits the failure as an
+  // "error" event, which then throws again and lands on console.error. A logger
+  // missing `info` would trigger that on every completed job.
+  const logger = normalizeLogger(config.logger);
+
+  // QueueEvents subscribes to a key derived from the prefix. Defaulting to a
+  // literal "queue" here silently dropped every event for a queue created
+  // with a custom prefix, so inherit the queue's own prefix instead.
+  const prefix = config.prefix ?? queue.opts.prefix ?? "queue";
+
+  const queueEvents = new ManagedQueueEvents(queue.name, {
+    prefix,
+    connection: connection ?? queue.opts.connection,
+  });
+
+  queueEvents.on(
+    "completed",
+    (
+      args: { jobId: string; returnvalue: string; prev?: string },
+      _id: string,
+    ) => {
+      logger?.info(`Job ${args.jobId} completed`);
+    },
+  );
+
+  queueEvents.on(
+    "failed",
+    (
+      args: { jobId: string; failedReason: string; prev?: string },
+      _id: string,
+    ) => {
+      logger?.error(`Job ${args.jobId} failed`, args.failedReason);
+    },
+  );
+
+  queueEvents.on(
+    "progress",
+    (args: { jobId: string; data: unknown }, _id: string) => {
+      logger?.info(`Job ${args.jobId} progress`, args.data);
+    },
+  );
+
+  queueEvents.on("error", (error: Error) => {
+    // QueueEvents duplicates the caller's client, so it authenticates with the
+    // same password and BullMQ re-emits any AUTH failure here. ioredis attaches
+    // the failing command to that error, and for AUTH its args are the password
+    // in plaintext — logging it as-is would write the credential to the app's
+    // log on every reconnect attempt against a misconfigured server.
+    logger?.error(`Queue "${queue.name}" events error`, redactError(error));
+  });
+
+  return queueEvents;
+}
+
+export type { QueueEventsOptions } from "bullmq";
+export { QueueEvents } from "bullmq";

package/src/queue/index.ts

@@ -0,0 +1,6 @@
+export * from "./queue.js";
+export * from "./worker.js";
+export * from "./events.js";
+export type { QueueConfig } from "./queue.js";
+export type { WorkerConfig } from "./worker.js";
+export type { QueueEventsConfig } from "./events.js";

package/src/queue/queue.ts

@@ -0,0 +1,60 @@
+import { Queue, type QueueOptions, type JobsOptions } from "bullmq";
+import { type Redis as RedisClient } from "ioredis";
+
+export interface QueueConfig {
+  name: string;
+  connection: RedisClient;
+  prefix?: string;
+  defaultJobOptions?: JobsOptions;
+  settings?: QueueOptions["settings"];
+}
+
+export function createQueue(config: QueueConfig): Queue {
+  const {
+    name,
+    connection,
+    prefix = "queue",
+    defaultJobOptions,
+    settings,
+    ...options
+  } = config;
+
+  const defaults: JobsOptions = {
+    removeOnComplete: 100,
+    removeOnFail: 1000,
+    attempts: 3,
+    backoff: {
+      type: "exponential",
+      delay: 1000,
+    },
+  };
+
+  // Merged key by key rather than by spreading. A spread writes `undefined` for
+  // every key the caller left unset, which erases the default instead of falling
+  // through to it — and that is how a config built by spreading another object
+  // (`{ ...base, attempts: maybeUndefined }`) quietly loses the package default.
+  // `null` is a deliberate value rather than a missing one, so it passes
+  // through: `removeOnComplete: null` is how BullMQ is told to keep a job.
+  const merged: Record<string, unknown> = { ...defaults };
+
+  for (const [key, value] of Object.entries(defaultJobOptions ?? {})) {
+    if (value !== undefined) {
+      merged[key] = value;
+    }
+  }
+
+  const queueOptions: QueueOptions = {
+    prefix,
+    connection,
+    defaultJobOptions: merged as JobsOptions,
+    ...(settings === undefined ? {} : { settings }),
+    ...options,
+  };
+
+  const queue = new Queue(name, queueOptions);
+
+  return queue;
+}
+
+export type { QueueOptions, JobsOptions } from "bullmq";
+export { Queue } from "bullmq";