@holo-js/queue-redis 0.1.3

package/dist/index.d.ts

@@ -0,0 +1,143 @@
+import { Job, ConnectionOptions } from 'bullmq';
+import Redis from 'ioredis';
+
+type QueueJsonValue = null | string | number | boolean | readonly QueueJsonValue[] | {
+    readonly [key: string]: QueueJsonValue;
+};
+interface QueueJobEnvelope<TPayload extends QueueJsonValue = QueueJsonValue> {
+    readonly id: string;
+    readonly name: string;
+    readonly connection: string;
+    readonly queue: string;
+    readonly payload: TPayload;
+    readonly attempts: number;
+    readonly maxAttempts: number;
+    readonly createdAt: number;
+    readonly availableAt?: number;
+}
+interface QueueDriverDispatchResult<TResult = unknown> {
+    readonly jobId: string;
+    readonly synchronous: boolean;
+    readonly result?: TResult;
+}
+interface QueueReservedJob<TPayload extends QueueJsonValue = QueueJsonValue> {
+    readonly reservationId: string;
+    readonly reservedAt: number;
+    readonly envelope: QueueJobEnvelope<TPayload>;
+}
+interface QueueReleaseOptions {
+    readonly delaySeconds?: number;
+}
+interface QueueDriverFactoryContext {
+    execute<TPayload extends QueueJsonValue = QueueJsonValue, TResult = unknown>(job: QueueJobEnvelope<TPayload>): Promise<TResult>;
+}
+interface QueueAsyncDriver {
+    readonly name: string;
+    readonly driver: 'redis';
+    readonly mode: 'async';
+    dispatch<TPayload extends QueueJsonValue = QueueJsonValue, TResult = unknown>(job: QueueJobEnvelope<TPayload>): Promise<QueueDriverDispatchResult<TResult>>;
+    reserve<TPayload extends QueueJsonValue = QueueJsonValue>(input: {
+        readonly queueNames?: readonly string[];
+        readonly workerId?: string;
+        readonly timeout?: number;
+    }): Promise<QueueReservedJob<TPayload> | null>;
+    acknowledge(job: QueueReservedJob): Promise<void>;
+    release(job: QueueReservedJob, options?: QueueReleaseOptions): Promise<void>;
+    delete(job: QueueReservedJob): Promise<void>;
+    clear(input?: {
+        readonly queueNames?: readonly string[];
+    }): Promise<number>;
+    close(): Promise<void>;
+}
+interface NormalizedQueueRedisConnectionConfig {
+    readonly name: string;
+    readonly driver: 'redis';
+    readonly connection: string;
+    readonly queue: string;
+    readonly retryAfter: number;
+    readonly blockFor: number;
+    readonly redis: {
+        readonly url?: string;
+        readonly clusters?: readonly {
+            readonly url?: string;
+            readonly host: string;
+            readonly port: number;
+        }[];
+        readonly host: string;
+        readonly port: number;
+        readonly password?: string;
+        readonly username?: string;
+        readonly db: number;
+    };
+}
+interface QueueDriverFactory<TConfig extends NormalizedQueueRedisConnectionConfig = NormalizedQueueRedisConnectionConfig> {
+    readonly driver: TConfig['driver'];
+    create(connection: TConfig, context: QueueDriverFactoryContext): QueueAsyncDriver;
+}
+type RedisQueuedEnvelope = QueueJobEnvelope<QueueJsonValue>;
+type BullJobInstance = Job<RedisQueuedEnvelope, unknown, string>;
+type QueueBullConnection = ConnectionOptions | Redis | InstanceType<(typeof Redis)['Cluster']>;
+type RedisClusterStartupNode = {
+    readonly host: string;
+    readonly port: number;
+    readonly tls?: Record<string, never>;
+};
+declare function normalizeRedisErrorMessage(error: unknown): string;
+declare function isQueueEnvelope(value: unknown): value is QueueJobEnvelope<QueueJsonValue>;
+declare function toRedisSocketPath(value: string): string;
+declare function parseClusterNodeUrl(url: string, label: string): RedisClusterStartupNode;
+declare function resolveClusterStartupNodes(connection: NormalizedQueueRedisConnectionConfig): readonly RedisClusterStartupNode[];
+declare function resolveBullConnectionOptions(connection: NormalizedQueueRedisConnectionConfig): ConnectionOptions;
+declare function resolveBullConnection(connection: NormalizedQueueRedisConnectionConfig): QueueBullConnection;
+declare class RedisQueueDriverError extends Error {
+    constructor(connectionName: string, action: string, cause: unknown);
+}
+declare function wrapRedisError(connectionName: string, action: string, error: unknown): RedisQueueDriverError;
+declare function resolveAttempts(job: BullJobInstance): number;
+declare class RedisQueueDriver implements QueueAsyncDriver {
+    private readonly context;
+    readonly name: string;
+    readonly driver: "redis";
+    readonly mode: "async";
+    private readonly connection;
+    private readonly bullConnection;
+    private readonly managedRedisConnection?;
+    private readonly queues;
+    private readonly workers;
+    private readonly reservations;
+    private queueCursor;
+    constructor(connection: NormalizedQueueRedisConnectionConfig, context: QueueDriverFactoryContext);
+    private getQueue;
+    private getWorker;
+    private normalizeQueueNames;
+    private rotateQueueNames;
+    private createReservedJob;
+    private getReservation;
+    private settleReservation;
+    dispatch<TPayload extends QueueJsonValue = QueueJsonValue, TResult = unknown>(job: QueueJobEnvelope<TPayload>): Promise<QueueDriverDispatchResult<TResult>>;
+    reserve<TPayload extends QueueJsonValue = QueueJsonValue>(input: {
+        readonly queueNames: readonly string[];
+        readonly workerId: string;
+    }): Promise<QueueReservedJob<TPayload> | null>;
+    acknowledge(job: QueueReservedJob): Promise<void>;
+    release(job: QueueReservedJob, options?: QueueReleaseOptions): Promise<void>;
+    delete(job: QueueReservedJob): Promise<void>;
+    clear(input?: {
+        readonly queueNames?: readonly string[];
+    }): Promise<number>;
+    close(): Promise<void>;
+}
+declare const redisQueueDriverFactory: QueueDriverFactory<NormalizedQueueRedisConnectionConfig>;
+declare const redisQueueDriverInternals: {
+    isQueueEnvelope: typeof isQueueEnvelope;
+    normalizeRedisErrorMessage: typeof normalizeRedisErrorMessage;
+    parseClusterNodeUrl: typeof parseClusterNodeUrl;
+    resolveAttempts: typeof resolveAttempts;
+    resolveBullConnection: typeof resolveBullConnection;
+    resolveBullConnectionOptions: typeof resolveBullConnectionOptions;
+    resolveClusterStartupNodes: typeof resolveClusterStartupNodes;
+    toRedisSocketPath: typeof toRedisSocketPath;
+    wrapRedisError: typeof wrapRedisError;
+};
+
+export { type NormalizedQueueRedisConnectionConfig, type QueueAsyncDriver, type QueueDriverDispatchResult, type QueueDriverFactory, type QueueDriverFactoryContext, type QueueJobEnvelope, type QueueJsonValue, type QueueReleaseOptions, type QueueReservedJob, RedisQueueDriver, RedisQueueDriverError, redisQueueDriverFactory, redisQueueDriverInternals };

package/dist/index.mjs

@@ -0,0 +1,382 @@
+// src/index.ts
+import { randomUUID } from "crypto";
+import {
+  Queue as BullQueue,
+  Worker as BullWorker
+} from "bullmq";
+import Redis from "ioredis";
+function normalizeRedisErrorMessage(error) {
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return String(error);
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isQueueEnvelope(value) {
+  return isRecord(value) && typeof value.id === "string" && typeof value.name === "string" && typeof value.connection === "string" && typeof value.queue === "string" && "payload" in value && typeof value.attempts === "number" && Number.isInteger(value.attempts) && value.attempts >= 0 && typeof value.maxAttempts === "number" && Number.isInteger(value.maxAttempts) && value.maxAttempts >= 1 && typeof value.createdAt === "number" && Number.isFinite(value.createdAt) && (typeof value.availableAt === "undefined" || typeof value.availableAt === "number" && Number.isFinite(value.availableAt));
+}
+function isRedisSocketConnectionTarget(value) {
+  return value.startsWith("unix://") || value.startsWith("/");
+}
+function toRedisSocketPath(value) {
+  return value.startsWith("unix://") ? value.slice("unix://".length) : value;
+}
+function parseClusterNodeUrl(url, label) {
+  try {
+    const parsed = new URL(url);
+    if (parsed.protocol !== "redis:" && parsed.protocol !== "rediss:") {
+      throw new Error(`unsupported protocol "${parsed.protocol}"`);
+    }
+    if (!parsed.hostname) {
+      throw new Error("missing hostname");
+    }
+    return {
+      host: parsed.hostname,
+      port: parsed.port ? Number.parseInt(parsed.port, 10) : 6379,
+      ...parsed.protocol === "rediss:" ? { tls: {} } : {}
+    };
+  } catch (error) {
+    throw new Error(`[Holo Queue] ${label} is invalid: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+function resolveClusterStartupNodes(connection) {
+  return (connection.redis.clusters ?? []).map((node, index) => {
+    const label = `Redis queue connection "${connection.name}" cluster node ${index + 1}`;
+    if (typeof node.url === "string") {
+      return parseClusterNodeUrl(node.url, `${label} url`);
+    }
+    if (isRedisSocketConnectionTarget(node.host)) {
+      throw new Error(`[Holo Queue] ${label} cannot use a Unix socket path in Redis cluster mode.`);
+    }
+    return {
+      host: node.host,
+      port: node.port
+    };
+  });
+}
+function resolveBullConnectionOptions(connection) {
+  const redisHost = connection.redis.host;
+  return {
+    ...typeof connection.redis.url === "string" ? { url: connection.redis.url } : typeof redisHost === "string" && isRedisSocketConnectionTarget(redisHost) ? { path: toRedisSocketPath(redisHost) } : {
+      host: redisHost,
+      port: connection.redis.port
+    },
+    username: connection.redis.username,
+    password: connection.redis.password,
+    db: connection.redis.db,
+    maxRetriesPerRequest: null
+  };
+}
+function resolveBullConnection(connection) {
+  if (connection.redis.clusters && connection.redis.clusters.length > 0 && typeof connection.redis.url !== "string") {
+    if (connection.redis.db !== 0) {
+      throw new Error(
+        `[Holo Queue] Redis queue connection "${connection.name}" cannot select redis.db=${connection.redis.db} in cluster mode; Redis Cluster only supports database 0.`
+      );
+    }
+    const startupNodes = [...resolveClusterStartupNodes(connection)];
+    return new Redis.Cluster(startupNodes.map(({ host, port }) => ({ host, port })), {
+      redisOptions: {
+        username: connection.redis.username,
+        password: connection.redis.password,
+        lazyConnect: true,
+        maxRetriesPerRequest: null,
+        ...startupNodes.some((node) => typeof node.tls !== "undefined") ? { tls: {} } : {}
+      }
+    });
+  }
+  if (typeof connection.redis.url === "string") {
+    return new Redis(connection.redis.url, {
+      username: connection.redis.username,
+      password: connection.redis.password,
+      db: connection.redis.db,
+      lazyConnect: true,
+      maxRetriesPerRequest: null,
+      ...connection.redis.url.startsWith("rediss://") ? { tls: {} } : {}
+    });
+  }
+  return resolveBullConnectionOptions(connection);
+}
+var RedisQueueDriverError = class extends Error {
+  constructor(connectionName, action, cause) {
+    super(
+      `[Holo Queue] Redis queue connection "${connectionName}" failed to ${action}: ${normalizeRedisErrorMessage(cause)}`,
+      { cause }
+    );
+    this.name = "RedisQueueDriverError";
+  }
+};
+function wrapRedisError(connectionName, action, error) {
+  if (error instanceof RedisQueueDriverError) {
+    return error;
+  }
+  return new RedisQueueDriverError(connectionName, action, error);
+}
+function resolveAttempts(job) {
+  const attemptsStarted = typeof job.attemptsStarted === "number" && Number.isInteger(job.attemptsStarted) ? job.attemptsStarted : 0;
+  const attemptsMade = typeof job.attemptsMade === "number" && Number.isInteger(job.attemptsMade) ? job.attemptsMade : 0;
+  return Math.max(
+    attemptsStarted > 0 ? attemptsStarted - 1 : 0,
+    attemptsMade,
+    0
+  );
+}
+var RedisQueueDriver = class {
+  constructor(connection, context) {
+    this.context = context;
+    this.name = connection.name;
+    this.connection = connection;
+    const bullConnection = resolveBullConnection(connection);
+    this.bullConnection = bullConnection;
+    this.managedRedisConnection = bullConnection instanceof Redis || bullConnection instanceof Redis.Cluster ? bullConnection : void 0;
+  }
+  name;
+  driver = "redis";
+  mode = "async";
+  connection;
+  bullConnection;
+  managedRedisConnection;
+  queues = /* @__PURE__ */ new Map();
+  workers = /* @__PURE__ */ new Map();
+  reservations = /* @__PURE__ */ new Map();
+  queueCursor = 0;
+  getQueue(queueName) {
+    const cached = this.queues.get(queueName);
+    if (cached) {
+      return cached;
+    }
+    const queue = new BullQueue(queueName, {
+      connection: this.bullConnection,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true
+      }
+    });
+    this.queues.set(queueName, queue);
+    return queue;
+  }
+  async getWorker(queueName) {
+    const cached = this.workers.get(queueName);
+    if (cached) {
+      return cached;
+    }
+    const worker = new BullWorker(
+      queueName,
+      null,
+      {
+        autorun: false,
+        concurrency: 1,
+        connection: this.bullConnection,
+        drainDelay: this.connection.blockFor,
+        lockDuration: this.connection.retryAfter * 1e3,
+        removeOnComplete: { count: 0 },
+        removeOnFail: { count: 0 }
+      }
+    );
+    await worker.waitUntilReady();
+    this.workers.set(queueName, worker);
+    return worker;
+  }
+  normalizeQueueNames(queueNames) {
+    if (!queueNames || queueNames.length === 0) {
+      return [.../* @__PURE__ */ new Set([
+        this.connection.queue,
+        ...this.queues.keys(),
+        ...this.workers.keys()
+      ])];
+    }
+    return [...new Set(queueNames)];
+  }
+  rotateQueueNames(queueNames) {
+    if (queueNames.length <= 1) {
+      return queueNames;
+    }
+    const offset = this.queueCursor % queueNames.length;
+    this.queueCursor = (this.queueCursor + 1) % queueNames.length;
+    return Object.freeze([
+      ...queueNames.slice(offset),
+      ...queueNames.slice(0, offset)
+    ]);
+  }
+  createReservedJob(job, token, queueName) {
+    if (!job.id) {
+      throw new Error("BullMQ returned a reserved job without an id.");
+    }
+    if (!isQueueEnvelope(job.data)) {
+      throw new Error(`BullMQ returned a malformed payload for job "${job.id}".`);
+    }
+    const attempts = resolveAttempts(job);
+    const envelope = Object.freeze({
+      id: job.id,
+      name: job.data.name,
+      connection: job.data.connection,
+      queue: job.data.queue || queueName,
+      payload: job.data.payload,
+      attempts,
+      maxAttempts: job.data.maxAttempts,
+      ...typeof job.data.availableAt === "number" ? { availableAt: job.data.availableAt } : {},
+      createdAt: job.data.createdAt
+    });
+    this.reservations.set(token, {
+      job,
+      token
+    });
+    return {
+      reservationId: token,
+      envelope,
+      reservedAt: Date.now()
+    };
+  }
+  getReservation(reserved) {
+    const reservation = this.reservations.get(reserved.reservationId);
+    if (!reservation) {
+      throw new Error(`Queue reservation "${reserved.reservationId}" is not active.`);
+    }
+    return reservation;
+  }
+  async settleReservation(reserved, action, callback) {
+    const reservation = this.getReservation(reserved);
+    try {
+      await callback(reservation);
+    } catch (error) {
+      throw wrapRedisError(this.name, action, error);
+    } finally {
+      this.reservations.delete(reserved.reservationId);
+    }
+  }
+  async dispatch(job) {
+    try {
+      const delay = typeof job.availableAt === "number" ? Math.max(job.availableAt - Date.now(), 0) : void 0;
+      const queued = await this.getQueue(job.queue).add(job.name, job, {
+        attempts: job.maxAttempts,
+        ...typeof delay === "number" ? { delay } : {},
+        jobId: job.id,
+        removeOnComplete: true,
+        removeOnFail: true,
+        timestamp: job.createdAt
+      });
+      return {
+        jobId: queued.id ?? job.id,
+        synchronous: false
+      };
+    } catch (error) {
+      throw wrapRedisError(this.name, "enqueue job", error);
+    }
+  }
+  async reserve(input) {
+    try {
+      const queueNames = this.rotateQueueNames(this.normalizeQueueNames(input.queueNames));
+      for (const queueName of queueNames) {
+        const token2 = `${input.workerId}:${randomUUID()}`;
+        const worker2 = await this.getWorker(queueName);
+        const job2 = await worker2.getNextJob(token2, { block: false });
+        if (job2) {
+          return this.createReservedJob(job2, token2, queueName);
+        }
+      }
+      const [blockingQueue] = queueNames;
+      if (!blockingQueue || this.connection.blockFor <= 0) {
+        return null;
+      }
+      const token = `${input.workerId}:${randomUUID()}`;
+      const worker = await this.getWorker(blockingQueue);
+      const job = await worker.getNextJob(token, { block: true });
+      if (!job) {
+        return null;
+      }
+      return this.createReservedJob(job, token, blockingQueue);
+    } catch (error) {
+      throw wrapRedisError(this.name, "reserve job", error);
+    }
+  }
+  async acknowledge(job) {
+    await this.settleReservation(job, "acknowledge job", async (reservation) => {
+      await reservation.job.moveToCompleted(null, reservation.token, false);
+    });
+  }
+  async release(job, options) {
+    await this.settleReservation(job, "release job", async (reservation) => {
+      if (typeof options?.delaySeconds === "number" && options.delaySeconds > 0) {
+        await reservation.job.moveToDelayed(Date.now() + options.delaySeconds * 1e3, reservation.token);
+        return;
+      }
+      await reservation.job.moveToWait(reservation.token);
+    });
+  }
+  async delete(job) {
+    await this.settleReservation(job, "delete job", async (reservation) => {
+      reservation.job.discard();
+      await reservation.job.moveToFailed(new Error("[Holo Queue] Job deleted."), reservation.token, false);
+    });
+  }
+  async clear(input) {
+    try {
+      const queueNames = this.normalizeQueueNames(input?.queueNames);
+      let cleared = 0;
+      for (const queueName of queueNames) {
+        const queue = this.getQueue(queueName);
+        cleared += await queue.getJobCountByTypes("wait", "waiting", "paused", "prioritized", "delayed");
+        await queue.drain(true);
+      }
+      return cleared;
+    } catch (error) {
+      throw wrapRedisError(this.name, "clear queued jobs", error);
+    }
+  }
+  async close() {
+    const resources = [
+      ...this.workers.values(),
+      ...this.queues.values()
+    ];
+    this.reservations.clear();
+    this.workers.clear();
+    this.queues.clear();
+    let closeRejection;
+    try {
+      const results = await Promise.allSettled(resources.map(async (resource) => {
+        if (resource instanceof BullWorker) {
+          await resource.close(true);
+          return;
+        }
+        await resource.close();
+      }));
+      closeRejection = results.find((result) => result.status === "rejected");
+    } finally {
+      if (this.managedRedisConnection) {
+        try {
+          await this.managedRedisConnection.quit();
+        } catch {
+          this.managedRedisConnection.disconnect();
+        }
+      }
+    }
+    if (closeRejection) {
+      throw wrapRedisError(this.name, "close driver", closeRejection.reason);
+    }
+  }
+};
+var redisQueueDriverFactory = {
+  driver: "redis",
+  create(connection, context) {
+    return new RedisQueueDriver(connection, context);
+  }
+};
+var redisQueueDriverInternals = {
+  isQueueEnvelope,
+  normalizeRedisErrorMessage,
+  parseClusterNodeUrl,
+  resolveAttempts,
+  resolveBullConnection,
+  resolveBullConnectionOptions,
+  resolveClusterStartupNodes,
+  toRedisSocketPath,
+  wrapRedisError
+};
+export {
+  RedisQueueDriver,
+  RedisQueueDriverError,
+  redisQueueDriverFactory,
+  redisQueueDriverInternals
+};

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@holo-js/queue-redis",
+  "version": "0.1.3",
+  "description": "Holo-JS Framework - Redis queue driver",
+  "type": "module",
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "stub": "tsup",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "vitest --run"
+  },
+  "peerDependencies": {
+    "@holo-js/queue": "^0.1.3"
+  },
+  "dependencies": {
+    "bullmq": "^5.71.0",
+    "ioredis": "catalog:",
+    "tslib": "^2.8.1"
+  },
+  "devDependencies": {
+    "@holo-js/queue": "workspace:*",
+    "@types/node": "^22.10.2",
+    "tsup": "^8.3.5",
+    "typescript": "^5.7.2",
+    "vitest": "^2.1.8"
+  }
+}