@treatwell/moleculer-essentials 1.0.0

package/dist/mixins/queue.mixin.d.mts

@@ -0,0 +1,150 @@
+import { a as CustomServiceSchema } from '../context-factory-BWO3xPWE.mjs';
+import { Redis } from 'ioredis';
+import { QueueOptions, Queue, JobsOptions, Job, QueueEventsOptions, QueueEvents, QueueBaseOptions, FlowProducer, RepeatOptions, WorkerOptions, Worker } from 'bullmq';
+import { Service } from 'moleculer';
+import 'bson';
+import 'zod/v4';
+import 'ajv/dist/2019.js';
+
+type QueueMixinOptions = {
+    brokerURL: string | (<TService extends Service = Service>(svc: TService) => Promise<string>);
+};
+/**
+ * connection is mandatory for BullMQ but is managed by each mixins.
+ */
+type WithoutConnection<T> = Omit<T, 'connection'>;
+
+/**
+ * This Mixin add the capability to launch a job.
+ */
+declare function QueueClient<N extends string>(queueName: N, opts: WithoutConnection<QueueOptions> & QueueMixinOptions): Partial<CustomServiceSchema<unknown, {
+    _getQueues(): Map<string, Queue>;
+    /**
+     * Add a job to a specific queue. The queue name needs to be passed as parameters
+     * to enable multiple queue clients in the same service.
+     */
+    addJob<D = unknown>(qName: N, name: string, data: D, jOpts?: JobsOptions): Promise<Job>;
+    /**
+     * Add jobs to a specific queue. The queue name needs to be passed as parameters
+     * to enable multiple queue clients in the same service.
+     */
+    addBulkJob<D = unknown>(qName: N, name: string, data: D[], jOpts?: JobsOptions): Promise<Job[]>;
+    /**
+     * Get a queue bundle if you need specific features
+     */
+    getQueue(qName: N): Queue;
+}, Partial<CustomServiceSchema<unknown, {
+    getStore(storeName: string): Map<string, {
+        services: Set<unknown>;
+        client: Redis;
+        onClose: () => Promise<void> | void;
+    }>;
+    getFromStore(storeName: string, key: string): Redis | null;
+    removeServiceFromStore(storeName: string, key: string): Promise<boolean>;
+    setClientToStore(storeName: string, key: string, client: Redis, onClose: () => Promise<void> | void): void;
+}, unknown, unknown>>[], unknown>>;
+
+/**
+ * This Mixin add the capability to wait on a job.
+ * WARNING: Needs to be used with QueueClient mixin.
+ */
+declare function QueueEventsClient<N extends string>(queueName: N, opts: WithoutConnection<QueueEventsOptions> & QueueMixinOptions): Partial<CustomServiceSchema<unknown, {
+    _getQueueEvents(): Map<string, {
+        events: QueueEvents;
+        running: boolean;
+    }>;
+    /**
+     * Return the QueueEvents instance for the given queue name.
+     * Will run the QueueEvents if it's not already running (lazy connect).
+     * Will throw an error if the QueueEvents is not found.
+     */
+    getQueueEvents(qName: N): QueueEvents;
+    /**
+     * Same as addJob but will also wait for the job to finish before returning.
+     */
+    addAndWait<T = unknown, D = unknown>(qName: N, name: string, data: D, jOpts?: JobsOptions, ttl?: number): Promise<T>;
+    /**
+     * Same as addBulkJob but will also wait for the jobs to finish before returning.
+     */
+    addBulkAndWait<T = unknown, D = unknown>(qName: N, name: string, data: D[], jOpts?: JobsOptions, ttl?: number): Promise<T[]>;
+}, Partial<CustomServiceSchema<unknown, {
+    getStore(storeName: string): Map<string, {
+        services: Set<unknown>;
+        client: Redis;
+        onClose: () => Promise<void> | void;
+    }>;
+    getFromStore(storeName: string, key: string): Redis | null;
+    removeServiceFromStore(storeName: string, key: string): Promise<boolean>;
+    setClientToStore(storeName: string, key: string, client: Redis, onClose: () => Promise<void> | void): void;
+}, unknown, unknown>>[], unknown>>;
+
+/**
+ * This Mixin add the capability to launch a BullMQ flows.
+ */
+declare function QueueFlowProducerMixin(opts: WithoutConnection<QueueBaseOptions> & QueueMixinOptions): Partial<CustomServiceSchema<unknown, {
+    getFlowProducer(): FlowProducer;
+}, unknown, unknown>>;
+
+type RepeatableJob = {
+    name: string;
+    data?: unknown;
+} & RepeatOptions;
+type QueueStaticRepeatableJobsOptions = {
+    /**
+     * Set to `false` to prevent deleting repeatable jobs not registered.
+     */
+    autoRemove?: boolean;
+};
+/**
+ * This Mixin allow to register repeatable jobs and remove other ones.
+ * It should mainly be setup on the same service as the related QueueWorker.
+ */
+declare function QueueStaticRepeatableJobs(queueName: string, jobs: RepeatableJob[], opts: QueueStaticRepeatableJobsOptions & QueueMixinOptions): Partial<CustomServiceSchema<unknown, {
+    registerRepeatableJobs(): Promise<void>;
+}, Partial<CustomServiceSchema<unknown, {
+    getStore(storeName: string): Map<string, {
+        services: Set<unknown>;
+        client: Redis;
+        onClose: () => Promise<void> | void;
+    }>;
+    getFromStore(storeName: string, key: string): Redis | null;
+    removeServiceFromStore(storeName: string, key: string): Promise<boolean>;
+    setClientToStore(storeName: string, key: string, client: Redis, onClose: () => Promise<void> | void): void;
+}, unknown, unknown>>[], unknown>>;
+
+type JobMeta = {
+    job?: Job;
+    jobWorkerToken?: string;
+};
+type JobProcessorOptions = {
+    /**
+     * Use the job name as the action name.
+     * When false (default), the action name is 'processor'.
+     */
+    useNamedFunctions?: boolean;
+    /**
+     * Skip normal info logs at the start and end of the job.
+     * Will never disable logs on errors.
+     */
+    skipNormalLogs?: boolean;
+    /**
+     * Allow to log additional fields from the job data.
+     * Currently, doesn't support nested fields.
+     */
+    logDataFields?: string[];
+    /**
+     * Optional wrapper that MUST call the passed function.
+     */
+    wrapper?: (job: Job, token: string | undefined, fn: (job: Job, token?: string) => Promise<unknown>) => Promise<unknown>;
+};
+/**
+ * This Mixin create the worker system.
+ */
+declare function QueueWorker(queueName: string, opts: WithoutConnection<WorkerOptions> & QueueMixinOptions, processorOptions?: JobProcessorOptions): Partial<CustomServiceSchema<unknown, {
+    getWorker(): Worker;
+    processJob(job: Job, token?: string): Promise<any>;
+    _processJob(job: Job, token?: string): Promise<any>;
+}, unknown, unknown>>;
+
+export { QueueClient, QueueEventsClient, QueueFlowProducerMixin, QueueStaticRepeatableJobs, QueueWorker };
+export type { JobMeta, JobProcessorOptions, QueueMixinOptions, QueueStaticRepeatableJobsOptions, RepeatableJob };

package/dist/mixins/queue.mixin.mjs

@@ -0,0 +1,414 @@
+import { Errors } from 'moleculer';
+import { Queue, QueueEvents, FlowProducer, Worker } from 'bullmq';
+import { isFunction } from 'lodash';
+import { w as wrapMixin } from '../index-DNJWwcZu.mjs';
+import { GlobalStoreMixin } from './global-store.mixin.mjs';
+import { Redis } from 'ioredis';
+import { parseURL } from 'ioredis/built/utils';
+
+function createRedisConnection(url) {
+  let tls;
+  if (url.startsWith("rediss://")) {
+    tls = {};
+  }
+  const connection = new Redis({
+    ...parseURL(url),
+    tls,
+    maxRetriesPerRequest: null,
+    connectTimeout: 1e3 * 60
+  });
+  connection.setMaxListeners(0);
+  return connection;
+}
+
+function QueueClient(queueName, opts) {
+  const kKey = Symbol("Queue Client Key");
+  return wrapMixin({
+    mixins: [GlobalStoreMixin()],
+    methods: {
+      _getQueues() {
+        return this.$queues;
+      },
+      /**
+       * Add a job to a specific queue. The queue name needs to be passed as parameters
+       * to enable multiple queue clients in the same service.
+       */
+      addJob(qName, name, data, jOpts) {
+        const q = this._getQueues().get(qName);
+        if (!q) {
+          throw new Errors.MoleculerServerError(
+            `Queue '${qName}' does not exists on '${this.name}'`
+          );
+        }
+        return q.add(name, data, jOpts);
+      },
+      /**
+       * Add jobs to a specific queue. The queue name needs to be passed as parameters
+       * to enable multiple queue clients in the same service.
+       */
+      addBulkJob(qName, name, data, jOpts) {
+        const q = this._getQueues().get(qName);
+        if (!q) {
+          throw new Errors.MoleculerServerError(
+            `Queue '${qName}' does not exists on '${this.name}'`
+          );
+        }
+        return q.addBulk(data.map((d) => ({ name, data: d, opts: jOpts })));
+      },
+      /**
+       * Get a queue bundle if you need specific features
+       */
+      getQueue(qName) {
+        const q = this._getQueues().get(qName);
+        if (!q) {
+          throw new Errors.MoleculerServerError(
+            `Queue '${qName}' does not exists on '${this.name}'`
+          );
+        }
+        return q;
+      }
+    },
+    created() {
+      if (!this.$queues) {
+        this.$queues = /* @__PURE__ */ new Map();
+      }
+    },
+    async started() {
+      if (this._getQueues().has(queueName)) {
+        throw new Error(
+          `Queue client '${queueName}' already exists on service '${this.name}'`
+        );
+      }
+      const key = isFunction(opts.brokerURL) ? await opts.brokerURL(this) : opts.brokerURL;
+      this[kKey] = key;
+      let connection = this.getFromStore("redis", key);
+      if (!connection) {
+        connection = createRedisConnection(key);
+        this.setClientToStore(
+          "redis",
+          key,
+          connection,
+          () => connection?.disconnect()
+        );
+      }
+      const queue = new Queue(queueName, { connection, ...opts });
+      queue.setMaxListeners(0);
+      queue.on(
+        "error",
+        (err) => this.logger.error(`Queue ${queueName} got error`, { err })
+      );
+      this._getQueues().set(queueName, queue);
+    },
+    async stopped() {
+      const q = this._getQueues().get(queueName);
+      if (q) {
+        await q.close();
+      }
+      await this.removeServiceFromStore("redis", this[kKey]);
+    }
+  });
+}
+
+function QueueEventsClient(queueName, opts) {
+  const kKey = Symbol("Queue Events Client Key");
+  return wrapMixin({
+    mixins: [GlobalStoreMixin()],
+    methods: {
+      _getQueueEvents() {
+        return this.$queuesEvents;
+      },
+      /**
+       * Return the QueueEvents instance for the given queue name.
+       * Will run the QueueEvents if it's not already running (lazy connect).
+       * Will throw an error if the QueueEvents is not found.
+       */
+      getQueueEvents(qName) {
+        const queueEvents = this._getQueueEvents().get(qName);
+        if (!queueEvents) {
+          throw new Error(`QueueEvents '${qName}' not found`);
+        }
+        const qEvents = queueEvents.events;
+        if (!queueEvents.running) {
+          qEvents.run().catch((error) => qEvents.emit("error", error));
+          queueEvents.running = true;
+        }
+        return qEvents;
+      },
+      /**
+       * Same as addJob but will also wait for the job to finish before returning.
+       */
+      async addAndWait(qName, name, data, jOpts, ttl) {
+        const job = await this.addJob(qName, name, data, jOpts);
+        return job.waitUntilFinished(this.getQueueEvents(qName), ttl);
+      },
+      /**
+       * Same as addBulkJob but will also wait for the jobs to finish before returning.
+       */
+      async addBulkAndWait(qName, name, data, jOpts, ttl) {
+        const jobs = await this.addBulkJob(qName, name, data, jOpts);
+        const q = this.getQueueEvents(qName);
+        return Promise.all(jobs.map((j) => j.waitUntilFinished(q, ttl)));
+      }
+    },
+    created() {
+      if (!this.$queuesEvents) {
+        this.$queuesEvents = /* @__PURE__ */ new Map();
+      }
+    },
+    async started() {
+      if (this._getQueueEvents().get(queueName)) {
+        throw new Error(
+          `Queue client '${queueName}' already exists on service '${this.name}'`
+        );
+      }
+      const key = isFunction(opts.brokerURL) ? await opts.brokerURL(this) : opts.brokerURL;
+      this[kKey] = key;
+      let connection = this.getFromStore("redis", key);
+      if (!connection) {
+        connection = createRedisConnection(key);
+        this.setClientToStore(
+          "redis",
+          key,
+          connection,
+          () => connection?.disconnect()
+        );
+      }
+      const qEvents = new QueueEvents(queueName, {
+        connection,
+        ...opts,
+        autorun: false
+      });
+      qEvents.setMaxListeners(0);
+      qEvents.on(
+        "error",
+        (err) => this.logger.error(`QueueEvents on ${queueName} got error`, { err })
+      );
+      this._getQueueEvents().set(queueName, {
+        events: qEvents,
+        running: false
+      });
+    },
+    async stopped() {
+      const q = this._getQueueEvents().get(queueName);
+      if (q) {
+        await q.events.close();
+      }
+      await this.removeServiceFromStore("redis", this[kKey]);
+    }
+  });
+}
+
+const kConnection$1 = Symbol("Queue FlowProducer Connection");
+function QueueFlowProducerMixin(opts) {
+  return wrapMixin({
+    methods: {
+      getFlowProducer() {
+        return this.$flowProducer;
+      }
+    },
+    async started() {
+      if (this.$flowProducer) {
+        throw new Error(
+          `Queue Flow producer already exists on service '${this.name}'`
+        );
+      }
+      const url = isFunction(opts.brokerURL) ? await opts.brokerURL(this) : opts.brokerURL;
+      const connection = createRedisConnection(url);
+      this.$flowProducer = new FlowProducer({ connection, ...opts });
+      this[kConnection$1] = connection;
+    },
+    async stopped() {
+      await this.$flowProducer?.close();
+      this[kConnection$1]?.disconnect();
+    }
+  });
+}
+
+function isPropertyEqual(a, b) {
+  if (a && b && a === b) {
+    return true;
+  }
+  return !a && !b;
+}
+function isSameRepeatableJob(repeatableJob, job) {
+  return isPropertyEqual(repeatableJob.name, job.name) && isPropertyEqual(repeatableJob.pattern, job.pattern) && isPropertyEqual(repeatableJob.every, job.every?.toString()) && isPropertyEqual(repeatableJob.tz, job.tz);
+}
+function getFilteredJobs(qName, jobs) {
+  const allowList = new Set(
+    process.env.SCHEDULER_JOBS_ALLOWLIST?.split(",") || []
+  );
+  const denyList = new Set(
+    process.env.SCHEDULER_JOBS_DENYLIST?.split(",") || []
+  );
+  return jobs.filter((j) => {
+    if (allowList.size > 0) {
+      return allowList.has(`${qName}:${j.name}`);
+    }
+    if (denyList.size > 0) {
+      return !denyList.has(`${qName}:${j.name}`);
+    }
+    return true;
+  });
+}
+const mixinStore = Symbol("QueueStaticRepeatableJobsMixinStore");
+function QueueStaticRepeatableJobs(queueName, jobs, opts) {
+  const kKey = Symbol("Queue Static Repeatable Jobs Key");
+  return wrapMixin({
+    mixins: [GlobalStoreMixin()],
+    methods: {
+      async registerRepeatableJobs() {
+        const key = isFunction(opts.brokerURL) ? await opts.brokerURL(this) : opts.brokerURL;
+        this[kKey] = key;
+        let connection = this.getFromStore("redis", key);
+        if (!connection) {
+          connection = createRedisConnection(key);
+          this.setClientToStore(
+            "redis",
+            key,
+            connection,
+            () => connection?.disconnect()
+          );
+        }
+        const queue = new Queue(queueName, { connection });
+        const filteredJobs = getFilteredJobs(queueName, jobs);
+        await Promise.all(
+          filteredJobs.map(
+            ({ name, data, ...repeat }) => queue.add(name, data, { repeat, removeOnComplete: true })
+          )
+        );
+        if (opts.autoRemove !== false) {
+          const repeatableJobs = await queue.getRepeatableJobs();
+          const rjToRemove = repeatableJobs.filter(
+            (rj) => !filteredJobs.some((j) => isSameRepeatableJob(rj, j))
+          );
+          if (rjToRemove.length) {
+            await Promise.all(
+              rjToRemove.map((rj) => queue.removeRepeatableByKey(rj.key))
+            );
+          }
+        }
+        await queue.close();
+        await this.removeServiceFromStore("redis", key);
+      }
+    },
+    created() {
+      if (!this[mixinStore]) {
+        this[mixinStore] = /* @__PURE__ */ new Set();
+      }
+      if (this[mixinStore].has(this[kKey])) {
+        throw new Error(
+          `Queue ${queueName} has already registered static repeatable jobs`
+        );
+      }
+      this[mixinStore].add(queueName);
+    },
+    events: {
+      // TODO Improve this to only be run once in a while globally on the cluster
+      "$broker.started": {
+        async handler() {
+          await this.registerRepeatableJobs();
+        }
+      }
+    }
+  });
+}
+
+const kConnection = Symbol("Queue Worker Connection");
+function QueueWorker(queueName, opts, processorOptions = {}) {
+  return wrapMixin({
+    metadata: {
+      worker: true,
+      [`worker-${queueName}`]: true
+    },
+    methods: {
+      getWorker() {
+        return this.$worker;
+      },
+      async processJob(job, token) {
+        if (processorOptions.wrapper) {
+          return processorOptions.wrapper(
+            job,
+            token,
+            this._processJob.bind(this)
+          );
+        }
+        return this._processJob(job, token);
+      },
+      async _processJob(job, token) {
+        const methodName = processorOptions.useNamedFunctions ? job.name : "processor";
+        if (!this.actions[methodName]) {
+          throw new Error(`action '${methodName}' does not exist.`);
+        }
+        const logBase = {
+          msg: `Job ${job.name} on ${queueName} (${job.id})`,
+          name: job.name,
+          id: job.id
+        };
+        if (processorOptions.logDataFields) {
+          processorOptions.logDataFields.forEach((field) => {
+            logBase[field] = job.data?.[field];
+          });
+        }
+        if (processorOptions.skipNormalLogs !== true) {
+          this.logger.info(logBase);
+        }
+        const start = Date.now();
+        try {
+          const res = await this.actions[methodName](job.data, {
+            meta: { job, jobWorkerToken: token }
+          });
+          if (processorOptions.skipNormalLogs !== true) {
+            this.logger.info({
+              ...logBase,
+              msg: `${logBase.msg} succeeded`,
+              duration: Date.now() - start
+            });
+          }
+          return res;
+        } catch (err) {
+          this.logger.info({
+            ...logBase,
+            msg: `${logBase.msg} failed`,
+            duration: Date.now() - start,
+            err
+          });
+          throw err;
+        }
+      }
+    },
+    async started() {
+      if (this.$worker) {
+        throw new Error(
+          `A QueueWorker mixin was already on service '${this.name}'`
+        );
+      }
+      const url = isFunction(opts.brokerURL) ? await opts.brokerURL(this) : opts.brokerURL;
+      const connection = createRedisConnection(url);
+      const worker = new Worker(queueName, this.processJob.bind(this), {
+        connection,
+        ...opts,
+        autorun: false
+      });
+      worker.on(
+        "error",
+        (err) => this.logger.error(`Worker on ${queueName} got error`, { err })
+      );
+      this.$worker = worker;
+      this[kConnection] = connection;
+    },
+    events: {
+      "$broker.started": {
+        handler() {
+          const worker = this.getWorker();
+          worker.run().catch((error) => worker.emit("error", error));
+        }
+      }
+    },
+    async stopped() {
+      await this.$worker?.close();
+      this[kConnection]?.disconnect();
+    }
+  });
+}
+
+export { QueueClient, QueueEventsClient, QueueFlowProducerMixin, QueueStaticRepeatableJobs, QueueWorker };

package/dist/mixins/redis.mixin.cjs

@@ -0,0 +1,50 @@
+'use strict';
+
+var ioredis = require('ioredis');
+var mixins_globalStore_mixin = require('./global-store.mixin.cjs');
+var index = require('../index-82e1CXJX.cjs');
+
+function RedisMixin(options, { reuseClient = true, getOptions } = {}) {
+  const kKey = Symbol("Redis Key Symbol");
+  return index.wrapMixin({
+    mixins: [mixins_globalStore_mixin.GlobalStoreMixin()],
+    methods: {
+      getRedis() {
+        return this.redis;
+      }
+    },
+    async started() {
+      const finalOptions = {
+        ...options,
+        ...await getOptions?.(this)
+      };
+      const key = [
+        finalOptions.host,
+        finalOptions.port,
+        finalOptions.db,
+        finalOptions.username,
+        finalOptions.path,
+        reuseClient ? "" : Math.random()
+      ].filter(Boolean).join("|");
+      this[kKey] = key;
+      this.redis = this.getFromStore("redis", key);
+      if (!this.redis) {
+        this.redis = new ioredis.Redis({
+          ...finalOptions,
+          lazyConnect: true
+        });
+        this.setClientToStore(
+          "redis",
+          key,
+          this.redis,
+          () => this.redis?.disconnect()
+        );
+      }
+    },
+    async stopped() {
+      await this.removeServiceFromStore("redis", this[kKey]);
+    }
+  });
+}
+
+exports.RedisMixin = RedisMixin;

package/dist/mixins/redis.mixin.d.cts

@@ -0,0 +1,27 @@
+import { a as CustomServiceSchema } from '../context-factory-BWO3xPWE.cjs';
+import { RedisOptions, Redis } from 'ioredis';
+import { Service } from 'moleculer';
+import 'bson';
+import 'zod/v4';
+import 'ajv/dist/2019.js';
+
+type AllowedOptions = Omit<RedisOptions, 'lazyConnect'>;
+type RedisMixinOptions = {
+    reuseClient?: boolean;
+    getOptions?: <TService extends Service = Service>(svc: TService) => Promise<AllowedOptions>;
+};
+declare function RedisMixin(options: AllowedOptions, { reuseClient, getOptions }?: RedisMixinOptions): Partial<CustomServiceSchema<unknown, {
+    getRedis(): Redis;
+}, Partial<CustomServiceSchema<unknown, {
+    getStore(storeName: string): Map<string, {
+        services: Set<unknown>;
+        client: Redis;
+        onClose: () => Promise<void> | void;
+    }>;
+    getFromStore(storeName: string, key: string): Redis | null;
+    removeServiceFromStore(storeName: string, key: string): Promise<boolean>;
+    setClientToStore(storeName: string, key: string, client: Redis, onClose: () => Promise<void> | void): void;
+}, unknown, unknown>>[], unknown>>;
+
+export { RedisMixin };
+export type { RedisMixinOptions };

package/dist/mixins/redis.mixin.d.mts

@@ -0,0 +1,27 @@
+import { a as CustomServiceSchema } from '../context-factory-BWO3xPWE.mjs';
+import { RedisOptions, Redis } from 'ioredis';
+import { Service } from 'moleculer';
+import 'bson';
+import 'zod/v4';
+import 'ajv/dist/2019.js';
+
+type AllowedOptions = Omit<RedisOptions, 'lazyConnect'>;
+type RedisMixinOptions = {
+    reuseClient?: boolean;
+    getOptions?: <TService extends Service = Service>(svc: TService) => Promise<AllowedOptions>;
+};
+declare function RedisMixin(options: AllowedOptions, { reuseClient, getOptions }?: RedisMixinOptions): Partial<CustomServiceSchema<unknown, {
+    getRedis(): Redis;
+}, Partial<CustomServiceSchema<unknown, {
+    getStore(storeName: string): Map<string, {
+        services: Set<unknown>;
+        client: Redis;
+        onClose: () => Promise<void> | void;
+    }>;
+    getFromStore(storeName: string, key: string): Redis | null;
+    removeServiceFromStore(storeName: string, key: string): Promise<boolean>;
+    setClientToStore(storeName: string, key: string, client: Redis, onClose: () => Promise<void> | void): void;
+}, unknown, unknown>>[], unknown>>;
+
+export { RedisMixin };
+export type { RedisMixinOptions };